Skip to content
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
0c23be0
feat(python): backend-agnostic native graph + Router (unification pro…
YangXu1990uiuc Jul 1, 2026
5eff162
refactor(python): trim NodeType to exercised ops; doc mixed candidate…
YangXu1990uiuc Jul 1, 2026
24b0ff8
refactor(python): drop BATCHNORM / BATCHNORM_INFERENCE from NodeType …
YangXu1990uiuc Jul 1, 2026
6641f14
refactor(python): remove conv ops from native graph (unused foundation)
YangXu1990uiuc Jul 1, 2026
7d1b7e2
docs(python): use generic 'python DSLs' for backend examples
YangXu1990uiuc Jul 1, 2026
b91c7f7
refactor(python): unify engines into one flat engine-id space (no cuD…
YangXu1990uiuc Jul 2, 2026
cbcad29
feat(python): make cudnn.pygraph engine-aware in place (transparent f…
YangXu1990uiuc Jul 2, 2026
d996a12
feat(python): native GEMM-family lowering + fix cuDNN execute path (p…
YangXu1990uiuc Jul 2, 2026
92dd12e
feat(python): reduction output-shape + SF reordering lowering (GEMM-f…
YangXu1990uiuc Jul 2, 2026
a93c78c
feat(python): native block-scale (nvfp4) lowering on Blackwell + fixe…
YangXu1990uiuc Jul 2, 2026
0858c57
feat(python): native moe_grouped_matmul lowering + parity (GEMM-famil…
YangXu1990uiuc Jul 2, 2026
fb5c9bf
fix(python): IR-uid -> C++-uid translation at execute; native rmsnorm…
YangXu1990uiuc Jul 2, 2026
d2ce901
refactor(python): Python IR owns the uid namespace end to end
YangXu1990uiuc Jul 2, 2026
6ff43ce
feat(python): full pointwise coverage — 54 ops, table-driven, mode ==…
YangXu1990uiuc Jul 2, 2026
1735ff3
feat(python): norm family via one declarative table (10 ops, generic …
YangXu1990uiuc Jul 2, 2026
e81bba0
feat(python): conv + structural ops; collapse ALL structured ops into…
YangXu1990uiuc Jul 2, 2026
80f5b33
feat(python): sdpa family via generic kwarg capture — full ~130-arg s…
YangXu1990uiuc Jul 2, 2026
5af82ce
feat(python)!: THE FLIP — cudnn.pygraph is now the Python graph class
YangXu1990uiuc Jul 2, 2026
68c6d8e
fix(python): classic validate() timing + omit unset compute_data_type
YangXu1990uiuc Jul 2, 2026
19927ed
docs(router): codify the extension contract for the future heuristics MR
YangXu1990uiuc Jul 2, 2026
cd0f5cf
fix(python): plan-selection lifecycle + registration validation (revi…
YangXu1990uiuc Jul 2, 2026
8510659
feat(python): compiled-plan engine lifecycle + ExecutionContext (revi…
YangXu1990uiuc Jul 2, 2026
d0c5e8b
fix(python): IR port direction, tensor identity ownership, parity gap…
YangXu1990uiuc Jul 2, 2026
b351498
fix(python): address coderabbit inline findings (broadcast checks, cu…
YangXu1990uiuc Jul 2, 2026
203852a
fix(python): review follow-up — replan invalidation, slot-based dispa…
YangXu1990uiuc Jul 2, 2026
83ffded
refactor(python): one-shot planning (classic conformance) + retire Na…
YangXu1990uiuc Jul 2, 2026
a5ad541
fix(python): stable two-level plan indices; land the two missed patch…
YangXu1990uiuc Jul 2, 2026
9c373d4
fix: callback graph shim for score_mod closures; serialize returns cl…
YangXu1990uiuc Jul 2, 2026
d5fc9b9
fix(python): review round 4 — explicit planning state, split plan-ind…
YangXu1990uiuc Jul 3, 2026
02d6455
fix(python): push ragged_offset_multiplier on output tensors at lowering
YangXu1990uiuc Jul 3, 2026
229528b
fix(python): push reordering_type on output tensors; consolidate outp…
YangXu1990uiuc Jul 3, 2026
d3a624f
fix(python): whole-surface freeze + output layout contract; split cuT…
YangXu1990uiuc Jul 3, 2026
401ce73
refactor(python): retire pygraph name collision; drop NativeGraph; ch…
YangXu1990uiuc Jul 3, 2026
921dcf5
refactor(python): 'cudnn' never means 'the backend' in names — both s…
YangXu1990uiuc Jul 3, 2026
8862efa
fix(python): classic-parity batch from internal CI — signatures, wrap…
YangXu1990uiuc Jul 6, 2026
7ad1610
chore: remove internal test file accidentally included
YangXu1990uiuc Jul 6, 2026
701d9fe
fix(python): renames are label writes (exempt from freeze); push outp…
YangXu1990uiuc Jul 6, 2026
f8774d8
test: skip introspection/one-shot tests when cudnn.pygraph is monkey-…
YangXu1990uiuc Jul 7, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -76,3 +76,7 @@ cutedsl = [
"apache-tvm-ffi",
"torch-c-dlpack-ext",
]
cutile = [
"cuda-tile",
"cuda-python",
]
16 changes: 12 additions & 4 deletions python/cudnn/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ def is_windows():
"data_type",
"tensor_reordering",
"heur_mode",
"pygraph",
"tensor",
"knob",
"cudnnGraphNotSupportedError",
Expand Down Expand Up @@ -108,7 +107,7 @@ def _set_data_type(


_pybind_module.tensor.set_data_type = _set_data_type
pygraph.tensor = _tensor
_pybind_module.pygraph.tensor = _tensor


def _library_device_pointer(input_tensor):
Expand Down Expand Up @@ -194,8 +193,8 @@ def _execute_plan_at_index(
)


pygraph.execute = _execute
pygraph.execute_plan_at_index = _execute_plan_at_index
_pybind_module.pygraph.execute = _execute
_pybind_module.pygraph.execute_plan_at_index = _execute_plan_at_index


def load_cudnn():
Expand Down Expand Up @@ -255,6 +254,15 @@ def _dlopen_cudnn():
else:
_dlopen_cudnn()

# The graph API: a Python-native IR with pluggable execution backends. The
# public ``cudnn.pygraph`` IS the Python class; the C++ graph builder stays
# internal at ``cudnn._pybind_module.pygraph`` and is reached only through
# lowering (a graph is pure-Python or pure-C++, never mixed). Imported before
# .graph/.wrapper, which reference cudnn.pygraph at module load.
from .graph_types import NodeType, Tensor
from .pygraph import pygraph, NativeGraph, GraphContext
from .nodes import Node

from .graph import graph, jit, graph_cache
from .wrapper import Graph

Expand Down
35 changes: 35 additions & 0 deletions python/cudnn/engines/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
"""Execution backends for NativeGraph.

Pluggable execution backends in one flat engine-id space with the cuDNN backend.
The Router builds a ranked plan list at ``create_execution_plans()`` time; graph
construction stays backend-agnostic. See ``docs/python_native_graph_router.md``.

Backends:
- ReferenceMatmulEngine: pure-PyTorch correctness oracle (CPU/GPU, no JIT deps)
- MatmulCuTileEngine: NVIDIA cuTile matmul (Blackwell SM100+); optional deps
"""

from .base import BaseEngine
from .engine_ids import PYTHON_ENGINE_ID_BASE, CUDNN_HEURISTIC_ENGINE_ID, is_python_engine
from .router import Router, PlanConfig, default_router
from .reference_matmul_engine import ReferenceMatmulEngine

__all__ = [
"BaseEngine",
"Router",
"PlanConfig",
"default_router",
"ReferenceMatmulEngine",
"PYTHON_ENGINE_ID_BASE",
"CUDNN_HEURISTIC_ENGINE_ID",
"is_python_engine",
]

# cuTile backend has optional native deps (cuda-tile / cuda-python); expose it
# only when importable so a plain install still gets the reference backend.
try:
from .matmul_cutile_engine import MatmulCuTileEngine # noqa: F401

__all__.append("MatmulCuTileEngine")
except Exception: # noqa: BLE001
MatmulCuTileEngine = None # type: ignore
90 changes: 90 additions & 0 deletions python/cudnn/engines/base.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
"""Base class for NativeGraph execution backends (engines).

This module defines the abstract interface every execution backend must
implement. A backend is one of the interchangeable implementations the Router
dispatches to (Python DSLs, a naive reference, the cuDNN Graph backend, ...) —
see ``docs/python_native_graph_router.md``.

Create a custom backend by subclassing ``BaseEngine`` and implementing
``execute()``; override ``check_support()`` so the Router can decide whether
this backend can run a given graph.

Example:
class MyEngine(BaseEngine):
name = "my_engine"
engine_id = PYTHON_ENGINE_ID_BASE + 7 # stable id it owns

def check_support(self, graph):
for node in graph.nodes:
if node.node_type != NodeType.MATMUL:
raise NotImplementedError(...)

def execute(self, graph, tensor_data):
... # write results into caller-provided output buffers
"""

from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Dict

from .engine_ids import PYTHON_ENGINE_ID_BASE

if TYPE_CHECKING:
from ..pygraph import NativeGraph


class BaseEngine(ABC):
"""Abstract base class for graph execution backends.

A backend executes the operations defined in a NativeGraph. Different
backends use different implementations (PyTorch reference, cuTile, other
Python-DSL fusion engines, ...). Each declares a stable ``engine_id`` in the
reserved Python-engine region (see ``engine_ids``); the Router includes it in
the plan list at ``create_execution_plans()`` time when ``check_support()``
accepts the graph.

Attributes:
name: Human-readable identifier.
engine_id: Stable id in the shared flat engine-id space, in the reserved
Python region (>= PYTHON_ENGINE_ID_BASE). Subclasses MUST override.
default_knobs: Optional default tuning knobs for this engine's plan.
"""

name: str = "base"
engine_id: int = PYTHON_ENGINE_ID_BASE
default_knobs: Any = None
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

def __init__(self):
pass

def check_support(self, graph: "NativeGraph") -> None:
"""Raise if this backend cannot execute ``graph``.

Called by the Router during ``create_execution_plans()``. Raise
``NotImplementedError`` / ``ValueError`` / ``RuntimeError`` (unsupported
op, layout, or hardware) to decline the graph; the Router then tries the
next candidate, falling back to the cuDNN backend if none accept.

Default: accept everything (subclasses should narrow this).
"""
_ = graph

def get_workspace_size(self) -> int:
"""Workspace bytes this backend needs (default 0)."""
return 0

@abstractmethod
def execute(
self,
graph: "NativeGraph",
tensor_data: Dict[int, Any],
) -> None:
"""Execute the whole graph.

``tensor_data`` maps tensor UIDs (inputs + outputs) to their device
data. The backend writes results directly into the caller-provided
output buffers (matching cuDNN's execution model).
"""
raise NotImplementedError(f"Engine '{self.name}' must implement execute()")

def __repr__(self) -> str:
return f"{self.__class__.__name__}(name={self.name!r}, engine_id={self.engine_id})"
32 changes: 32 additions & 0 deletions python/cudnn/engines/engine_ids.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
"""Engine-id namespace shared by cuDNN and Python backends.

Execution engines live in one flat integer id space, exactly like cuDNN's own
backend engines (which have small ids 0..N, each with knobs). Python engines
occupy a reserved high region so the two never collide and a heuristics query
can return a single ranked list mixing both, e.g.:

[(engine_id=1048576, knobs), (engine_id=1, knobs), (engine_id=5, knobs), ...]

Dispatch is a single predicate on the id: ``is_python_engine(id)`` -> run via the
Python engine registry; otherwise lower to the cuDNN C++ backend.

Each Python engine declares a *stable* ``engine_id`` in this range (it owns its
id, the way a cuDNN engine does), so ids don't shift with registration order —
autotune results and pinned plans stay reproducible across runs.
"""

# Start of the reserved Python-engine id region. 1<<20 (~1.05M) is far above any
# plausible cuDNN engine count, so the two id spaces can never collide without
# having to know cuDNN's actual maximum.
PYTHON_ENGINE_ID_BASE = 1 << 20

# The cuDNN side of the plan list: "delegate to the loaded backend's own
# heuristics". Deliberately ONE entry — the backend's engine set varies by
# backend version and is only discoverable per graph at plan time, never
# statically enumerable by the frontend.
CUDNN_HEURISTIC_ENGINE_ID = -1


def is_python_engine(engine_id: int) -> bool:
"""True iff ``engine_id`` names a Python engine (vs a cuDNN backend engine)."""
return engine_id >= PYTHON_ENGINE_ID_BASE
Loading