diff --git a/tests/pytorch/attention/test_attention.py b/tests/pytorch/attention/test_attention.py index 7dc6caeb81..3150c06abb 100644 --- a/tests/pytorch/attention/test_attention.py +++ b/tests/pytorch/attention/test_attention.py @@ -45,7 +45,8 @@ ) from transformer_engine.pytorch.utils import get_cudnn_version import transformer_engine_torch as tex -from transformer_engine.pytorch.tensor.quantized_tensor import ( +from transformer_engine.pytorch.quantized_tensor import ( + Quantizer, prepare_for_saving, restore_from_saved, ) diff --git a/tests/pytorch/distributed/run_numerics_exact.py b/tests/pytorch/distributed/run_numerics_exact.py index ccbc3259bb..3605b3c708 100644 --- a/tests/pytorch/distributed/run_numerics_exact.py +++ b/tests/pytorch/distributed/run_numerics_exact.py @@ -22,8 +22,8 @@ ) from transformer_engine.pytorch import NVFP4Quantizer from transformer_engine.pytorch.constants import NVFP4_BLOCK_SCALING_SIZE -from transformer_engine.pytorch.experimental import quantization_nvfp4 -from transformer_engine.pytorch.experimental import utils +from transformer_engine.pytorch.custom_recipes import quantization_nvfp4 +from transformer_engine.pytorch.custom_recipes import utils from run_layer_with_overlap import _compare_tensors @@ -486,7 +486,7 @@ def _test_linear(parallel_mode=None, sequence_parallel=False, **kwargs): sequence_parallel (bool): Enable sequence parallelism if True. kwargs (dict): Additional arguments for the linear layer. - QUANTIZATION options: nvfp4 <=> experimental nvfp4 as a reference + QUANTIZATION options: nvfp4 <=> custom nvfp4 as a reference """ params_dtype = torch.bfloat16 use_bias = kwargs.get("bias", True) diff --git a/tests/pytorch/distributed/test_fusible_ops_with_userbuffers.py b/tests/pytorch/distributed/test_fusible_ops_with_userbuffers.py index 24112cc9ff..61c813b8f2 100644 --- a/tests/pytorch/distributed/test_fusible_ops_with_userbuffers.py +++ b/tests/pytorch/distributed/test_fusible_ops_with_userbuffers.py @@ -34,6 +34,7 @@ Float8Tensor, ) + # Import utility functions _current_file = pathlib.Path(__file__).resolve() sys.path.append(str(_current_file.parent.parent)) diff --git a/tests/pytorch/distributed/test_numerics_exact.py b/tests/pytorch/distributed/test_numerics_exact.py index fd6ef65e09..72aa786646 100644 --- a/tests/pytorch/distributed/test_numerics_exact.py +++ b/tests/pytorch/distributed/test_numerics_exact.py @@ -14,7 +14,7 @@ Distributed numerics tests This numerical test aims for zero tolerance test for absolute confidence in numerics. - In the case of NVFP4, with the experimental NVFP4 quantization, we matched bitwise + In the case of NVFP4, with the custom NVFP4 quantization, we matched bitwise result with the native silicon. For distrbuted test cases, we can do the same by thing by comparing BF16 AG results with the low precision AG results at layer level. """ diff --git a/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py b/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py index 77cfaaffe8..6009643ffa 100644 --- a/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py +++ b/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py @@ -8,8 +8,8 @@ import transformer_engine_torch as tex from transformer_engine.pytorch.constants import TE_DType from transformer_engine.pytorch import NVFP4Quantizer -from transformer_engine.pytorch.experimental.quantization_nvfp4 import NVFP4QuantizerRef -from transformer_engine.pytorch.experimental import utils +from transformer_engine.pytorch.custom_recipes.quantization_nvfp4 import NVFP4QuantizerRef +from transformer_engine.pytorch.custom_recipes import utils recipe_available, reason_for_no_recipe = te.is_nvfp4_available(return_reason=True) diff --git a/tests/pytorch/nvfp4/test_nvfp4_module_exact.py b/tests/pytorch/nvfp4/test_nvfp4_module_exact.py index 44f222b9d1..0292063ab9 100644 --- a/tests/pytorch/nvfp4/test_nvfp4_module_exact.py +++ b/tests/pytorch/nvfp4/test_nvfp4_module_exact.py @@ -6,8 +6,8 @@ import torch import transformer_engine.pytorch as te from transformer_engine.common import recipe -from transformer_engine.pytorch.experimental import quantization_nvfp4 -from transformer_engine.pytorch.experimental import utils +from transformer_engine.pytorch.custom_recipes import quantization_nvfp4 +from transformer_engine.pytorch.custom_recipes import utils recipe_available, reason_for_no_recipe = te.is_nvfp4_available(return_reason=True) diff --git a/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py b/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py index 8c24445573..2467c7e2e1 100644 --- a/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py +++ b/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py @@ -7,10 +7,10 @@ import transformer_engine.pytorch as te import transformer_engine_torch as tex from transformer_engine.pytorch import NVFP4Quantizer -from transformer_engine.pytorch.experimental.quantization_nvfp4 import NVFP4QuantizerRef +from transformer_engine.pytorch.custom_recipes.quantization_nvfp4 import NVFP4QuantizerRef +from transformer_engine.pytorch.custom_recipes import utils from transformer_engine.common.recipe import NVFP4BlockScaling from transformer_engine.pytorch.constants import TE_DType -from transformer_engine.pytorch.experimental import utils recipe_available, reason_for_no_recipe = te.is_nvfp4_available(return_reason=True) diff --git a/tests/pytorch/nvfp4/test_nvfp4_rht_quantize_exact.py b/tests/pytorch/nvfp4/test_nvfp4_rht_quantize_exact.py index 6f2f846a36..904dfc2eab 100644 --- a/tests/pytorch/nvfp4/test_nvfp4_rht_quantize_exact.py +++ b/tests/pytorch/nvfp4/test_nvfp4_rht_quantize_exact.py @@ -12,10 +12,10 @@ import transformer_engine.pytorch as te import transformer_engine_torch as tex from transformer_engine.pytorch import NVFP4Quantizer -from transformer_engine.common.recipe import NVFP4BlockScaling +from transformer_engine.pytorch.custom_recipes.quantization_nvfp4 import NVFP4QuantizerRef +from transformer_engine.pytorch.custom_recipes import utils from transformer_engine.pytorch.constants import TE_DType -from transformer_engine.pytorch.experimental.quantization_nvfp4 import NVFP4QuantizerRef -from transformer_engine.pytorch.experimental import utils +from transformer_engine.common.recipe import NVFP4BlockScaling import pytest import torch diff --git a/tests/pytorch/test_custom_recipe.py b/tests/pytorch/test_custom_recipe.py index 516354a34b..64f1c3d159 100644 --- a/tests/pytorch/test_custom_recipe.py +++ b/tests/pytorch/test_custom_recipe.py @@ -17,6 +17,48 @@ Float8CurrentScalingQuantizer, ) import transformer_engine.pytorch.ops as te_ops +from transformer_engine.pytorch.custom_recipes.quantization_nvfp4 import ( + nvfp4_ref_rht_2d_quantizer_factory, +) + + +@pytest.mark.parametrize("module_type", ["Linear", "LayerNormLinear", "OpsLinear"]) +def test_custom_recipe_sanity_modules_nvfp4(module_type): + """Test modules with NVFP4 custom recipe support""" + available, reason = te.is_fp8_available(return_reason=True) + if not torch.cuda.is_available() or not available: + pytest.skip(f"FP8 unsupported on this device: {reason}") + + torch.manual_seed(0) + + # Simple linear layer with dims divisible by 16 + in_features = 64 + out_features = 64 + batch = 32 + + if module_type == "Linear": + model = Linear(in_features, out_features, params_dtype=torch.bfloat16, bias=False).cuda() + elif module_type == "LayerNormLinear": + model = LayerNormLinear( + in_features, out_features, params_dtype=torch.bfloat16, bias=False + ).cuda() + else: # OpsLinear + model = te_ops.Linear( + in_features, out_features, device="cuda", dtype=torch.bfloat16, bias=False + ) + inp = torch.randn(batch, in_features, device="cuda", dtype=torch.bfloat16, requires_grad=True) + + # Use NVFP4 quantizer factory + custom_recipe = recipe.CustomRecipe(qfactory=nvfp4_ref_rht_2d_quantizer_factory) + + # Execute with custom recipe + with autocast(enabled=True, recipe=custom_recipe): + out = model(inp) + loss = out.float().sum() + loss.backward() + + # Basic sanity: gradients exist + assert inp.grad is not None @pytest.mark.parametrize("module_type", ["Linear", "LayerNormLinear", "OpsLinear", "LayerNormMLP"]) diff --git a/transformer_engine/debug/pytorch/debug_quantization.py b/transformer_engine/debug/pytorch/debug_quantization.py index 185bf15d05..7f45a24e20 100644 --- a/transformer_engine/debug/pytorch/debug_quantization.py +++ b/transformer_engine/debug/pytorch/debug_quantization.py @@ -15,7 +15,7 @@ import transformer_engine_torch as tex from transformer_engine.common.recipe import Recipe -from transformer_engine.pytorch.tensor.quantized_tensor import ( +from transformer_engine.pytorch.quantized_tensor import ( QuantizedTensor, Quantizer, QuantizedTensorStorage, diff --git a/transformer_engine/pytorch/__init__.py b/transformer_engine/pytorch/__init__.py index 77c71b8119..9d894a389b 100644 --- a/transformer_engine/pytorch/__init__.py +++ b/transformer_engine/pytorch/__init__.py @@ -66,24 +66,24 @@ def torch_version() -> tuple[int, ...]: from transformer_engine.pytorch import optimizers from transformer_engine.pytorch.export import onnx_export from transformer_engine.pytorch.cross_entropy import parallel_cross_entropy -from transformer_engine.pytorch.tensor import Quantizer +from transformer_engine.pytorch.quantized_tensor import QuantizedTensorStorage +from transformer_engine.pytorch.quantized_tensor import QuantizedTensor +from transformer_engine.pytorch.quantized_tensor import Quantizer +from transformer_engine.pytorch.quantized_tensor import prepare_for_saving +from transformer_engine.pytorch.quantized_tensor import restore_from_saved from transformer_engine.pytorch.tensor import Float8Quantizer from transformer_engine.pytorch.tensor import Float8CurrentScalingQuantizer from transformer_engine.pytorch.tensor import MXFP8Quantizer from transformer_engine.pytorch.tensor import Float8BlockQuantizer from transformer_engine.pytorch.tensor import NVFP4Quantizer -from transformer_engine.pytorch.tensor import QuantizedTensorStorage from transformer_engine.pytorch.tensor import Float8TensorStorage from transformer_engine.pytorch.tensor import MXFP8TensorStorage from transformer_engine.pytorch.tensor import Float8BlockwiseQTensorStorage from transformer_engine.pytorch.tensor import NVFP4TensorStorage -from transformer_engine.pytorch.tensor import QuantizedTensor from transformer_engine.pytorch.tensor import Float8Tensor from transformer_engine.pytorch.tensor import MXFP8Tensor from transformer_engine.pytorch.tensor import Float8BlockwiseQTensor from transformer_engine.pytorch.tensor import NVFP4Tensor -from transformer_engine.pytorch.tensor import prepare_for_saving -from transformer_engine.pytorch.tensor import restore_from_saved try: torch._dynamo.config.error_on_nested_jit_trace = False diff --git a/transformer_engine/pytorch/attention/dot_product_attention/backends.py b/transformer_engine/pytorch/attention/dot_product_attention/backends.py index 6dfe0d31b3..6c19d868a1 100644 --- a/transformer_engine/pytorch/attention/dot_product_attention/backends.py +++ b/transformer_engine/pytorch/attention/dot_product_attention/backends.py @@ -24,7 +24,7 @@ Float8Quantizer, Float8CurrentScalingQuantizer, ) -from transformer_engine.pytorch.tensor.quantized_tensor import ( +from transformer_engine.pytorch.quantized_tensor import ( QuantizedTensorStorage, prepare_for_saving, restore_from_saved, diff --git a/transformer_engine/pytorch/attention/dot_product_attention/context_parallel.py b/transformer_engine/pytorch/attention/dot_product_attention/context_parallel.py index a474cb809a..e5ee8cc7db 100644 --- a/transformer_engine/pytorch/attention/dot_product_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/dot_product_attention/context_parallel.py @@ -21,7 +21,7 @@ ) from transformer_engine.pytorch.quantization import FP8GlobalStateManager from transformer_engine.pytorch.tensor.float8_tensor import Float8Tensor -from transformer_engine.pytorch.tensor.quantized_tensor import QuantizedTensorStorage +from transformer_engine.pytorch.quantized_tensor import QuantizedTensorStorage from transformer_engine.pytorch.jit import jit_fuser from transformer_engine.pytorch.constants import ( dist_group_type, @@ -33,7 +33,7 @@ gather_along_first_dim, reduce_scatter_along_first_dim, ) -from transformer_engine.pytorch.tensor.quantized_tensor import ( +from transformer_engine.pytorch.quantized_tensor import ( prepare_for_saving, restore_from_saved, ) diff --git a/transformer_engine/pytorch/cpp_extensions/fused_attn.py b/transformer_engine/pytorch/cpp_extensions/fused_attn.py index 94a12c4a09..f80c001a1d 100644 --- a/transformer_engine/pytorch/cpp_extensions/fused_attn.py +++ b/transformer_engine/pytorch/cpp_extensions/fused_attn.py @@ -15,7 +15,7 @@ NVTE_Softmax_Type, NVTE_Fused_Attn_Backend, ) -from ..tensor.quantized_tensor import Quantizer +from ..quantized_tensor import Quantizer __all__ = [ diff --git a/transformer_engine/pytorch/cpp_extensions/gemm.py b/transformer_engine/pytorch/cpp_extensions/gemm.py index a45fafb68a..dd04112982 100644 --- a/transformer_engine/pytorch/cpp_extensions/gemm.py +++ b/transformer_engine/pytorch/cpp_extensions/gemm.py @@ -11,10 +11,10 @@ from ..constants import TE_DType from ..utils import get_sm_count, _empty_tensor -from ..tensor.quantized_tensor import Quantizer +from ..quantized_tensor import Quantizer from ..tensor.storage.float8_blockwise_tensor_storage import Float8BlockwiseQTensorStorage -from ..tensor.utils import is_experimental -from ..experimental.gemm import experimental_gemm +from ..tensor.utils import is_custom +from ..custom_recipes.gemm import custom_gemm from ...debug.pytorch.debug_quantization import DebugQuantizer __all__ = [ @@ -79,9 +79,9 @@ def general_gemm( if not out.is_contiguous(): raise ValueError("Output tensor is not contiguous.") - # If A or B are experimental tensors -> dispatch to quantizers's qgemm implementation - if is_experimental(A) or is_experimental(B): - return experimental_gemm( + # If A or B are custom tensors -> dispatch to quantizers's qgemm implementation + if is_custom(A) or is_custom(B): + return custom_gemm( A, B, workspace, diff --git a/transformer_engine/pytorch/cpu_offload.py b/transformer_engine/pytorch/cpu_offload.py index 648b21eb4d..6edc126200 100644 --- a/transformer_engine/pytorch/cpu_offload.py +++ b/transformer_engine/pytorch/cpu_offload.py @@ -10,7 +10,7 @@ import torch from transformer_engine.debug.pytorch.debug_state import TEDebugState -from .tensor.quantized_tensor import QuantizedTensorStorage +from .quantized_tensor import QuantizedTensorStorage from .tensor.float8_tensor import Float8Tensor __all__ = ["get_cpu_offload_context"] diff --git a/transformer_engine/pytorch/experimental/__init__.py b/transformer_engine/pytorch/custom_recipes/__init__.py similarity index 100% rename from transformer_engine/pytorch/experimental/__init__.py rename to transformer_engine/pytorch/custom_recipes/__init__.py diff --git a/transformer_engine/pytorch/experimental/gemm.py b/transformer_engine/pytorch/custom_recipes/gemm.py similarity index 90% rename from transformer_engine/pytorch/experimental/gemm.py rename to transformer_engine/pytorch/custom_recipes/gemm.py index 0bd740d85d..cc98a8a57a 100644 --- a/transformer_engine/pytorch/experimental/gemm.py +++ b/transformer_engine/pytorch/custom_recipes/gemm.py @@ -2,21 +2,21 @@ # # See LICENSE for license information. -"""GEMM API for experimental middleware between Transformer Engine and Kitchen.""" +"""GEMM API that enables custom GEMM logic for custom quantization recipes.""" from typing import Iterable, Optional import torch -from transformer_engine.pytorch.experimental.quantization import ( +from transformer_engine.pytorch.custom_recipes.quantization import ( MMParams, GEMMType, ) -from transformer_engine.pytorch.tensor.quantized_tensor import QuantizedTensorStorage, Quantizer -from transformer_engine.pytorch.tensor.utils import is_experimental +from transformer_engine.pytorch.quantized_tensor import QuantizedTensorStorage, Quantizer +from transformer_engine.pytorch.tensor.utils import is_custom -def experimental_gemm( +def custom_gemm( A: QuantizedTensorStorage, B: QuantizedTensorStorage, workspace: torch.Tensor, # pylint: disable=unused-argument @@ -32,7 +32,7 @@ def experimental_gemm( grad: bool = False, ) -> Iterable[Optional[torch.Tensor]]: """Dispatch GEMM to quantizer's qgemm method.""" - assert is_experimental(A) and is_experimental(B), "A and B must be experimental tensors" + assert is_custom(A) and is_custom(B), "A and B must be custom tensors" A, B = B, A diff --git a/transformer_engine/pytorch/experimental/quantization.py b/transformer_engine/pytorch/custom_recipes/quantization.py similarity index 100% rename from transformer_engine/pytorch/experimental/quantization.py rename to transformer_engine/pytorch/custom_recipes/quantization.py diff --git a/transformer_engine/pytorch/experimental/quantization_nvfp4.py b/transformer_engine/pytorch/custom_recipes/quantization_nvfp4.py similarity index 98% rename from transformer_engine/pytorch/experimental/quantization_nvfp4.py rename to transformer_engine/pytorch/custom_recipes/quantization_nvfp4.py index fc50d07424..1ce9079eb1 100644 --- a/transformer_engine/pytorch/experimental/quantization_nvfp4.py +++ b/transformer_engine/pytorch/custom_recipes/quantization_nvfp4.py @@ -9,9 +9,9 @@ import torch -from transformer_engine.pytorch.experimental import quantization -from transformer_engine.pytorch.experimental import utils -from transformer_engine.pytorch.tensor.quantized_tensor import QuantizedTensorStorage, Quantizer +from transformer_engine.pytorch.custom_recipes import quantization +from transformer_engine.pytorch.custom_recipes import utils +from transformer_engine.pytorch.quantized_tensor import QuantizedTensorStorage, Quantizer def nvfp4_ref_rht_2d_quantizer_factory(role): @@ -229,8 +229,8 @@ class NVFP4TensorRef(QuantizedTensorStorage): _quantizer: Optional[Quantizer] = None @property - def experimental(self) -> bool: - """Flag to indicate this quantizer is using experimental Kitchen middleware.""" + def custom(self) -> bool: + """Flag to indicate this quantized tensor is custom.""" return True def prepare_for_saving( @@ -362,8 +362,8 @@ def __init__( self.with_random_sign_mask = with_random_sign_mask @property - def experimental(self) -> bool: - """Flag to indicate this quantizer is using experimental Kitchen middleware""" + def custom(self) -> bool: + """Flag to indicate this quantizer is custom.""" return True @staticmethod diff --git a/transformer_engine/pytorch/experimental/utils.py b/transformer_engine/pytorch/custom_recipes/utils.py similarity index 100% rename from transformer_engine/pytorch/experimental/utils.py rename to transformer_engine/pytorch/custom_recipes/utils.py diff --git a/transformer_engine/pytorch/distributed.py b/transformer_engine/pytorch/distributed.py index 5ed73f6783..8c14d5ab7f 100644 --- a/transformer_engine/pytorch/distributed.py +++ b/transformer_engine/pytorch/distributed.py @@ -41,7 +41,7 @@ from .tensor.mxfp8_tensor import MXFP8Quantizer from .tensor.nvfp4_tensor import NVFP4Quantizer from .tensor.float8_blockwise_tensor import Float8BlockQuantizer -from .tensor.quantized_tensor import QuantizedTensorStorage, QuantizedTensor, Quantizer +from .quantized_tensor import QuantizedTensorStorage, QuantizedTensor, Quantizer from .tensor.storage.float8_tensor_storage import Float8TensorStorage from .tensor.storage.mxfp8_tensor_storage import MXFP8TensorStorage from .tensor.storage.nvfp4_tensor_storage import NVFP4TensorStorage diff --git a/transformer_engine/pytorch/module/base.py b/transformer_engine/pytorch/module/base.py index d16455b5b4..7f571ce011 100644 --- a/transformer_engine/pytorch/module/base.py +++ b/transformer_engine/pytorch/module/base.py @@ -38,7 +38,7 @@ _fsdp_gather_tensors, ) from ..constants import dist_group_type -from ..tensor.quantized_tensor import QuantizedTensor, QuantizedTensorStorage, Quantizer +from ..quantized_tensor import QuantizedTensor, QuantizedTensorStorage, Quantizer from ..tensor.float8_tensor import Float8Quantizer, Float8CurrentScalingQuantizer from ..tensor.mxfp8_tensor import MXFP8Quantizer from ..tensor.float8_blockwise_tensor import Float8BlockQuantizer diff --git a/transformer_engine/pytorch/module/grouped_linear.py b/transformer_engine/pytorch/module/grouped_linear.py index a5bf21ee17..aae85e2cab 100644 --- a/transformer_engine/pytorch/module/grouped_linear.py +++ b/transformer_engine/pytorch/module/grouped_linear.py @@ -43,7 +43,7 @@ from ..cpu_offload import is_cpu_offload_enabled from ..tensor.float8_tensor import Float8CurrentScalingQuantizer, Float8Quantizer -from ..tensor.quantized_tensor import ( +from ..quantized_tensor import ( QuantizedTensorStorage, Quantizer, prepare_for_saving, diff --git a/transformer_engine/pytorch/module/layernorm_linear.py b/transformer_engine/pytorch/module/layernorm_linear.py index 05f2e9cde4..933c7cde53 100644 --- a/transformer_engine/pytorch/module/layernorm_linear.py +++ b/transformer_engine/pytorch/module/layernorm_linear.py @@ -16,7 +16,7 @@ from transformer_engine.common.recipe import Recipe from transformer_engine.pytorch import torch_version -from transformer_engine.pytorch.tensor.utils import is_experimental +from transformer_engine.pytorch.tensor.utils import is_custom from .base import ( fill_userbuffers_buffer_for_all_gather, get_workspace, @@ -56,7 +56,7 @@ from ..jit import no_torch_dynamo from ..graph import is_graph_capturing from ._common import apply_normalization, noop_cat, WeightGradStore -from ..tensor.quantized_tensor import ( +from ..quantized_tensor import ( QuantizedTensor, QuantizedTensorStorage, Quantizer, @@ -194,13 +194,13 @@ def forward( # Avoid quantized norm kernel if norm output will be returned # or if a gather of ln_out must be in high precision. - experimental = is_experimental(input_quantizer) + custom = is_custom(input_quantizer) with_quantized_norm = ( fp8 and not debug and not return_layernorm_output and not return_layernorm_output_gathered - and not experimental # TODO(negvet): and not FP8GlobalStateManager.get_fp8_recipe().custom() + and not custom # TODO(negvet): and not FP8GlobalStateManager.get_fp8_recipe().custom() ) # Apply normalization @@ -246,8 +246,8 @@ def forward( quantizer = None if fp8 or debug: quantizer = input_quantizer - # experimental recipe doesn't need to support quantized AG - if not with_quantized_norm and not experimental: + # custom recipe doesn't need to support quantized AG + if not with_quantized_norm and not custom: ln_out = quantizer(ln_out) quantizer.set_usage(rowwise=True, columnwise=False) if ub_overlap_ag_fprop: # Initialize Userbuffers all-gather diff --git a/transformer_engine/pytorch/module/layernorm_mlp.py b/transformer_engine/pytorch/module/layernorm_mlp.py index a2ddb970af..bae0f28251 100644 --- a/transformer_engine/pytorch/module/layernorm_mlp.py +++ b/transformer_engine/pytorch/module/layernorm_mlp.py @@ -17,7 +17,7 @@ from transformer_engine.common.recipe import Recipe from transformer_engine.pytorch import torch_version -from transformer_engine.pytorch.tensor.utils import is_experimental +from transformer_engine.pytorch.tensor.utils import is_custom from .base import ( fill_userbuffers_buffer_for_all_gather, get_workspace, @@ -70,7 +70,7 @@ from ..tensor.float8_blockwise_tensor import Float8BlockQuantizer from ._common import apply_normalization, WeightGradStore from ..cpu_offload import is_cpu_offload_enabled, mark_activation_offload -from ..tensor.quantized_tensor import ( +from ..quantized_tensor import ( QuantizedTensorStorage, Quantizer, prepare_for_saving, @@ -268,13 +268,13 @@ def forward( # high precision layernorm output and output of the linear are returned # for debug: : layernorm output = High precision to enable processing of this norm - experimental = is_experimental(fc1_input_quantizer) + custom = is_custom(fc1_input_quantizer) with_quantized_norm = ( fp8 and not debug and not return_layernorm_output and not return_layernorm_output_gathered - and not experimental + and not custom ) # Apply normalization @@ -314,8 +314,8 @@ def forward( quantizer = None if fp8 or debug: quantizer = fc1_input_quantizer - # experimental recipe doesn't need to support quantized AG - if not with_quantized_norm and not experimental: + # custom recipe doesn't need to support quantized AG + if not with_quantized_norm and not custom: ln_out = fc1_input_quantizer(ln_out) fc1_input_quantizer.set_usage(rowwise=True, columnwise=False) if ub_overlap_ag: diff --git a/transformer_engine/pytorch/module/linear.py b/transformer_engine/pytorch/module/linear.py index 3069c21d9f..ccb84e6642 100644 --- a/transformer_engine/pytorch/module/linear.py +++ b/transformer_engine/pytorch/module/linear.py @@ -57,7 +57,7 @@ from ..constants import GemmParallelModes, dist_group_type from ..jit import no_torch_dynamo from ..graph import is_graph_capturing -from ..tensor.quantized_tensor import ( +from ..quantized_tensor import ( QuantizedTensor, QuantizedTensorStorage, Quantizer, @@ -66,7 +66,7 @@ ) from ..tensor.float8_tensor import Float8CurrentScalingQuantizer, Float8Quantizer from ..tensor.mxfp8_tensor import MXFP8Quantizer -from ..tensor.utils import is_experimental +from ..tensor.utils import is_custom from ..export import is_in_onnx_export_mode, assert_warmed_up from ..cpu_offload import is_cpu_offload_enabled, mark_activation_offload from ...debug.pytorch.debug_state import TEDebugState @@ -153,8 +153,8 @@ def forward( ub_obj = get_ub(ub_name + "_fprop", fp8) ub_type = tex.CommOverlapType.AG - # experimental recipe check - experimental = is_experimental(input_quantizer) or is_experimental(weight_quantizer) + # custom recipe check + custom = is_custom(input_quantizer) or is_custom(weight_quantizer) # ------------------------------------------------------ # Prepare input tensor @@ -178,7 +178,7 @@ def forward( if fp8 or debug: if input_quantizer is None: raise ValueError("Missing quantizer for input tensor") - if not isinstance(inputmat, QuantizedTensorStorage) and not experimental: + if not isinstance(inputmat, QuantizedTensorStorage) and not custom: own_quantized_input = True input_quantizer.set_usage(rowwise=True, columnwise=backward_needs_input) if isinstance( @@ -448,7 +448,7 @@ def forward( ctx.main_grad_func = lambda: weight.main_grad ctx.debug = debug - ctx.experimental = experimental + ctx.custom = custom ctx.cpu_offloading = cpu_offloading ctx.is_first_microbatch = is_first_microbatch ctx.use_bias = bias is not None @@ -616,7 +616,7 @@ def backward(ctx, grad_output: torch.Tensor) -> Tuple[Union[torch.Tensor, None], if isinstance(inputmat, QuantizedTensorStorage): # Input tensor is already quantized pass - elif ctx.debug or ctx.experimental: + elif ctx.debug or ctx.custom: # Debug quantizer will be applied immediately before wgrad GEMM pass else: diff --git a/transformer_engine/pytorch/ops/_common.py b/transformer_engine/pytorch/ops/_common.py index 52ca84b5df..a07ffea43f 100644 --- a/transformer_engine/pytorch/ops/_common.py +++ b/transformer_engine/pytorch/ops/_common.py @@ -13,7 +13,7 @@ from .. import torch_version from ..quantization import FP8GlobalStateManager from ..tensor.float8_tensor import Float8Tensor -from ..tensor.quantized_tensor import QuantizedTensorStorage +from ..quantized_tensor import QuantizedTensorStorage from ..utils import canonicalize_dtype diff --git a/transformer_engine/pytorch/ops/fused/userbuffers_backward_linear.py b/transformer_engine/pytorch/ops/fused/userbuffers_backward_linear.py index d95b2298fe..fd1820d15d 100644 --- a/transformer_engine/pytorch/ops/fused/userbuffers_backward_linear.py +++ b/transformer_engine/pytorch/ops/fused/userbuffers_backward_linear.py @@ -21,7 +21,7 @@ get_ub, get_workspace, ) -from ...tensor.quantized_tensor import Quantizer +from ...quantized_tensor import Quantizer from ...tensor.mxfp8_tensor import MXFP8Quantizer from ...utils import canonicalize_device, canonicalize_dtype, clear_tensor_data from ..basic import BasicLinear, Bias, ReduceScatter diff --git a/transformer_engine/pytorch/ops/fused/userbuffers_forward_linear.py b/transformer_engine/pytorch/ops/fused/userbuffers_forward_linear.py index e20de53da3..057eb576d7 100644 --- a/transformer_engine/pytorch/ops/fused/userbuffers_forward_linear.py +++ b/transformer_engine/pytorch/ops/fused/userbuffers_forward_linear.py @@ -21,7 +21,7 @@ get_workspace, _2X_ACC_FPROP, ) -from ...tensor.quantized_tensor import Quantizer +from ...quantized_tensor import Quantizer from ...tensor.float8_tensor import Float8Quantizer, Float8CurrentScalingQuantizer from ...tensor.storage.float8_tensor_storage import Float8TensorStorage from .._common import maybe_dequantize, is_quantized_tensor diff --git a/transformer_engine/pytorch/ops/fuser.py b/transformer_engine/pytorch/ops/fuser.py index 8ae112022c..6026a40b65 100644 --- a/transformer_engine/pytorch/ops/fuser.py +++ b/transformer_engine/pytorch/ops/fuser.py @@ -28,7 +28,7 @@ fuse_userbuffers_backward_linear, fuse_userbuffers_forward_linear, ) -from transformer_engine.pytorch.tensor.quantized_tensor import ( +from transformer_engine.pytorch.quantized_tensor import ( prepare_for_saving, restore_from_saved, ) diff --git a/transformer_engine/pytorch/permutation.py b/transformer_engine/pytorch/permutation.py index ea3e67a57c..f73bc9a966 100644 --- a/transformer_engine/pytorch/permutation.py +++ b/transformer_engine/pytorch/permutation.py @@ -10,7 +10,7 @@ import transformer_engine_torch as tex import transformer_engine.pytorch.triton.permutation as triton_permutation from transformer_engine.pytorch.constants import TE_DType -from transformer_engine.pytorch.tensor.quantized_tensor import QuantizedTensor +from transformer_engine.pytorch.quantized_tensor import QuantizedTensor from transformer_engine.pytorch.tensor.float8_tensor import Float8Tensor from transformer_engine.pytorch.tensor.float8_blockwise_tensor import Float8BlockwiseQTensor from transformer_engine.pytorch.tensor.mxfp8_tensor import MXFP8Tensor diff --git a/transformer_engine/pytorch/tensor/quantized_tensor.py b/transformer_engine/pytorch/quantized_tensor.py similarity index 89% rename from transformer_engine/pytorch/tensor/quantized_tensor.py rename to transformer_engine/pytorch/quantized_tensor.py index a524d5c8de..15f5b6bd5e 100644 --- a/transformer_engine/pytorch/tensor/quantized_tensor.py +++ b/transformer_engine/pytorch/quantized_tensor.py @@ -2,10 +2,10 @@ # # See LICENSE for license information. -"""Tensor with quantized data""" +"""Pure Python base classes for quantization.""" from __future__ import annotations -from typing import Callable, Optional, Tuple, Iterable, Any, Dict, Union +from typing import Optional, Tuple, Iterable, Any, Dict, Union import abc import copy import warnings @@ -14,6 +14,11 @@ from torch.utils._pytree import tree_map from transformer_engine.common.recipe import Recipe +from transformer_engine.pytorch.tensor._quantization_helpers import ( + _QuantizeFunc, + _IdentityFunc, + _stride_from_shape, +) class QuantizedTensorStorage: @@ -310,73 +315,6 @@ def is_quantizable(self, inp: torch.Tensor) -> bool: # pylint: disable=unused-a return True -class _QuantizeFunc(torch.autograd.Function): - """Quantize tensor""" - - @staticmethod - def forward( - _ctx: Optional[torch.autograd.function.FunctionCtx], # unused - tensor: torch.Tensor, - quantize_impl: Callable, - ) -> QuantizedTensor: - # pylint: disable=missing-function-docstring - return quantize_impl(tensor) - - @staticmethod - def backward( - _ctx: torch.autograd.function.FunctionCtx, # unused - grad: torch.Tensor, - ) -> Tuple[Optional[torch.Tensor], ...]: - # pylint: disable=missing-function-docstring - # Assume that we want gradients in full precision - return grad, None - - -class _IdentityFunc(torch.autograd.Function): - """Identity function - - If constructor keyword-arguments are provided, then construct a - new Float8Tensor using the provided tensor's attributes. - - """ - - @staticmethod - def forward( - ctx, tensor: QuantizedTensor, init_kwargs: Optional[Dict[str, Any]] = None - ) -> QuantizedTensor: - # pylint: disable=missing-function-docstring - - # Return input tensor if constructor kwargs are not provided - if init_kwargs is None: - return tensor.detach() - - # Construct new tensor if constructor kwargs are provided - ctx.input_dtype = tensor.dtype - kwargs = tensor.get_metadata() - for key, val in init_kwargs.items(): - kwargs[key] = val - return type(tensor)(tensor.shape, tensor.dtype, **kwargs) - - @staticmethod - def backward(ctx, grad_output): - # pylint: disable=missing-function-docstring - grad_input = grad_output - if grad_input.dtype == ctx.input_dtype: - grad_input = grad_input.detach() - else: - grad_input = grad_input.to(ctx.input_dtype) - return grad_input, None - - -def _stride_from_shape(shape: list[int]): - if len(shape) == 0: - return [] - rstride = [1] - for d in reversed(shape[1:]): - rstride.append(rstride[-1] * d) - return list(reversed(rstride)) - - class QuantizedTensor(torch.Tensor): """Abstract base class for tensor with quantized data diff --git a/transformer_engine/pytorch/tensor/__init__.py b/transformer_engine/pytorch/tensor/__init__.py index 7689e20194..ada624a902 100644 --- a/transformer_engine/pytorch/tensor/__init__.py +++ b/transformer_engine/pytorch/tensor/__init__.py @@ -6,7 +6,7 @@ import torch -from .quantized_tensor import ( +from ..quantized_tensor import ( QuantizedTensorStorage, QuantizedTensor, Quantizer, diff --git a/transformer_engine/pytorch/tensor/_quantization_helpers.py b/transformer_engine/pytorch/tensor/_quantization_helpers.py new file mode 100644 index 0000000000..2214edbff2 --- /dev/null +++ b/transformer_engine/pytorch/tensor/_quantization_helpers.py @@ -0,0 +1,84 @@ +# Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Private helper functions and classes for quantized tensor implementations. + +This module contains internal autograd functions and utilities that support +the quantization machinery. +""" + +from __future__ import annotations +from typing import Callable, Optional, Tuple, Any, Dict, TYPE_CHECKING +import torch + +if TYPE_CHECKING: + from transformer_engine.pytorch.quantized_tensor import QuantizedTensor + + +class _QuantizeFunc(torch.autograd.Function): + """Quantize tensor""" + + @staticmethod + def forward( + _ctx: Optional[torch.autograd.function.FunctionCtx], # unused + tensor: torch.Tensor, + quantize_impl: Callable, + ) -> QuantizedTensor: + # pylint: disable=missing-function-docstring + return quantize_impl(tensor) + + @staticmethod + def backward( + _ctx: torch.autograd.function.FunctionCtx, # unused + grad: torch.Tensor, + ) -> Tuple[Optional[torch.Tensor], ...]: + # pylint: disable=missing-function-docstring + # Assume that we want gradients in full precision + return grad, None + + +class _IdentityFunc(torch.autograd.Function): + """Identity function + + If constructor keyword-arguments are provided, then construct a + new Float8Tensor using the provided tensor's attributes. + + """ + + @staticmethod + def forward( + ctx, tensor: QuantizedTensor, init_kwargs: Optional[Dict[str, Any]] = None + ) -> QuantizedTensor: + # pylint: disable=missing-function-docstring + + # Return input tensor if constructor kwargs are not provided + if init_kwargs is None: + return tensor.detach() + + # Construct new tensor if constructor kwargs are provided + ctx.input_dtype = tensor.dtype + kwargs = tensor.get_metadata() + for key, val in init_kwargs.items(): + kwargs[key] = val + return type(tensor)(tensor.shape, tensor.dtype, **kwargs) + + @staticmethod + def backward(ctx, grad_output): + # pylint: disable=missing-function-docstring + grad_input = grad_output + if grad_input.dtype == ctx.input_dtype: + grad_input = grad_input.detach() + else: + grad_input = grad_input.to(ctx.input_dtype) + return grad_input, None + + +def _stride_from_shape(shape: list[int]): + """Calculate stride from shape for contiguous tensors""" + if len(shape) == 0: + return [] + rstride = [1] + for d in reversed(shape[1:]): + rstride.append(rstride[-1] * d) + return list(reversed(rstride)) diff --git a/transformer_engine/pytorch/tensor/float8_blockwise_tensor.py b/transformer_engine/pytorch/tensor/float8_blockwise_tensor.py index 48762499b9..8054374c81 100644 --- a/transformer_engine/pytorch/tensor/float8_blockwise_tensor.py +++ b/transformer_engine/pytorch/tensor/float8_blockwise_tensor.py @@ -14,11 +14,8 @@ from transformer_engine.common.recipe import Float8BlockScaling, Recipe from .storage.float8_blockwise_tensor_storage import Float8BlockwiseQTensorStorage -from .quantized_tensor import ( - QuantizedTensor, - Quantizer, - _IdentityFunc, -) +from ..quantized_tensor import QuantizedTensor, Quantizer +from ._quantization_helpers import _IdentityFunc from ..utils import devices_match, round_up_to_nearest_multiple aten = torch.ops.aten diff --git a/transformer_engine/pytorch/tensor/float8_tensor.py b/transformer_engine/pytorch/tensor/float8_tensor.py index a4e68e53b0..de112bb3fd 100644 --- a/transformer_engine/pytorch/tensor/float8_tensor.py +++ b/transformer_engine/pytorch/tensor/float8_tensor.py @@ -14,11 +14,8 @@ from transformer_engine.common.recipe import DelayedScaling, Float8CurrentScaling, Recipe from ..utils import canonicalize_process_group, devices_match from .storage.float8_tensor_storage import Float8TensorStorage, _FromFloat8Func -from .quantized_tensor import ( - QuantizedTensor, - Quantizer, - _IdentityFunc, -) +from ..quantized_tensor import QuantizedTensor, Quantizer +from ._quantization_helpers import _IdentityFunc from ..constants import dist_group_type aten = torch.ops.aten diff --git a/transformer_engine/pytorch/tensor/mxfp8_tensor.py b/transformer_engine/pytorch/tensor/mxfp8_tensor.py index 700de24c4e..5ef5708fdb 100644 --- a/transformer_engine/pytorch/tensor/mxfp8_tensor.py +++ b/transformer_engine/pytorch/tensor/mxfp8_tensor.py @@ -17,11 +17,8 @@ from ..utils import devices_match, round_up_to_nearest_multiple from .storage.mxfp8_tensor_storage import MXFP8TensorStorage, _FromMXFP8Func -from .quantized_tensor import ( - QuantizedTensor, - Quantizer, - _IdentityFunc, -) +from ..quantized_tensor import QuantizedTensor, Quantizer +from ._quantization_helpers import _IdentityFunc aten = torch.ops.aten diff --git a/transformer_engine/pytorch/tensor/nvfp4_tensor.py b/transformer_engine/pytorch/tensor/nvfp4_tensor.py index 5e2eeed726..7a5f8858f2 100644 --- a/transformer_engine/pytorch/tensor/nvfp4_tensor.py +++ b/transformer_engine/pytorch/tensor/nvfp4_tensor.py @@ -22,7 +22,8 @@ ) from .storage.nvfp4_tensor_storage import NVFP4TensorStorage, _FromNVFP4Func -from .quantized_tensor import QuantizedTensor, Quantizer, _IdentityFunc +from ..quantized_tensor import QuantizedTensor, Quantizer +from ._quantization_helpers import _IdentityFunc aten = torch.ops.aten diff --git a/transformer_engine/pytorch/tensor/storage/float8_blockwise_tensor_storage.py b/transformer_engine/pytorch/tensor/storage/float8_blockwise_tensor_storage.py index 9040ea3a43..c2d5e8b3fa 100644 --- a/transformer_engine/pytorch/tensor/storage/float8_blockwise_tensor_storage.py +++ b/transformer_engine/pytorch/tensor/storage/float8_blockwise_tensor_storage.py @@ -13,12 +13,10 @@ from transformer_engine_torch import DType as TE_DType from transformer_engine_torch import Float8BlockScaleTensorFormat -from ..quantized_tensor import QuantizedTensorStorage +from ...quantized_tensor import QuantizedTensorStorage, Quantizer from ...constants import TE_DType_To_Torch -from ..quantized_tensor import Quantizer - from ...utils import _empty_tensor diff --git a/transformer_engine/pytorch/tensor/storage/float8_tensor_storage.py b/transformer_engine/pytorch/tensor/storage/float8_tensor_storage.py index b9533edb6e..a31f6a3799 100644 --- a/transformer_engine/pytorch/tensor/storage/float8_tensor_storage.py +++ b/transformer_engine/pytorch/tensor/storage/float8_tensor_storage.py @@ -12,12 +12,10 @@ import transformer_engine_torch as tex from transformer_engine_torch import DType as TE_DType -from ..quantized_tensor import QuantizedTensorStorage +from ...quantized_tensor import QuantizedTensorStorage, Quantizer from ...constants import TE_DType as torch_to_transformer_engine_dtype -from ..quantized_tensor import Quantizer - from ...utils import is_non_tn_fp8_gemm_supported, _empty_tensor diff --git a/transformer_engine/pytorch/tensor/storage/mxfp8_tensor_storage.py b/transformer_engine/pytorch/tensor/storage/mxfp8_tensor_storage.py index c1f30146c9..2cca0829db 100644 --- a/transformer_engine/pytorch/tensor/storage/mxfp8_tensor_storage.py +++ b/transformer_engine/pytorch/tensor/storage/mxfp8_tensor_storage.py @@ -13,12 +13,10 @@ import transformer_engine_torch as tex from transformer_engine_torch import DType as TE_DType -from ..quantized_tensor import QuantizedTensorStorage +from ...quantized_tensor import QuantizedTensorStorage, Quantizer from ...constants import TE_DType as torch_to_transformer_engine_dtype -from ..quantized_tensor import Quantizer - from ...utils import _empty_tensor diff --git a/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py b/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py index 350103f7ca..67543a8e2a 100644 --- a/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py +++ b/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py @@ -16,10 +16,9 @@ # import transformer_engine_torch as tex from transformer_engine_torch import DType as TE_DType -from ..quantized_tensor import QuantizedTensorStorage +from ...quantized_tensor import QuantizedTensorStorage, Quantizer # from ...constants import TE_DType as torch_to_transformer_engine_dtype -from ..quantized_tensor import Quantizer from ...utils import _empty_tensor diff --git a/transformer_engine/pytorch/tensor/utils.py b/transformer_engine/pytorch/tensor/utils.py index 72c465edb2..8354823b32 100644 --- a/transformer_engine/pytorch/tensor/utils.py +++ b/transformer_engine/pytorch/tensor/utils.py @@ -4,13 +4,13 @@ """Helper functions for using fp8 tensors as weights""" -import os -from typing import Optional, List, Union +from typing import Optional, Union, List import torch + import transformer_engine_torch as tex from transformer_engine_torch import multi_tensor_scale, multi_tensor_compute_scale_and_scale_inv -from .quantized_tensor import QuantizedTensor, Quantizer, QuantizedTensorStorage +from ..quantized_tensor import QuantizedTensor, Quantizer, QuantizedTensorStorage from .float8_tensor import Float8Tensor, Float8Quantizer, Float8CurrentScalingQuantizer from .mxfp8_tensor import MXFP8Tensor, MXFP8Quantizer from .float8_blockwise_tensor import Float8BlockwiseQTensor, Float8BlockQuantizer @@ -459,18 +459,13 @@ def post_all_gather_processing(model_weights: Union[torch.Tensor, List[torch.Ten raise ValueError(f"post_processing for {type(model_weight)} is not supported") -def is_experimental(x: Optional[Union[Quantizer, QuantizedTensorStorage]] = None) -> bool: - """Check if an environment or object is using experimental Kitchen middleware. +def is_custom(x: Optional[Union[Quantizer, QuantizedTensorStorage]] = None) -> bool: + """Check if an object is custom. Returns False if x is a torch.Tensor. """ - # Detect if the environment is experimental - if x is None: - return int(os.getenv("QAT_PARAMS", "0")) > 0 - - # Detect if the object is experimental - if isinstance(x, torch.Tensor): + if x is None or isinstance(x, torch.Tensor): return False if not isinstance(x, (Quantizer, QuantizedTensorStorage)): raise AssertionError("Object must be a Quantizer or QuantizedTensorStorage instance") - return hasattr(x, "experimental") and x.experimental + return hasattr(x, "custom") and x.custom diff --git a/transformer_engine/pytorch/utils.py b/transformer_engine/pytorch/utils.py index 2be0aed4a8..90c6289963 100644 --- a/transformer_engine/pytorch/utils.py +++ b/transformer_engine/pytorch/utils.py @@ -12,7 +12,7 @@ import torch from . import torch_version -from .tensor.quantized_tensor import Quantizer +from .quantized_tensor import Quantizer from ..debug.pytorch.debug_quantization import DebugQuantizedTensor