Cherry pick Habana software v1.21.0 commits

.azure-pipelines/scripts/ut/3x/run_3x_pt.sh

@@ -25,6 +25,7 @@ cd /neural-compressor/test/3x || exit 1
 rm -rf tensorflow
 rm -rf torch/algorithms/fp8_quant
 rm -rf torch/quantization/fp8_quant
+rm -rf torch/algorithms/fp8_quant_xpu
 
 LOG_DIR=/neural-compressor/log_dir
 mkdir -p ${LOG_DIR}

examples/3.x_api/pytorch/nlp/huggingface_models/language-modeling/quantization/weight_only/requirements.txt

@@ -13,3 +13,4 @@ peft
 auto_round
 numba
 tbb
+safetensors >= 0.5.3

examples/3.x_api/pytorch/nlp/huggingface_models/language-modeling/quantization/weight_only/run_benchmark.sh

@@ -79,6 +79,8 @@ function run_benchmark {
         model_name_or_path="facebook/opt-125m"
     elif [ "${topology}" = "llama2_7b_gptq_int4" ]; then
         model_name_or_path="meta-llama/Llama-2-7b-hf"
+    elif [ "${topology}" = "llama2_7b_gptq_int4_blockwise" ]; then
+        model_name_or_path="meta-llama/Llama-2-7b-hf"
     elif [ "${topology}" = "llama2_7b_gptq_nf4_dq_bnb" ]; then
         model_name_or_path="meta-llama/Llama-2-7b-hf"
     elif [ "${topology}" = "llama2_7b_gptq_int4_dq_ggml" ]; then

examples/3.x_api/pytorch/nlp/huggingface_models/language-modeling/quantization/weight_only/run_clm_no_trainer.py

@@ -1,11 +1,10 @@
 import argparse
 import os
 import sys
+import gc
 
 sys.path.append('./')
 import time
-import json
-import re
 import torch
 from datasets import load_dataset
 from functools import lru_cache
@@ -15,6 +14,7 @@
 from torch.utils.data import DataLoader
 from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer
 from neural_compressor.torch.utils import is_hpex_available
+from neural_compressor.common.utils import LazyImport
 
 
 if is_hpex_available():
@@ -98,6 +98,9 @@
                     help='Calibration dataset sequence max length, '
                         'this should align with your model config, '
                         'and your dataset builder args: args.pad_max_length.')
+parser.add_argument("--gptq_blockwise", action="store_true",
+                    help="Whether to quantize blockwise.")
+parser.add_argument("--blockwise_load_folder", default=None, type=str, help="Directory to load blockwise checkpoints from.")
 
 # =============AWQ configs====================
 parser.add_argument("--use_auto_scale", action="store_true",
@@ -150,6 +153,10 @@
 
 args = parser.parse_args()
 calib_size = 1
+quant_start_time = time.time()
+
+if args.use_hf_format:
+    assert not args.blockwise_gptq, "blockwise_gptq is not supported with use_hf_format"
 
 
 def compare_versions(v1, v2):
@@ -268,7 +275,7 @@ def evaluate(self, model):
         return acc
 
 
-def get_user_model():
+def get_user_model(empty_model=False):
     torchscript = False
     if args.woq_algo in ["AWQ", "TEQ"]:
         torchscript = True
@@ -281,16 +288,28 @@ def get_user_model():
             args.model,
             trust_remote_code=args.trust_remote_code,
             attn_implementation=args.autoround_attn_implementation,
-            revision=args.revision,
+            revision=args.revision
         )
     else:
-        user_model = AutoModelForCausalLM.from_pretrained(
-            args.model,
-            torchscript=torchscript,  # torchscript will force `return_dict=False` to avoid jit errors
-            trust_remote_code=args.trust_remote_code,
-            revision=args.revision,
-        )
-    tokenizer = AutoTokenizer.from_pretrained(args.model)
+        from neural_compressor.torch.algorithms.layer_wise import load_first_layer_only
+        config = AutoConfig.from_pretrained(args.model)
+
+        if empty_model or args.gptq_blockwise:
+            from accelerate import init_empty_weights
+            with init_empty_weights():
+                user_model = AutoModelForCausalLM.from_config(config)
+
+        if args.gptq_blockwise:  # if block-wise, load only first block and peripherals
+            load_first_layer_only(user_model, args.model)
+
+        elif not empty_model:
+            user_model = AutoModelForCausalLM.from_pretrained(
+                args.model,
+                torchscript=torchscript,  # torchscript will force `return_dict=False` to avoid jit errors
+                trust_remote_code=args.trust_remote_code,
+                revision=args.revision,
+            )
+    tokenizer = AutoTokenizer.from_pretrained(args.model) 
     user_model = user_model.float()
     if args.woq_algo == 'AutoRound':
         user_model.to(torch.float32)
@@ -303,8 +322,6 @@ def get_user_model():
         from peft import PeftModel
         user_model = PeftModel.from_pretrained(user_model, args.peft_model_id)
 
-    # to channels last
-    user_model = user_model.to(memory_format=torch.channels_last)
     user_model.eval()
     return user_model, tokenizer
 
@@ -450,6 +467,8 @@ def run_fn_for_gptq(model, dataloader_for_calibration, *args):
                 double_quant_use_sym=args.double_quant_use_sym,
                 double_quant_group_size=args.double_quant_group_size,
                 quant_lm_head=args.quant_lm_head,
+                use_layer_wise=False,
+                use_block_wise=args.gptq_blockwise
             )
         user_model = prepare(model=user_model, quant_config=quant_config)
         run_fn_for_gptq(user_model, dataloader_for_calibration)
@@ -554,31 +573,47 @@ def run_fn_for_gptq(model, dataloader_for_calibration, *args):
             example_inputs=example_inputs,
         )
 
-    print("saving weight-only quantized model")
-    if args.use_hf_format:
-        user_model.save(args.output_dir, format="huggingface")
+    if args.gptq_blockwise:
+        from neural_compressor.torch.algorithms.layer_wise import LWQ_WORKSPACE
+        kwargs = {'blockwise': True, 'blockwise_load_folder': args.blockwise_load_folder}
+        user_model.save(args.output_dir, **kwargs)
         tokenizer.save_pretrained(args.output_dir)
-    else:
-        user_model.save(args.output_dir)
-    print("saved weight-only quantized model")
+        config = AutoConfig.from_pretrained(args.model)
+        config.save_pretrained(args.output_dir)
 
+        if not args.blockwise_load_folder:
+            import shutil
+            shutil.rmtree(LWQ_WORKSPACE, ignore_errors=True)
+        args.gptq_blockwise = False # no need to do block-wise after quant (affects get_user_model)
+    else:
+        if args.use_hf_format:
+            user_model.save(args.output_dir, format="huggingface")
+            tokenizer.save_pretrained(args.output_dir)
+        else:
+            user_model.save(args.output_dir)
+    print(f'Quantization took: {time.time() - quant_start_time} seconds')
 
 if args.load:
-    print("load weight-only quantized model")
-
-    from neural_compressor.torch.quantization import load
     if args.use_hf_format:
         user_model = load(args.model, format="huggingface", device=device)
     else:
-        user_model, _ = get_user_model()
+        from neural_compressor.torch.quantization import load
+        kwargs = {'sharded_checkpoints': True} if args.gptq_blockwise else {}
+        empty_model = False
+        if args.gptq_blockwise:
+            empty_model = True
+            args.gptq_blockwise = False
+
+        user_model, _ = get_user_model(empty_model=empty_model)
+        tokenizer = AutoTokenizer.from_pretrained(args.model)
         config = AutoConfig.from_pretrained(args.model)
+
         user_model = load(
             os.path.abspath(os.path.expanduser(args.output_dir)),
             user_model,
-            device=device,
+            device="hpu" if is_hpex_available() else "cpu", **kwargs
         )
         setattr(user_model, "config", config)
-    tokenizer = AutoTokenizer.from_pretrained(args.model)
 else:
     user_model, tokenizer = get_user_model()
 

examples/3.x_api/pytorch/nlp/huggingface_models/language-modeling/quantization/weight_only/run_quant.sh

@@ -55,6 +55,9 @@ function run_tuning {
     elif [ "${topology}" = "llama2_7b_gptq_int4" ]; then
         model_name_or_path="meta-llama/Llama-2-7b-hf"
         extra_cmd=$extra_cmd" --woq_algo GPTQ --woq_bits 4 --woq_group_size 128 --woq_scheme asym --woq_use_mse_search --gptq_use_max_length"
+    elif [ "${topology}" = "llama2_7b_gptq_int4_blockwise" ]; then
+        model_name_or_path="meta-llama/Llama-2-7b-hf"
+        extra_cmd=$extra_cmd" --woq_algo GPTQ --woq_bits 4 --woq_group_size 128 --woq_scheme asym --woq_use_mse_search --gptq_use_max_length --gptq_blockwise"        
     elif [ "${topology}" = "llama2_7b_gptq_nf4_dq_bnb" ]; then
         model_name_or_path="meta-llama/Llama-2-7b-hf"
         extra_cmd=$extra_cmd" --woq_algo GPTQ --woq_bits 4 --woq_group_size 128 --woq_scheme asym --woq_use_mse_search --gptq_use_max_length"

neural_compressor/common/base_config.py

@@ -189,6 +189,7 @@ class BaseConfig(ABC):
 
     name = BASE_CONFIG
     params_list = []
+    _is_initialized = False
 
     def __init__(self, white_list: Optional[List[OP_NAME_OR_MODULE_TYPE]] = DEFAULT_WHITE_LIST) -> None:
         """Initialize the BaseConfig.
@@ -220,6 +221,14 @@ def _post_init(self):
                 f"The white list should be one of {DEFAULT_WHITE_LIST}, {EMPTY_WHITE_LIST},"
                 " a not empty list, but got {self.white_list}"
             )
+        self._is_initialized = True
+
+    def __setattr__(self, name, value):
+        """Override the setattr function to propagate updates."""
+        super().__setattr__(name, value)
+        if self._is_initialized and name in self.params_list:
+            self._is_initialized = False
+            self._post_init()
 
     @property
     def white_list(self):
@@ -683,6 +692,13 @@ def __add__(self, other: BaseConfig) -> BaseConfig:
             self.config_list.append(other)
         return self
 
+    def __setattr__(self, name, value):
+        """Override the setattr function to propagate updates."""
+        ABC.__setattr__(self, name, value)
+        for config in self.config_list:
+            if hasattr(config, name):
+                setattr(config, name, value)
+
     def to_dict(self, params_list=[], operator2str=None):
         """Converts the configuration object to a dictionary.
 
@@ -884,7 +900,6 @@ class Options:
 
     def __init__(self, random_seed=1978, workspace=DEFAULT_WORKSPACE, resume_from=None, tensorboard=False):
         """Init an Option object."""
-        os.makedirs(workspace, exist_ok=True)
         self.random_seed = random_seed
         self.workspace = workspace
         self.resume_from = resume_from

neural_compressor/common/utils/constants.py

@@ -34,6 +34,7 @@
 TEQ = "teq"  # pragma: no cover
 AUTOROUND = "autoround"
 FP8_QUANT = "fp8_quant"
+HYBRID_GPTQ = "hybrid_gptq"
 MX_QUANT = "mx_quant"
 MIXED_PRECISION = "mixed_precision"
 
@@ -51,12 +52,13 @@
 
 
 class Mode(Enum):
-    """Enumeration class representing different modes of the quantizer execution."""
+    """Enumeration class representing different modes of the quantization."""
 
     PREPARE = "prepare"
     CONVERT = "convert"
     QUANTIZE = "quantize"
     LOAD = "load"
+    SAVE = "save"
 
 
 SERVER_PROCESSOR_BRAND_KEY_WORLD_LST = ["Xeon"]

neural_compressor/common/utils/logger.py

@@ -161,6 +161,8 @@ def _get_log_msg(mode):
         log_msg = "Conversion"
     elif mode == Mode.LOAD:  # pragma: no cover
         log_msg = "Loading"
+    elif mode == Mode.SAVE:  # pragma: no cover
+        log_msg = "Saving"
     return log_msg
 
 

neural_compressor/torch/algorithms/fp8_quant/_core/common.py

@@ -19,13 +19,20 @@
 import numpy as np
 import torch
 from enum import Enum, auto
-
+from functools import lru_cache 
 from .._quant_common.quant_config import get_hqt_config
 from ..utils.logger import logger
 from neural_compressor.torch.algorithms.fp8_quant.model_configs import ModuleConfig
 
 UNMEASURED_MODELS = "UnmeasuredModels"
 
+def dequant_original_fp8_weight_if_needed(mod: torch.nn.Module, param: torch.Tensor) -> torch.Tensor:
+    if param.dtype in [torch.float8_e4m3fn]:
+        if hasattr(mod, "get_dequant_weights_func"):
+            dequant_weights_func = mod.get_dequant_weights_func()
+            if dequant_weights_func is not None:
+                param = dequant_weights_func(mod)
+    return param
 
 class QuantTensorType(Enum):
     MEASUREMENTS = auto()
@@ -185,3 +192,8 @@ def convert_scales_to_tensors_dict(scales_obj, scales_file_format, hp_dtype, dev
 def get_device_type_for_scales(mod):
     config = get_hqt_config(mod).cfg
     return config["device_for_scales"]
+
+
+@lru_cache
+def is_runtime_scale_patching():
+    return os.getenv("RUNTIME_SCALE_PATCHING", "False").lower() in ["true", "1"]

neural_compressor/torch/algorithms/fp8_quant/_core/fp_utils.py

@@ -13,14 +13,17 @@
 # limitations under the License.
 
 import torch
-import habana_frameworks.torch.core as htcore
-import habana_frameworks.torch.utils.experimental as htexp
 from .common import ModuleConfig
-from neural_compressor.torch.utils.auto_accelerator import auto_detect_accelerator
+from neural_compressor.torch.utils.auto_accelerator import auto_detect_accelerator, INCAcceleratorType
 cur_accelerator = auto_detect_accelerator()
 
-GAUDI2 = htexp.synDeviceType.synDeviceGaudi2
-GAUDI3 = htexp.synDeviceType.synDeviceGaudi3
+descale_fcn = lambda x, scale: torch.mul(x, scale)
+scale_fcn = lambda x, scale: torch.div(x, scale)
+cast_fcn = lambda x, dtype: x.to(dtype=dtype)
+cast_to_fp8_fcn = lambda x, dtype, scale_inv=None: torch.ops.hpu.cast_to_fp8_v2(x, scale_inv, False, False, dtype)[0]
+
+GAUDI2 = INCAcceleratorType.GAUDI2
+GAUDI3 = INCAcceleratorType.GAUDI3
 
 EXP_WIDTH = {
     torch.float32: 8,
@@ -65,22 +68,30 @@ def get_default_exp_bias(dtype):
     torch.bfloat16: torch.finfo(torch.bfloat16).max,
     torch.float8_e4m3fn: torch.finfo(torch.float8_e4m3fn).max,
     # float8_e4m3fn data type is 8-bit floating point consist of Exponent: 4, Mantissa: 3, bias: 7. It's supported by Gaudi3.
-    torch.float8_e5m2: torch.finfo(torch.float8_e5m2).max,
+    torch.float8_e5m2: torch.finfo(torch.float8_e5m2).max
     # float8_e5m2 data type is 8-bit floating point consist of Exponent: 5, Mantissa: 2, bias: 15. IEEE 754, with NaN and inf.
-    torch.float8_e4m3fnuz: torch.finfo(torch.float8_e4m3fnuz).max,
-    # float8_e4m3fnuz data type is 8-bit floating point consist of Exponent: 4, Mantissa: 3, bias: 8 with 1 sign bit. It's supported by Gaudi2.
 }
-
+# TODO FSW-12066 solve fp_utils
+try:
+    MAX_RANGE[torch.float8_e4m3fnuz] = torch.finfo(torch.float8_e4m3fnuz).max
+    # float8_e4m3fnuz data type is 8-bit floating point consist of Exponent: 4, Mantissa: 3, bias: 8 with 1 sign bit. It's supported by Gaudi2.
+except AttributeError as e:
+    pass
 
 def get_fullscale(dtype, device, exp_bias=None):
     default_exp_bias = get_default_exp_bias(dtype)
+    fullscale = 1
     if device == GAUDI2 and dtype == torch.float8_e4m3fn:
-        fullscale = MAX_RANGE[torch.float8_e4m3fnuz]
+        # TODO FSW-12066 solve fp_utils
+        try:
+            fullscale = MAX_RANGE[torch.float8_e4m3fnuz]
+        except AttributeError as e:
+            pass
     else:
         fullscale = MAX_RANGE[dtype]
     exp_bias = default_exp_bias if exp_bias is None else exp_bias
     fullscale = fullscale * (2 ** (default_exp_bias - exp_bias))
-    return fullscale
+    return float(fullscale)
 
 
 def get_fullscales_by_expbias_set(dtype, device, expbias_set):
@@ -97,8 +108,8 @@ def get_fp8_hw_alligned_scales(dtype, device):
 
 
 DEVICES_SCALE_FACTORS = {
-    htexp.synDeviceType.synDeviceGaudi2: 4,
-    htexp.synDeviceType.synDeviceGaudi3: 1,
+    INCAcceleratorType.GAUDI2: 4,
+    INCAcceleratorType.GAUDI3: 1,
 }
 FP8_143_SCALES = {
     device: get_fp8_hw_alligned_scales(torch.float8_e4m3fn, device) for device in DEVICES_SCALE_FACTORS.keys()