From 043574965273d4a70bc5beeb7d44eba1c4a0ddad Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 31 Aug 2023 19:48:47 +0000 Subject: [PATCH 1/9] add new arg for gptq --- src/transformers/utils/quantization_config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index e302d621baa1..9b698947653d 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -346,6 +346,9 @@ class GPTQConfig(QuantizationConfigMixin): The pad token id. Needed to prepare the dataset when `batch_size` > 1. disable_exllama (`bool`, *optional*, defaults to `False`): Whether to use exllama backend. Only works with `bits` = 4. + max_input_length (`int`, *optional*) + The maximum input length. This is needed to initialize a buffer that depends on the maximum expected input + length. It is specific to the exllama backend with act-order. """ def __init__( @@ -365,6 +368,7 @@ def __init__( batch_size: int = 1, pad_token_id: Optional[int] = None, disable_exllama: bool = False, + max_input_length: Optional[int] = None, **kwargs, ): self.quant_method = QuantizationMethod.GPTQ @@ -383,11 +387,12 @@ def __init__( self.batch_size = batch_size self.pad_token_id = pad_token_id self.disable_exllama = disable_exllama + self.max_input_length = max_input_length self.post_init() def get_loading_attributes(self): attibutes_dict = copy.deepcopy(self.__dict__) - loading_attibutes = ["disable_exllama", "use_cuda_fp16"] + loading_attibutes = ["disable_exllama", "use_cuda_fp16", "max_input_length"] loading_attibutes_dict = {i: j for i, j in attibutes_dict.items() if i in loading_attibutes} return loading_attibutes_dict From 513cab37774bdaf917c7d5591c9e0d7780b8e82a Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 31 Aug 2023 20:58:12 +0000 Subject: [PATCH 2/9] add tests --- tests/quantization/gptq/test_gptq.py | 61 ++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index c7530471fa27..b9b5fbb6ef61 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -86,6 +86,8 @@ class GPTQTest(unittest.TestCase): EXPECTED_OUTPUTS = set() EXPECTED_OUTPUTS.add("Hello my name is John and I am a professional photographer. I") + EXPECTED_OUTPUTS.add("Hello my name is John, I am a professional photographer and I") + EXPECTED_OUTPUTS.add("Hello my name is John, I am a student in the University of") EXPECTED_OUTPUTS.add("Hello my name is John and I am a very good looking man.") EXPECTED_OUTPUTS.add("Hello my name is Alyson, I am a student in the") EXPECTED_OUTPUTS.add("Hello my name is Alyson and I am a very sweet,") @@ -236,6 +238,65 @@ class GPTQTestDeviceMapExllama(GPTQTest): disable_exllama = False +@slow +@require_optimum +@require_auto_gptq +@require_torch_gpu +@require_accelerate +class GPTQTestActOrderExllama(unittest.TestCase): + EXPECTED_OUTPUTS = set() + EXPECTED_OUTPUTS.add("Hello my name is Katie and I am a 20 year") + model_name = "TheBloke/Llama-2-7B-GPTQ" + revision = "gptq-4bit-128g-actorder_True" + input_text = "Hello my name is" + + @classmethod + def setUpClass(cls): + """ + Setup quantized model + """ + + cls.quantization_config = GPTQConfig(bits=4, disable_exllama=False, max_input_length=4028) + cls.quantized_model = AutoModelForCausalLM.from_pretrained( + cls.model_name, + revision=cls.revision, + torch_dtype=torch.float16, + device_map={"": 0}, + quantization_config=cls.quantization_config, + ) + cls.tokenizer = AutoTokenizer.from_pretrained(cls.model_name, use_fast=True) + + def check_inference_correctness(self, model): + # Check that inference pass works on the model + encoded_input = self.tokenizer(self.input_text, return_tensors="pt") + + # Check the exactness of the results + output_sequences = model.generate(input_ids=encoded_input["input_ids"].to(0), max_new_tokens=10) + + # Get the generation + self.assertIn(self.tokenizer.decode(output_sequences[0], skip_special_tokens=True), self.EXPECTED_OUTPUTS) + + def test_generate_quality(self): + self.check_inference_correctness(self.quantized_model) + + def test_exllama_max_input_length(self): + """ + Test if the max_input_length works with exllama + act_order + """ + + prompt = "I am in Paris and" * 1000 + inp = self.tokenizer(prompt, return_tensors="pt").to(0) + self.assertTrue(inp["input_ids"].shape[1] > 4028) + with self.assertRaises(RuntimeError) as cm: + self.quantized_model.generate(**inp, num_beams=1, min_new_tokens=3, max_new_tokens=3) + self.assertTrue("temp_state buffer is too small" in str(cm.exception)) + + prompt = "I am in Paris and" * 500 + inp = self.tokenizer(prompt, return_tensors="pt").to(0) + self.assertTrue(inp["input_ids"].shape[1] < 4028) + self.quantized_model.generate(**inp, num_beams=1, min_new_tokens=3, max_new_tokens=3) + + # fail when run all together @pytest.mark.skip @require_accelerate From 4db188ac9e474136df519143afd2f1b2fbe50629 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 31 Aug 2023 20:58:35 +0000 Subject: [PATCH 3/9] add min version autogptq --- src/transformers/modeling_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 66de85397ab3..1db5d5c4cc19 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2560,7 +2560,7 @@ def from_pretrained( logger.warning( "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a " "`quantization_config` attribute and has already quantized weights. However, loading attributes" - " (e.g. disable_exllama, use_cuda_fp16) will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored." + " (e.g. disable_exllama, use_cuda_fp16, max_input_length) will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored." ) if ( quantization_method_from_args == QuantizationMethod.GPTQ @@ -2570,7 +2570,11 @@ def from_pretrained( raise RuntimeError("GPU is required to quantize or run quantize model.") elif not (is_optimum_available() and is_auto_gptq_available()): raise ImportError( - "Loading GPTQ quantized model requires optimum library : `pip install optimum` and auto-gptq library 'pip install auto-gptq'" + "Loading GPTQ quantized model requires optimum : `pip install optimum` and auto-gptq library 'pip install auto-gptq'" + ) + elif version.parse(importlib.metadata.version("auto_gptq")) >= version.parse("0.4.2"): + raise ImportError( + "You need a version of auto_gptq > 0.4.2 to use GPTQ: `pip install --upgrade auto-gptq`" ) else: # Need to protect the import From bffa86a1a3bb97f567d8dfd6646ceb57a2df78c4 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 31 Aug 2023 20:59:22 +0000 Subject: [PATCH 4/9] fix order --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 1db5d5c4cc19..16544dd4b66e 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2572,7 +2572,7 @@ def from_pretrained( raise ImportError( "Loading GPTQ quantized model requires optimum : `pip install optimum` and auto-gptq library 'pip install auto-gptq'" ) - elif version.parse(importlib.metadata.version("auto_gptq")) >= version.parse("0.4.2"): + elif version.parse(importlib.metadata.version("auto_gptq")) < version.parse("0.4.2"): raise ImportError( "You need a version of auto_gptq > 0.4.2 to use GPTQ: `pip install --upgrade auto-gptq`" ) From ae91f07dbaffd86fcb134a9f359b2cf70b819e5c Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 31 Aug 2023 21:06:11 +0000 Subject: [PATCH 5/9] skip test --- tests/quantization/gptq/test_gptq.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index b9b5fbb6ef61..8a4b9ae5b3ff 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -279,6 +279,8 @@ def check_inference_correctness(self, model): def test_generate_quality(self): self.check_inference_correctness(self.quantized_model) + # this test will fail until the next release of optimum + @pytest.mark.skip def test_exllama_max_input_length(self): """ Test if the max_input_length works with exllama + act_order From ac2cdb55598474b6731646d569151f7639ae89b6 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 31 Aug 2023 21:17:51 +0000 Subject: [PATCH 6/9] fix --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 16544dd4b66e..f60e266de77c 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2574,7 +2574,7 @@ def from_pretrained( ) elif version.parse(importlib.metadata.version("auto_gptq")) < version.parse("0.4.2"): raise ImportError( - "You need a version of auto_gptq > 0.4.2 to use GPTQ: `pip install --upgrade auto-gptq`" + "You need a version of auto_gptq >= 0.4.2 to use GPTQ: `pip install --upgrade auto-gptq`" ) else: # Need to protect the import From 8f52e9c0db947b11cd664c28be2d5560adb904a1 Mon Sep 17 00:00:00 2001 From: Marc Sun <57196510+SunMarc@users.noreply.github.com> Date: Fri, 1 Sep 2023 11:10:45 -0400 Subject: [PATCH 7/9] Update src/transformers/modeling_utils.py Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index f60e266de77c..bfaabb2690f1 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2570,7 +2570,7 @@ def from_pretrained( raise RuntimeError("GPU is required to quantize or run quantize model.") elif not (is_optimum_available() and is_auto_gptq_available()): raise ImportError( - "Loading GPTQ quantized model requires optimum : `pip install optimum` and auto-gptq library 'pip install auto-gptq'" + "Loading a GPTQ quantized model requires optimum (`pip install optimum`) and auto-gptq library (`pip install auto-gptq`)" ) elif version.parse(importlib.metadata.version("auto_gptq")) < version.parse("0.4.2"): raise ImportError( From 49034597026dbcc110c568daa4d36e5dd5de9f3a Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Fri, 1 Sep 2023 15:27:52 +0000 Subject: [PATCH 8/9] fix style --- tests/quantization/gptq/test_gptq.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index 8a4b9ae5b3ff..785d0ef20b2d 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -244,6 +244,12 @@ class GPTQTestDeviceMapExllama(GPTQTest): @require_torch_gpu @require_accelerate class GPTQTestActOrderExllama(unittest.TestCase): + """ + Test GPTQ model with exllama kernel and desc_act=True (also known as act-order). + More information on those arguments here: + https://huggingface.co/docs/transformers/main_classes/quantization#transformers.GPTQConfig + """ + EXPECTED_OUTPUTS = set() EXPECTED_OUTPUTS.add("Hello my name is Katie and I am a 20 year") model_name = "TheBloke/Llama-2-7B-GPTQ" @@ -267,6 +273,12 @@ def setUpClass(cls): cls.tokenizer = AutoTokenizer.from_pretrained(cls.model_name, use_fast=True) def check_inference_correctness(self, model): + """ + Test the generation quality of the quantized model and see that we are matching the expected output. + Given that we are operating on small numbers + the testing model is relatively small, we might not get + the same output across GPUs. So we'll generate few tokens (5-10) and check their output. + """ + # Check that inference pass works on the model encoded_input = self.tokenizer(self.input_text, return_tensors="pt") @@ -277,13 +289,16 @@ def check_inference_correctness(self, model): self.assertIn(self.tokenizer.decode(output_sequences[0], skip_special_tokens=True), self.EXPECTED_OUTPUTS) def test_generate_quality(self): + """ + Simple test to check the quality of the model by comapring the the generated tokens with the expected tokens + """ self.check_inference_correctness(self.quantized_model) # this test will fail until the next release of optimum @pytest.mark.skip - def test_exllama_max_input_length(self): + def test_max_input_length(self): """ - Test if the max_input_length works with exllama + act_order + Test if the max_input_length works. It modifies the maximum input length that of the model that runs with exllama backend. """ prompt = "I am in Paris and" * 1000 From 2c7b0262c2b712631cd0ab6c9685a80742caaa1c Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 5 Sep 2023 15:57:34 +0000 Subject: [PATCH 9/9] change model path --- tests/quantization/gptq/test_gptq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index 785d0ef20b2d..d715bd56c02e 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -252,7 +252,7 @@ class GPTQTestActOrderExllama(unittest.TestCase): EXPECTED_OUTPUTS = set() EXPECTED_OUTPUTS.add("Hello my name is Katie and I am a 20 year") - model_name = "TheBloke/Llama-2-7B-GPTQ" + model_name = "hf-internal-testing/Llama-2-7B-GPTQ" revision = "gptq-4bit-128g-actorder_True" input_text = "Hello my name is"