From c919fb14ee18764c00ebb9969ca124e235bdc90f Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 9 Oct 2025 08:03:56 +0000 Subject: [PATCH 1/5] make loading gpt-oss-BF16 faster. Linked to unsloth-zoo PR #314 --- unsloth/models/mapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index 600396ed46..28ef63986e 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -928,6 +928,7 @@ ), "unsloth/gpt-oss-20b-unsloth-bnb-4bit" : ( "unsloth/gpt-oss-20b", + "unsloth/gpt-oss-20b-BF16", "openai/gpt-oss-20b", "unsloth/gpt-oss-20b-unsloth-bnb-4bit", ), From 2f4b0c267e2e52bdddd413b2e0c4b8a9a8a68c4f Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 20 Nov 2025 07:34:00 +0000 Subject: [PATCH 2/5] fix model loading and clean merged model directory --- unsloth/save.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index 810bf9062a..326ba02a04 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2769,7 +2769,7 @@ def unsloth_save_pretrained_torchao( for _ in range(3): gc.collect() - from transformers import AutoModel, AutoTokenizer, TorchAoConfig + from transformers import AutoModelForCausalLM, AutoTokenizer, TorchAoConfig, AutoModelForImageTextToText, AutoProcessor from torchao import quantize_ if torchao_config is None: @@ -2778,23 +2778,35 @@ def unsloth_save_pretrained_torchao( print( "Unsloth: You did not specify a `torchao_config`, so defaulting to `Int8DynamicActivationInt8WeightConfig`" ) - torchao_config = Int8DynamicActivationInt8WeightConfig() + torchao_config = Int8DynamicActivationInt8WeightConfig(group_size=32) quantization_config = TorchAoConfig(quant_type = torchao_config) - tokenizer = AutoTokenizer.from_pretrained(arguments["save_directory"]) + is_vlm = False + if hasattr(self, "config") and hasattr(self.config, "architectures"): + is_vlm = any( + x.endswith(("ForConditionalGeneration", "ForVisionText2Text")) + for x in self.config.architectures + ) + is_vlm = is_vlm or hasattr(self.config, "vision_config") + auto_model = AutoModelForImageTextToText if is_vlm else AutoModelForCausalLM + auto_processor = AutoProcessor if is_vlm else AutoTokenizer + + tokenizer = auto_processor.from_pretrained(arguments["save_directory"]) # TorchAO must only use bfloat16 for loading (float16 fails) if HAS_TORCH_DTYPE: kwargs = {"torch_dtype": torch.bfloat16} else: kwargs = {"dtype": torch.bfloat16} - model = AutoModel.from_pretrained( + + model = auto_model.from_pretrained( arguments["save_directory"], device_map = "auto", quantization_config = quantization_config, **kwargs, ) + torchao_save_directory = save_directory + "-torchao" # TorchAO does not support safe_serialization right now 0.14.0 seems broken! @@ -2812,6 +2824,12 @@ def unsloth_save_pretrained_torchao( torchao_save_directory, safe_serialization = safe_serialization ) tokenizer.save_pretrained(torchao_save_directory) + if os.path.exists(save_directory): + try: + import shutil + shutil.rmtree(save_directory) + except: + pass for _ in range(3): gc.collect() From d75ae63c11b3956709916699620fcd358c5e763e Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 20 Nov 2025 07:42:50 +0000 Subject: [PATCH 3/5] revert default quant --- unsloth/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/save.py b/unsloth/save.py index 326ba02a04..d1ffc18497 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2778,7 +2778,7 @@ def unsloth_save_pretrained_torchao( print( "Unsloth: You did not specify a `torchao_config`, so defaulting to `Int8DynamicActivationInt8WeightConfig`" ) - torchao_config = Int8DynamicActivationInt8WeightConfig(group_size=32) + torchao_config = Int8DynamicActivationInt8WeightConfig() quantization_config = TorchAoConfig(quant_type = torchao_config) is_vlm = False From fdb789f7955617263e43fc60c494c4794a156309 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 07:44:33 +0000 Subject: [PATCH 4/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/save.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index d1ffc18497..0a8f02d90f 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2769,7 +2769,13 @@ def unsloth_save_pretrained_torchao( for _ in range(3): gc.collect() - from transformers import AutoModelForCausalLM, AutoTokenizer, TorchAoConfig, AutoModelForImageTextToText, AutoProcessor + from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + TorchAoConfig, + AutoModelForImageTextToText, + AutoProcessor, + ) from torchao import quantize_ if torchao_config is None: @@ -2806,7 +2812,6 @@ def unsloth_save_pretrained_torchao( **kwargs, ) - torchao_save_directory = save_directory + "-torchao" # TorchAO does not support safe_serialization right now 0.14.0 seems broken! @@ -2827,6 +2832,7 @@ def unsloth_save_pretrained_torchao( if os.path.exists(save_directory): try: import shutil + shutil.rmtree(save_directory) except: pass From 1a816ad1aebc5e8f88f74aa61af044410709723f Mon Sep 17 00:00:00 2001 From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:49:04 +0200 Subject: [PATCH 5/5] revert mapper.py --- unsloth/models/mapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index 139061c7cf..f2dad24225 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -928,7 +928,6 @@ ), "unsloth/gpt-oss-20b-unsloth-bnb-4bit" : ( "unsloth/gpt-oss-20b", - "unsloth/gpt-oss-20b-BF16", "openai/gpt-oss-20b", "unsloth/gpt-oss-20b-unsloth-bnb-4bit", ),