From 3c7785eae7b7041e76695c66e82a3d4a5d82d5c5 Mon Sep 17 00:00:00 2001 From: alkinun Date: Fri, 17 Jul 2026 20:11:55 +0300 Subject: [PATCH 01/15] Fix text-only VLM CPT packing truncation --- studio/backend/core/training/trainer.py | 12 ++- tests/utils/test_packing.py | 102 ++++++++++++++++++++++++ unsloth/models/rl_replacements.py | 25 ++++++ unsloth/trainer.py | 22 ++--- 4 files changed, 147 insertions(+), 14 deletions(-) diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 883a535a89e..e540a5fdbc1 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -3425,15 +3425,19 @@ def audio_vlm_collate_fn(examples): logger.info( f"CPT: using UnslothTrainer with embedding_learning_rate={embedding_lr}\n" ) + cpt_args = _UnslothTrainingArguments( + embedding_learning_rate = embedding_lr, + **config_args, + ) + if config_args.get("packing", False): + cpt_args.packing_strategy = "wrapped" + logger.info("CPT packing strategy: wrapped\n") trainer_kwargs = { "model": self.model, "tokenizer": sft_tokenizer, "train_dataset": dataset["dataset"], "data_collator": data_collator, - "args": _UnslothTrainingArguments( - embedding_learning_rate = embedding_lr, - **config_args, - ), + "args": cpt_args, } if eval_dataset is not None: trainer_kwargs["eval_dataset"] = eval_dataset diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py index a8557d85339..78882c94f56 100644 --- a/tests/utils/test_packing.py +++ b/tests/utils/test_packing.py @@ -14,6 +14,7 @@ # along with this program. If not, see . from unsloth import FastLanguageModel +import unsloth.trainer as trainer_module from unsloth.utils import attention_dispatch as attention_dispatch_utils from unsloth.utils.packing import ( configure_padding_free, @@ -160,6 +161,107 @@ def test_configure_padding_free(): assert config.remove_unused_columns is False +def _patch_fake_sft_trainer(): + class FakeSFTTrainer: + def __init__(self, *args, **kwargs): + self.model = args[0] if len(args) >= 1 else kwargs["model"] + self.args = args[1] if len(args) >= 2 else kwargs["args"] + self.data_collator = args[2] if len(args) >= 3 else kwargs.get("data_collator") + + trainer_module._patch_sft_trainer_auto_packing( + SimpleNamespace(SFTTrainer = FakeSFTTrainer) + ) + return FakeSFTTrainer + + +def _vlm_model(): + return SimpleNamespace( + config = SimpleNamespace( + architectures = ["Gemma4ForConditionalGeneration"], + model_type = "gemma4", + vision_config = SimpleNamespace(), + ), + max_seq_length = 16, + ) + + +def test_vlm_text_dataset_allows_explicit_packing(): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + + trainer = fake_trainer( + model = _vlm_model(), + args = config, + processing_class = object(), + train_dataset = Dataset.from_dict({"text": ["text-only CPT sample"]}), + ) + + assert config.packing is True + assert config.padding_free is True + assert trainer.model._unsloth_allow_packed_overlength is True + + +def test_vlm_vision_dataset_still_disables_packing(): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + + fake_trainer( + _vlm_model(), + config, + None, + Dataset.from_dict({"images": [None], "text": ["multimodal sample"]}), + object(), + ) + + assert config.packing is False + assert config.padding_free is False + + +def test_wrapped_packing_preserves_overlength_tokens(monkeypatch): + class CharacterTokenizer: + bos_token = None + eos_token = None + chat_template = None + + def __call__(self, texts, **kwargs): + if isinstance(texts, str): + texts = [texts] + input_ids = [[ord(char) for char in text] for text in texts] + if kwargs.get("truncation") and kwargs.get("max_length") is not None: + input_ids = [ids[: kwargs["max_length"]] for ids in input_ids] + return {"input_ids": input_ids} + + args = SimpleNamespace( + dataset_num_proc = 1, + dataset_text_field = "text", + max_length = 4, + packing_strategy = "wrapped", + ) + trainer = SimpleNamespace(model = None) + dataset = Dataset.from_dict({"text": ["abcdefghi"]}) + prepare_globals = SFTTrainer._prepare_dataset.__globals__ + pack_dataset = prepare_globals["pack_dataset"] + + def legacy_pack_dataset(dataset, seq_length, map_kwargs = None): + return pack_dataset(dataset, seq_length, "wrapped", map_kwargs) + + monkeypatch.setitem(prepare_globals, "pack_dataset", legacy_pack_dataset) + + packed = SFTTrainer._prepare_dataset( + trainer, + dataset, + CharacterTokenizer(), + args, + True, + None, + "train", + ) + + packed_ids = packed["input_ids"] + assert sum(len(input_ids) for input_ids in packed_ids) == 9 + assert all(len(input_ids) <= args.max_length for input_ids in packed_ids) + + class _DummyChild(torch.nn.Module): def __init__(self): super().__init__() diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index ffb845b04f4..0bba37e6fa7 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -458,6 +458,31 @@ def sft_trainer_prepare_dataset(function_name, function): if matched: # Use fast version! function = inspect.getsource(fast_sft_prepare_dataset) + function = function.replace( + "truncation = do_truncation,", + 'truncation = do_truncation and getattr(args, "packing_strategy", "bfd") != "wrapped",', + ) + function = function.replace( + "if do_truncation and max_seq_length > 0:", + "if do_truncation and not packing and max_seq_length > 0:", + ) + function = function.replace( + '''dataset = pack_dataset( + dataset.select_columns(used_column_names), + max_seq_length, + getattr(args, "packing_strategy", "bfd"), + map_kwargs, + )''', + '''import inspect as _inspect + _pack_kwargs = {"map_kwargs": map_kwargs} + if "strategy" in _inspect.signature(pack_dataset).parameters: + _pack_kwargs["strategy"] = getattr(args, "packing_strategy", "bfd") + dataset = pack_dataset( + dataset.select_columns(used_column_names), + max_seq_length, + **_pack_kwargs, + )''', + ) function = function.split("\n") function = "\n".join(" " * 4 + x for x in function) function = function.replace("def sft_prepare_dataset", "def _prepare_dataset") diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 83cb1758f01..3692113ba86 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -137,6 +137,13 @@ def _should_skip_auto_packing_error(exc: Exception) -> bool: return any(msg in message for msg in _AUTO_PACK_SKIP_MESSAGES) +def _is_vision_dataset(dataset) -> bool: + if dataset is None: + return False + column_names = getattr(dataset, "column_names", None) or () + return "image" in column_names or "images" in column_names + + # Unsloth gradient accumulation fix: from transformers import __version__ as transformers_version, ProcessorMixin @@ -500,27 +507,22 @@ def new_init(self, *args, **kwargs): model = kwargs.get("model") is_unsupported_model = False - is_vlm = False if model is not None: model_config = getattr(model, "config", None) if model_config is not None: model_types = get_transformers_model_type(model_config) is_unsupported_model = any(x in PADDING_FREE_BLOCKLIST for x in model_types) - architectures = getattr(model_config, "architectures", None) - if architectures is None: - architectures = [] - is_vlm = any(x.endswith("ForConditionalGeneration") for x in architectures) - is_vlm = is_vlm or hasattr(model_config, "vision_config") - processing_class = kwargs.get("processing_class") or kwargs.get("tokenizer") data_collator = kwargs.get("data_collator") + train_dataset = args[3] if len(args) >= 4 else kwargs.get("train_dataset") + is_vision_dataset = _is_vision_dataset(train_dataset) # Disable padding-free for VLMs / custom collators / blocklisted models blocked = ( (data_collator is not None) or isinstance(processing_class, ProcessorMixin) - or is_vlm + or is_vision_dataset or is_unsupported_model or ( os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1" @@ -537,8 +539,8 @@ def new_init(self, *args, **kwargs): reason = "custom data collator" if data_collator is None and isinstance(processing_class, ProcessorMixin): reason = "processor-based model" - elif is_vlm: - reason = "vision-language model" + elif is_vision_dataset: + reason = "vision dataset" elif is_unsupported_model: reason = f"unsupported model type(s): {', '.join(model_types)}" message = f"Unsloth: Sample packing skipped ({reason} detected)." From c608bd06acffb59f55d01473a008824db8a39359 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:14:11 +0000 Subject: [PATCH 02/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/utils/test_packing.py | 10 ++++++---- unsloth/models/rl_replacements.py | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py index 78882c94f56..30c154e7054 100644 --- a/tests/utils/test_packing.py +++ b/tests/utils/test_packing.py @@ -168,9 +168,7 @@ def __init__(self, *args, **kwargs): self.args = args[1] if len(args) >= 2 else kwargs["args"] self.data_collator = args[2] if len(args) >= 3 else kwargs.get("data_collator") - trainer_module._patch_sft_trainer_auto_packing( - SimpleNamespace(SFTTrainer = FakeSFTTrainer) - ) + trainer_module._patch_sft_trainer_auto_packing(SimpleNamespace(SFTTrainer = FakeSFTTrainer)) return FakeSFTTrainer @@ -242,7 +240,11 @@ def __call__(self, texts, **kwargs): prepare_globals = SFTTrainer._prepare_dataset.__globals__ pack_dataset = prepare_globals["pack_dataset"] - def legacy_pack_dataset(dataset, seq_length, map_kwargs = None): + def legacy_pack_dataset( + dataset, + seq_length, + map_kwargs = None, + ): return pack_dataset(dataset, seq_length, "wrapped", map_kwargs) monkeypatch.setitem(prepare_globals, "pack_dataset", legacy_pack_dataset) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 0bba37e6fa7..62d94be3db2 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -467,13 +467,13 @@ def sft_trainer_prepare_dataset(function_name, function): "if do_truncation and not packing and max_seq_length > 0:", ) function = function.replace( - '''dataset = pack_dataset( + """dataset = pack_dataset( dataset.select_columns(used_column_names), max_seq_length, getattr(args, "packing_strategy", "bfd"), map_kwargs, - )''', - '''import inspect as _inspect + )""", + """import inspect as _inspect _pack_kwargs = {"map_kwargs": map_kwargs} if "strategy" in _inspect.signature(pack_dataset).parameters: _pack_kwargs["strategy"] = getattr(args, "packing_strategy", "bfd") @@ -481,7 +481,7 @@ def sft_trainer_prepare_dataset(function_name, function): dataset.select_columns(used_column_names), max_seq_length, **_pack_kwargs, - )''', + )""", ) function = function.split("\n") function = "\n".join(" " * 4 + x for x in function) From b6817905ec0509fbfe18f11a0f16cfd4fc8220b1 Mon Sep 17 00:00:00 2001 From: alkinun Date: Fri, 17 Jul 2026 20:30:09 +0300 Subject: [PATCH 03/15] Handle streaming vision datasets in packing --- tests/utils/test_packing.py | 22 +++++++++++++++++++++- unsloth/trainer.py | 10 ++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py index 30c154e7054..12b687f2e3d 100644 --- a/tests/utils/test_packing.py +++ b/tests/utils/test_packing.py @@ -30,7 +30,7 @@ import pytest import torch -from datasets import Dataset +from datasets import Dataset, IterableDataset from trl import SFTConfig, SFTTrainer from trl.trainer.sft_trainer import DataCollatorForLanguageModeling @@ -215,6 +215,26 @@ def test_vlm_vision_dataset_still_disables_packing(): assert config.padding_free is False +def test_vlm_streaming_vision_dataset_without_metadata_disables_packing(): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + dataset = IterableDataset.from_generator( + lambda: iter([{"images": [None], "text": "multimodal sample"}]) + ) + assert dataset.column_names is None + + fake_trainer( + model = _vlm_model(), + args = config, + processing_class = object(), + train_dataset = dataset, + ) + + assert config.packing is False + assert config.padding_free is False + assert next(iter(dataset))["text"] == "multimodal sample" + + def test_wrapped_packing_preserves_overlength_tokens(monkeypatch): class CharacterTokenizer: bos_token = None diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 3692113ba86..465419531a8 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -140,8 +140,14 @@ def _should_skip_auto_packing_error(exc: Exception) -> bool: def _is_vision_dataset(dataset) -> bool: if dataset is None: return False - column_names = getattr(dataset, "column_names", None) or () - return "image" in column_names or "images" in column_names + column_names = getattr(dataset, "column_names", None) + if column_names is not None: + return "image" in column_names or "images" in column_names + try: + sample = next(iter(dataset)) + except (StopIteration, TypeError): + return False + return isinstance(sample, dict) and ("image" in sample or "images" in sample) # Unsloth gradient accumulation fix: From 0d8503eae72403fb0ccb5b38e1d97b5d4aa6b482 Mon Sep 17 00:00:00 2001 From: alkinun Date: Fri, 17 Jul 2026 21:12:20 +0300 Subject: [PATCH 04/15] Harden multimodal packing detection --- tests/utils/test_packing.py | 48 +++++++++++++++++++++++++++++++++++++ unsloth/trainer.py | 43 +++++++++++++++++++++++++++------ 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py index 12b687f2e3d..10da2932ce8 100644 --- a/tests/utils/test_packing.py +++ b/tests/utils/test_packing.py @@ -215,6 +215,25 @@ def test_vlm_vision_dataset_still_disables_packing(): assert config.padding_free is False +@pytest.mark.parametrize( + "vision_column", + ("pixel_values", "pixel_attention_mask", "image_grid_thw"), +) +def test_vlm_preprocessed_vision_dataset_disables_packing(vision_column): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + + fake_trainer( + model = _vlm_model(), + args = config, + processing_class = object(), + train_dataset = Dataset.from_dict({"input_ids": [[1]], vision_column: [None]}), + ) + + assert config.packing is False + assert config.padding_free is False + + def test_vlm_streaming_vision_dataset_without_metadata_disables_packing(): fake_trainer = _patch_fake_sft_trainer() config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) @@ -235,6 +254,35 @@ def test_vlm_streaming_vision_dataset_without_metadata_disables_packing(): assert next(iter(dataset))["text"] == "multimodal sample" +@pytest.mark.parametrize("data_collator", (None, object())) +def test_stateful_stream_is_not_consumed_during_detection(data_collator): + class StatefulDataset: + def __init__(self): + self.rows = iter([{"text": "first"}, {"text": "second"}]) + + def __iter__(self): + return self + + def __next__(self): + return next(self.rows) + + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + dataset = StatefulDataset() + + fake_trainer( + model = _vlm_model(), + args = config, + processing_class = object(), + data_collator = data_collator, + train_dataset = dataset, + ) + + assert config.packing is False + assert config.padding_free is False + assert next(dataset)["text"] == "first" + + def test_wrapped_packing_preserves_overlength_tokens(monkeypatch): class CharacterTokenizer: bos_token = None diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 465419531a8..298c19ba6ed 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -137,17 +137,43 @@ def _should_skip_auto_packing_error(exc: Exception) -> bool: return any(msg in message for msg in _AUTO_PACK_SKIP_MESSAGES) +_VISION_DATASET_KEYS = frozenset( + { + "image", + "images", + "image_grid_thw", + "image_position_ids", + "image_sizes", + "mm_token_type_ids", + "pixel_attention_mask", + "pixel_position_ids", + "pixel_values", + "pixel_values_videos", + "video", + "videos", + "video_grid_thw", + } +) + + def _is_vision_dataset(dataset) -> bool: if dataset is None: return False column_names = getattr(dataset, "column_names", None) if column_names is not None: - return "image" in column_names or "images" in column_names + return not _VISION_DATASET_KEYS.isdisjoint(column_names) try: - sample = next(iter(dataset)) - except (StopIteration, TypeError): + iterator = iter(dataset) + except TypeError: return False - return isinstance(sample, dict) and ("image" in sample or "images" in sample) + if iterator is dataset: + # Modality is unknown, but probing a one-shot stream would drop its first sample. + return True + try: + sample = next(iterator) + except StopIteration: + return False + return isinstance(sample, dict) and not _VISION_DATASET_KEYS.isdisjoint(sample) # Unsloth gradient accumulation fix: @@ -522,12 +548,15 @@ def new_init(self, *args, **kwargs): processing_class = kwargs.get("processing_class") or kwargs.get("tokenizer") data_collator = kwargs.get("data_collator") train_dataset = args[3] if len(args) >= 4 else kwargs.get("train_dataset") - is_vision_dataset = _is_vision_dataset(train_dataset) + is_processor = isinstance(processing_class, ProcessorMixin) + is_vision_dataset = ( + data_collator is None and not is_processor and _is_vision_dataset(train_dataset) + ) # Disable padding-free for VLMs / custom collators / blocklisted models blocked = ( (data_collator is not None) - or isinstance(processing_class, ProcessorMixin) + or is_processor or is_vision_dataset or is_unsupported_model or ( @@ -543,7 +572,7 @@ def new_init(self, *args, **kwargs): if blocked and requested_pack: reason = "custom data collator" - if data_collator is None and isinstance(processing_class, ProcessorMixin): + if data_collator is None and is_processor: reason = "processor-based model" elif is_vision_dataset: reason = "vision dataset" From cd9a54ee8732b8bf96f4bc921ab04ecb3d0e8fd6 Mon Sep 17 00:00:00 2001 From: alkinun Date: Fri, 17 Jul 2026 21:34:08 +0300 Subject: [PATCH 05/15] Preserve safe packing boundaries --- tests/utils/test_packing.py | 45 +++++++++++++++++++++++++++---- unsloth/models/rl_replacements.py | 2 +- unsloth/trainer.py | 14 ++-------- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py index 10da2932ce8..03728d4a21b 100644 --- a/tests/utils/test_packing.py +++ b/tests/utils/test_packing.py @@ -261,10 +261,7 @@ def __init__(self): self.rows = iter([{"text": "first"}, {"text": "second"}]) def __iter__(self): - return self - - def __next__(self): - return next(self.rows) + return (row for row in self.rows) fake_trainer = _patch_fake_sft_trainer() config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) @@ -280,7 +277,45 @@ def __next__(self): assert config.packing is False assert config.padding_free is False - assert next(dataset)["text"] == "first" + assert next(iter(dataset))["text"] == "first" + + +def test_bfd_packing_truncates_before_packing(monkeypatch): + class CharacterTokenizer: + bos_token = None + eos_token = None + chat_template = None + + def __call__(self, texts, **kwargs): + if isinstance(texts, list): + return {"input_ids": [[ord(char) for char in text] for text in texts]} + return {"input_ids": [ord(char) for char in texts]} + + args = SimpleNamespace( + dataset_num_proc = 1, + dataset_text_field = "text", + max_length = 4, + packing_strategy = "bfd", + ) + trainer = SimpleNamespace(model = None) + dataset = Dataset.from_dict({"prompt": ["abc"], "completion": ["defghij"]}) + prepare_globals = SFTTrainer._prepare_dataset.__globals__ + + def passthrough_pack_dataset(dataset, seq_length, strategy, map_kwargs): + return dataset + + monkeypatch.setitem(prepare_globals, "pack_dataset", passthrough_pack_dataset) + packed = SFTTrainer._prepare_dataset( + trainer, + dataset, + CharacterTokenizer(), + args, + True, + None, + "train", + ) + + assert len(packed["input_ids"][0]) == args.max_length def test_wrapped_packing_preserves_overlength_tokens(monkeypatch): diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 62d94be3db2..8f53c3b6e36 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -464,7 +464,7 @@ def sft_trainer_prepare_dataset(function_name, function): ) function = function.replace( "if do_truncation and max_seq_length > 0:", - "if do_truncation and not packing and max_seq_length > 0:", + 'if do_truncation and getattr(args, "packing_strategy", "bfd") != "wrapped" and max_seq_length > 0:', ) function = function.replace( """dataset = pack_dataset( diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 298c19ba6ed..62a283d1088 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -162,18 +162,8 @@ def _is_vision_dataset(dataset) -> bool: column_names = getattr(dataset, "column_names", None) if column_names is not None: return not _VISION_DATASET_KEYS.isdisjoint(column_names) - try: - iterator = iter(dataset) - except TypeError: - return False - if iterator is dataset: - # Modality is unknown, but probing a one-shot stream would drop its first sample. - return True - try: - sample = next(iterator) - except StopIteration: - return False - return isinstance(sample, dict) and not _VISION_DATASET_KEYS.isdisjoint(sample) + # Unknown-schema streams cannot be safely probed without potentially dropping a sample. + return True # Unsloth gradient accumulation fix: From b1bfe20abdaa368dbc703251163f9090e2d133b0 Mon Sep 17 00:00:00 2001 From: alkinun Date: Fri, 17 Jul 2026 21:56:12 +0300 Subject: [PATCH 06/15] Scope stream packing checks to VLMs --- tests/utils/test_packing.py | 51 +++++++++++++++++++++++++++++++++++++ unsloth/trainer.py | 27 +++++++++++++++----- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py index 03728d4a21b..804edce0891 100644 --- a/tests/utils/test_packing.py +++ b/tests/utils/test_packing.py @@ -183,6 +183,16 @@ def _vlm_model(): ) +def _text_model(): + return SimpleNamespace( + config = SimpleNamespace( + architectures = ["LlamaForCausalLM"], + model_type = "llama", + ), + max_seq_length = 16, + ) + + def test_vlm_text_dataset_allows_explicit_packing(): fake_trainer = _patch_fake_sft_trainer() config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) @@ -199,6 +209,21 @@ def test_vlm_text_dataset_allows_explicit_packing(): assert trainer.model._unsloth_allow_packed_overlength is True +def test_vlm_without_processing_class_still_disables_packing(): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + + fake_trainer( + _vlm_model(), + config, + None, + Dataset.from_dict({"text": ["text-only sample"]}), + ) + + assert config.packing is False + assert config.padding_free is False + + def test_vlm_vision_dataset_still_disables_packing(): fake_trainer = _patch_fake_sft_trainer() config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) @@ -208,6 +233,7 @@ def test_vlm_vision_dataset_still_disables_packing(): config, None, Dataset.from_dict({"images": [None], "text": ["multimodal sample"]}), + None, object(), ) @@ -280,6 +306,31 @@ def __iter__(self): assert next(iter(dataset))["text"] == "first" +def test_text_model_stream_without_metadata_keeps_packing(): + class StatefulDataset: + def __init__(self): + self.rows = iter([{"text": "first"}, {"text": "second"}]) + + def __iter__(self): + return (row for row in self.rows) + + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + dataset = StatefulDataset() + + trainer = fake_trainer( + model = _text_model(), + args = config, + processing_class = object(), + train_dataset = dataset, + ) + + assert config.packing is True + assert config.padding_free is True + assert trainer.model._unsloth_allow_packed_overlength is True + assert next(iter(dataset))["text"] == "first" + + def test_bfd_packing_truncates_before_packing(monkeypatch): class CharacterTokenizer: bos_token = None diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 62a283d1088..bfaf0d9026b 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -156,14 +156,14 @@ def _should_skip_auto_packing_error(exc: Exception) -> bool: ) -def _is_vision_dataset(dataset) -> bool: +def _is_vision_dataset(dataset, *, unknown_is_vision = False) -> bool: if dataset is None: return False column_names = getattr(dataset, "column_names", None) if column_names is not None: return not _VISION_DATASET_KEYS.isdisjoint(column_names) # Unknown-schema streams cannot be safely probed without potentially dropping a sample. - return True + return unknown_is_vision # Unsloth gradient accumulation fix: @@ -527,26 +527,39 @@ def new_init(self, *args, **kwargs): else: config_arg = kwargs.get("args") - model = kwargs.get("model") + model = args[0] if len(args) >= 1 else kwargs.get("model") + is_vlm = False is_unsupported_model = False if model is not None: model_config = getattr(model, "config", None) if model_config is not None: model_types = get_transformers_model_type(model_config) is_unsupported_model = any(x in PADDING_FREE_BLOCKLIST for x in model_types) + architectures = getattr(model_config, "architectures", None) or () + is_vlm = hasattr(model_config, "vision_config") or any( + x.endswith("ForConditionalGeneration") for x in architectures + ) - processing_class = kwargs.get("processing_class") or kwargs.get("tokenizer") - data_collator = kwargs.get("data_collator") + processing_class = ( + args[5] + if len(args) >= 6 + else kwargs.get("processing_class") or kwargs.get("tokenizer") + ) + data_collator = args[2] if len(args) >= 3 else kwargs.get("data_collator") train_dataset = args[3] if len(args) >= 4 else kwargs.get("train_dataset") is_processor = isinstance(processing_class, ProcessorMixin) + is_auto_processor_vlm = is_vlm and processing_class is None is_vision_dataset = ( - data_collator is None and not is_processor and _is_vision_dataset(train_dataset) + data_collator is None + and not is_processor + and _is_vision_dataset(train_dataset, unknown_is_vision = is_vlm) ) # Disable padding-free for VLMs / custom collators / blocklisted models blocked = ( (data_collator is not None) or is_processor + or is_auto_processor_vlm or is_vision_dataset or is_unsupported_model or ( @@ -564,6 +577,8 @@ def new_init(self, *args, **kwargs): reason = "custom data collator" if data_collator is None and is_processor: reason = "processor-based model" + elif is_auto_processor_vlm: + reason = "vision-language model with auto processor" elif is_vision_dataset: reason = "vision dataset" elif is_unsupported_model: From bb06e4c8f8f78c64d3dc33d2bd0e560dac77fd90 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:56:56 +0000 Subject: [PATCH 07/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/trainer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/unsloth/trainer.py b/unsloth/trainer.py index bfaf0d9026b..cea787216a0 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -541,9 +541,7 @@ def new_init(self, *args, **kwargs): ) processing_class = ( - args[5] - if len(args) >= 6 - else kwargs.get("processing_class") or kwargs.get("tokenizer") + args[5] if len(args) >= 6 else kwargs.get("processing_class") or kwargs.get("tokenizer") ) data_collator = args[2] if len(args) >= 3 else kwargs.get("data_collator") train_dataset = args[3] if len(args) >= 4 else kwargs.get("train_dataset") From 3af93545ff7b3b4dc3727be99389c376afaaddf5 Mon Sep 17 00:00:00 2001 From: alkinun Date: Fri, 17 Jul 2026 22:26:59 +0300 Subject: [PATCH 08/15] Narrow VLM packing detection --- tests/utils/test_packing.py | 29 ++++++++++++++++++++++++++++ unsloth/trainer.py | 38 +++++++++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py index 804edce0891..9b288dac31e 100644 --- a/tests/utils/test_packing.py +++ b/tests/utils/test_packing.py @@ -224,6 +224,35 @@ def test_vlm_without_processing_class_still_disables_packing(): assert config.padding_free is False +@pytest.mark.parametrize( + ("model_type", "architecture"), + ( + ("t5", "T5ForConditionalGeneration"), + ("bart", "BartForConditionalGeneration"), + ("whisper", "WhisperForConditionalGeneration"), + ("csm", "CsmForConditionalGeneration"), + ), +) +def test_nonvision_conditional_generation_keeps_packing(model_type, architecture): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + model = SimpleNamespace( + config = SimpleNamespace(model_type = model_type, architectures = [architecture]), + max_seq_length = 16, + ) + + trainer = fake_trainer( + model, + config, + None, + Dataset.from_dict({"text": ["text-only sample"]}), + ) + + assert config.packing is True + assert config.padding_free is True + assert trainer.model._unsloth_allow_packed_overlength is True + + def test_vlm_vision_dataset_still_disables_packing(): fake_trainer = _patch_fake_sft_trainer() config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) diff --git a/unsloth/trainer.py b/unsloth/trainer.py index cea787216a0..63ef7e7b371 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -156,6 +156,39 @@ def _should_skip_auto_packing_error(exc: Exception) -> bool: ) +def _is_vlm_config(config, model_types = ()) -> bool: + if any( + hasattr(config, attr) + for attr in ("vision_config", "img_processor", "image_token_index", "projector_config") + ): + return True + + architectures = getattr(config, "architectures", None) or () + try: + from transformers.models.auto import modeling_auto + + mappings = ( + getattr(modeling_auto, "MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES", {}) or {}, + getattr(modeling_auto, "MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES", {}) or {}, + ) + registry_types = set().union(*(mapping.keys() for mapping in mappings)) + registry_classes = set().union(*(mapping.values() for mapping in mappings)) + config_types = set(model_types or ()) + model_type = getattr(config, "model_type", None) + if model_type is not None: + config_types.add(model_type) + if not config_types.isdisjoint(registry_types) or any( + architecture in registry_classes for architecture in architectures + ): + return True + except Exception: + pass + return any( + isinstance(architecture, str) and architecture.endswith("ForVisionText2Text") + for architecture in architectures + ) + + def _is_vision_dataset(dataset, *, unknown_is_vision = False) -> bool: if dataset is None: return False @@ -535,10 +568,7 @@ def new_init(self, *args, **kwargs): if model_config is not None: model_types = get_transformers_model_type(model_config) is_unsupported_model = any(x in PADDING_FREE_BLOCKLIST for x in model_types) - architectures = getattr(model_config, "architectures", None) or () - is_vlm = hasattr(model_config, "vision_config") or any( - x.endswith("ForConditionalGeneration") for x in architectures - ) + is_vlm = _is_vlm_config(model_config, model_types) processing_class = ( args[5] if len(args) >= 6 else kwargs.get("processing_class") or kwargs.get("tokenizer") From a88e67e77cf15e4f16e4dae41ccab46c248cb441 Mon Sep 17 00:00:00 2001 From: alkinun Date: Fri, 17 Jul 2026 22:45:50 +0300 Subject: [PATCH 09/15] Align packing mode and eval safety --- tests/utils/test_packing.py | 93 ++++++++++++++++++++++--------- unsloth/models/rl_replacements.py | 22 ++++++-- unsloth/trainer.py | 15 ++++- 3 files changed, 98 insertions(+), 32 deletions(-) diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py index 9b288dac31e..a79c638b787 100644 --- a/tests/utils/test_packing.py +++ b/tests/utils/test_packing.py @@ -193,6 +193,21 @@ def _text_model(): ) +class _CharacterTokenizer: + bos_token = None + eos_token = None + chat_template = None + + def __call__(self, texts, **kwargs): + is_batched = isinstance(texts, list) + if not is_batched: + texts = [texts] + input_ids = [[ord(char) for char in text] for text in texts] + if kwargs.get("truncation") and kwargs.get("max_length") is not None: + input_ids = [ids[: kwargs["max_length"]] for ids in input_ids] + return {"input_ids": input_ids if is_batched else input_ids[0]} + + def test_vlm_text_dataset_allows_explicit_packing(): fake_trainer = _patch_fake_sft_trainer() config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) @@ -289,6 +304,26 @@ def test_vlm_preprocessed_vision_dataset_disables_packing(vision_column): assert config.padding_free is False +@pytest.mark.parametrize("dict_eval", (False, True)) +def test_vlm_vision_eval_dataset_disables_packing(dict_eval): + fake_trainer = _patch_fake_sft_trainer() + config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) + eval_dataset = Dataset.from_dict({"input_ids": [[1]], "pixel_values": [None]}) + if dict_eval: + eval_dataset = {"vision": eval_dataset} + + fake_trainer( + model = _vlm_model(), + args = config, + processing_class = object(), + train_dataset = Dataset.from_dict({"text": ["text-only training sample"]}), + eval_dataset = eval_dataset, + ) + + assert config.packing is False + assert config.padding_free is False + + def test_vlm_streaming_vision_dataset_without_metadata_disables_packing(): fake_trainer = _patch_fake_sft_trainer() config = SimpleNamespace(packing = True, padding_free = None, remove_unused_columns = True) @@ -361,16 +396,6 @@ def __iter__(self): def test_bfd_packing_truncates_before_packing(monkeypatch): - class CharacterTokenizer: - bos_token = None - eos_token = None - chat_template = None - - def __call__(self, texts, **kwargs): - if isinstance(texts, list): - return {"input_ids": [[ord(char) for char in text] for text in texts]} - return {"input_ids": [ord(char) for char in texts]} - args = SimpleNamespace( dataset_num_proc = 1, dataset_text_field = "text", @@ -388,7 +413,7 @@ def passthrough_pack_dataset(dataset, seq_length, strategy, map_kwargs): packed = SFTTrainer._prepare_dataset( trainer, dataset, - CharacterTokenizer(), + _CharacterTokenizer(), args, True, None, @@ -398,20 +423,7 @@ def passthrough_pack_dataset(dataset, seq_length, strategy, map_kwargs): assert len(packed["input_ids"][0]) == args.max_length -def test_wrapped_packing_preserves_overlength_tokens(monkeypatch): - class CharacterTokenizer: - bos_token = None - eos_token = None - chat_template = None - - def __call__(self, texts, **kwargs): - if isinstance(texts, str): - texts = [texts] - input_ids = [[ord(char) for char in text] for text in texts] - if kwargs.get("truncation") and kwargs.get("max_length") is not None: - input_ids = [ids[: kwargs["max_length"]] for ids in input_ids] - return {"input_ids": input_ids} - +def test_wrapped_strategy_without_packing_still_truncates(): args = SimpleNamespace( dataset_num_proc = 1, dataset_text_field = "text", @@ -420,6 +432,32 @@ def __call__(self, texts, **kwargs): ) trainer = SimpleNamespace(model = None) dataset = Dataset.from_dict({"text": ["abcdefghi"]}) + + prepared = SFTTrainer._prepare_dataset( + trainer, + dataset, + _CharacterTokenizer(), + args, + False, + None, + "train", + ) + + assert len(prepared["input_ids"][0]) == args.max_length + + +@pytest.mark.parametrize("legacy_api", (False, True)) +def test_wrapped_packing_preserves_overlength_tokens(monkeypatch, legacy_api): + args_kwargs = { + "dataset_num_proc": 1, + "dataset_text_field": "text", + "max_length": 4, + } + if not legacy_api: + args_kwargs["packing_strategy"] = "wrapped" + args = SimpleNamespace(**args_kwargs) + trainer = SimpleNamespace(model = None) + dataset = Dataset.from_dict({"text": ["abcdefghi"]}) prepare_globals = SFTTrainer._prepare_dataset.__globals__ pack_dataset = prepare_globals["pack_dataset"] @@ -430,12 +468,13 @@ def legacy_pack_dataset( ): return pack_dataset(dataset, seq_length, "wrapped", map_kwargs) - monkeypatch.setitem(prepare_globals, "pack_dataset", legacy_pack_dataset) + if legacy_api: + monkeypatch.setitem(prepare_globals, "pack_dataset", legacy_pack_dataset) packed = SFTTrainer._prepare_dataset( trainer, dataset, - CharacterTokenizer(), + _CharacterTokenizer(), args, True, None, diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 8f53c3b6e36..c45998bb415 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -458,13 +458,28 @@ def sft_trainer_prepare_dataset(function_name, function): if matched: # Use fast version! function = inspect.getsource(fast_sft_prepare_dataset) + function = function.replace( + " # All Unsloth Zoo code licensed under LGPLv3\n", + ''' # All Unsloth Zoo code licensed under LGPLv3 + import inspect as _inspect + try: + _unsloth_pack_has_strategy = "strategy" in _inspect.signature(pack_dataset).parameters + except Exception: + _unsloth_pack_has_strategy = True + _unsloth_wrapped_packing = packing and ( + getattr(args, "packing_strategy", None) == "wrapped" + or not _unsloth_pack_has_strategy + ) +''', + 1, + ) function = function.replace( "truncation = do_truncation,", - 'truncation = do_truncation and getattr(args, "packing_strategy", "bfd") != "wrapped",', + "truncation = do_truncation and not _unsloth_wrapped_packing,", ) function = function.replace( "if do_truncation and max_seq_length > 0:", - 'if do_truncation and getattr(args, "packing_strategy", "bfd") != "wrapped" and max_seq_length > 0:', + "if do_truncation and not _unsloth_wrapped_packing and max_seq_length > 0:", ) function = function.replace( """dataset = pack_dataset( @@ -473,8 +488,7 @@ def sft_trainer_prepare_dataset(function_name, function): getattr(args, "packing_strategy", "bfd"), map_kwargs, )""", - """import inspect as _inspect - _pack_kwargs = {"map_kwargs": map_kwargs} + """_pack_kwargs = {"map_kwargs": map_kwargs} if "strategy" in _inspect.signature(pack_dataset).parameters: _pack_kwargs["strategy"] = getattr(args, "packing_strategy", "bfd") dataset = pack_dataset( diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 63ef7e7b371..fec4eab226d 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -199,6 +199,15 @@ def _is_vision_dataset(dataset, *, unknown_is_vision = False) -> bool: return unknown_is_vision +def _is_vision_eval_dataset(dataset, *, unknown_is_vision = False) -> bool: + if isinstance(dataset, dict): + return any( + _is_vision_dataset(split, unknown_is_vision = unknown_is_vision) + for split in dataset.values() + ) + return _is_vision_dataset(dataset, unknown_is_vision = unknown_is_vision) + + # Unsloth gradient accumulation fix: from transformers import __version__ as transformers_version, ProcessorMixin @@ -575,12 +584,16 @@ def new_init(self, *args, **kwargs): ) data_collator = args[2] if len(args) >= 3 else kwargs.get("data_collator") train_dataset = args[3] if len(args) >= 4 else kwargs.get("train_dataset") + eval_dataset = args[4] if len(args) >= 5 else kwargs.get("eval_dataset") is_processor = isinstance(processing_class, ProcessorMixin) is_auto_processor_vlm = is_vlm and processing_class is None is_vision_dataset = ( data_collator is None and not is_processor - and _is_vision_dataset(train_dataset, unknown_is_vision = is_vlm) + and ( + _is_vision_dataset(train_dataset, unknown_is_vision = is_vlm) + or _is_vision_eval_dataset(eval_dataset, unknown_is_vision = is_vlm) + ) ) # Disable padding-free for VLMs / custom collators / blocklisted models From cc49802ce4af077ba95d7eba4f7776d65fd5c935 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:46:30 +0000 Subject: [PATCH 10/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl_replacements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index c45998bb415..d9792a8db8e 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -460,7 +460,7 @@ def sft_trainer_prepare_dataset(function_name, function): function = inspect.getsource(fast_sft_prepare_dataset) function = function.replace( " # All Unsloth Zoo code licensed under LGPLv3\n", - ''' # All Unsloth Zoo code licensed under LGPLv3 + """ # All Unsloth Zoo code licensed under LGPLv3 import inspect as _inspect try: _unsloth_pack_has_strategy = "strategy" in _inspect.signature(pack_dataset).parameters @@ -470,7 +470,7 @@ def sft_trainer_prepare_dataset(function_name, function): getattr(args, "packing_strategy", None) == "wrapped" or not _unsloth_pack_has_strategy ) -''', +""", 1, ) function = function.replace( From 4ee5df378ca8461703fec6963abcefa3e81f3cda Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 19 Jul 2026 11:52:10 +0000 Subject: [PATCH 11/15] Add qwen3_5/qwen3_next to PADDING_FREE_BLOCKLIST to avoid packed-sequence contamination --- unsloth/trainer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unsloth/trainer.py b/unsloth/trainer.py index fec4eab226d..dfacfd94b73 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -99,6 +99,8 @@ def __call__(self, examples): PADDING_FREE_BLOCKLIST = { "gemma2", # - gemma2: Uses slow_attention_softcapping which has torch.compile issues "gpt_oss", # - gpt_oss: Uses Flex Attention which doesn't handle padding_free correctly + "qwen3_5", # - qwen3_5 / qwen3_next: hybrid linear attention + conv1d leak across packed sequences + "qwen3_next", } From 8a887bc0a9f310c6ad10f6cd359e3594995da2b9 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 19 Jul 2026 12:39:15 +0000 Subject: [PATCH 12/15] Detect hybrid linear-attention models structurally instead of by name for packing guard --- unsloth/trainer.py | 61 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/unsloth/trainer.py b/unsloth/trainer.py index dfacfd94b73..45baf26ce0d 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -99,9 +99,11 @@ def __call__(self, examples): PADDING_FREE_BLOCKLIST = { "gemma2", # - gemma2: Uses slow_attention_softcapping which has torch.compile issues "gpt_oss", # - gpt_oss: Uses Flex Attention which doesn't handle padding_free correctly - "qwen3_5", # - qwen3_5 / qwen3_next: hybrid linear attention + conv1d leak across packed sequences - "qwen3_next", } +# Hybrid linear-attention / state-space models (Qwen3.5, Qwen3-Next, ...) carry a +# recurrent gated-delta state plus a causal conv1d. Sample packing / padding-free +# flattens the batch, so those ops leak state across sequence boundaries. Detected +# structurally by _is_hybrid_linear_attention_model rather than by model name. def _should_pack(config) -> bool: @@ -210,6 +212,56 @@ def _is_vision_eval_dataset(dataset, *, unknown_is_vision = False) -> bool: return _is_vision_dataset(dataset, unknown_is_vision = unknown_is_vision) +_HYBRID_CONFIG_MARKERS = ( + "linear_conv_kernel_dim", + "linear_key_head_dim", + "linear_value_head_dim", + "full_attention_interval", +) + + +def _is_hybrid_linear_attention_model(model) -> bool: + """Detect models mixing linear-attention / state-space mixers (gated-delta, + Mamba-style) with a causal conv1d, e.g. Qwen3.5 / Qwen3-Next. Packing and + padding-free flatten the batch, and those recurrent + conv ops leak state + across sequence boundaries, so they must not be packed. Uses composite + structural evidence rather than a model-name match.""" + if model is None: + return False + + # Config-level: explicit hybrid layer schedule or linear-attn markers. + for config in (getattr(model, "config", None), getattr(getattr(model, "config", None), "text_config", None)): + if config is None: + continue + layer_types = getattr(config, "layer_types", None) + if isinstance(layer_types, (list, tuple)) and any( + isinstance(t, str) and "linear_attention" in t for t in layer_types + ): + return True + if any(hasattr(config, marker) for marker in _HYBRID_CONFIG_MARKERS): + return True + + # Module-level: a mixer carrying a recurrent gated-delta op plus a conv1d. + named_modules = getattr(model, "named_modules", None) + if named_modules is None: + return False + seen = set() + for _, module in named_modules(): + if id(module) in seen: + continue + seen.add(id(module)) + cls = type(module).__name__ + if not (cls.endswith("GatedDeltaNet") or "LinearAttention" in cls or cls.endswith("Mamba2Mixer")): + continue + has_recurrent = any( + hasattr(module, attr) + for attr in ("chunk_gated_delta_rule", "recurrent_gated_delta_rule", "A_log") + ) + if has_recurrent and hasattr(module, "conv1d"): + return True + return False + + # Unsloth gradient accumulation fix: from transformers import __version__ as transformers_version, ProcessorMixin @@ -574,12 +626,14 @@ def new_init(self, *args, **kwargs): model = args[0] if len(args) >= 1 else kwargs.get("model") is_vlm = False is_unsupported_model = False + is_hybrid = False if model is not None: model_config = getattr(model, "config", None) if model_config is not None: model_types = get_transformers_model_type(model_config) is_unsupported_model = any(x in PADDING_FREE_BLOCKLIST for x in model_types) is_vlm = _is_vlm_config(model_config, model_types) + is_hybrid = _is_hybrid_linear_attention_model(model) processing_class = ( args[5] if len(args) >= 6 else kwargs.get("processing_class") or kwargs.get("tokenizer") @@ -605,6 +659,7 @@ def new_init(self, *args, **kwargs): or is_auto_processor_vlm or is_vision_dataset or is_unsupported_model + or is_hybrid or ( os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1" ) # Disable padding free on forced logits @@ -624,6 +679,8 @@ def new_init(self, *args, **kwargs): reason = "vision-language model with auto processor" elif is_vision_dataset: reason = "vision dataset" + elif is_hybrid: + reason = "hybrid linear-attention model" elif is_unsupported_model: reason = f"unsupported model type(s): {', '.join(model_types)}" message = f"Unsloth: Sample packing skipped ({reason} detected)." From 47cc8d4eeb6ce38a4f7ade3cf6c718b4b5e2e056 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 19 Jul 2026 12:39:48 +0000 Subject: [PATCH 13/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/trainer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 45baf26ce0d..61d41aad210 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -230,7 +230,10 @@ def _is_hybrid_linear_attention_model(model) -> bool: return False # Config-level: explicit hybrid layer schedule or linear-attn markers. - for config in (getattr(model, "config", None), getattr(getattr(model, "config", None), "text_config", None)): + for config in ( + getattr(model, "config", None), + getattr(getattr(model, "config", None), "text_config", None), + ): if config is None: continue layer_types = getattr(config, "layer_types", None) @@ -251,7 +254,9 @@ def _is_hybrid_linear_attention_model(model) -> bool: continue seen.add(id(module)) cls = type(module).__name__ - if not (cls.endswith("GatedDeltaNet") or "LinearAttention" in cls or cls.endswith("Mamba2Mixer")): + if not ( + cls.endswith("GatedDeltaNet") or "LinearAttention" in cls or cls.endswith("Mamba2Mixer") + ): continue has_recurrent = any( hasattr(module, attr) From 92f5efecfa46344580d9b50049ede4fd60e23deb Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 20 Jul 2026 04:32:30 +0000 Subject: [PATCH 14/15] Install wrapped-packing setup at the signature, not the Zoo license comment The _unsloth_wrapped_packing / _inspect setup block was injected by matching the exact 'All Unsloth Zoo code licensed under LGPLv3' comment line in the sourced sft_prepare_dataset. The unsloth_zoo dependency is only lower-bounded, so a newer Zoo that moves or drops that header made the setup a silent no-op while the truncation and pack_dataset rewrites still emitted references to those names, raising NameError on every SFT dataset preparation. Anchor the setup on the function signature instead (a structural location that always exists) and fail loudly if it cannot be found, so the helper variables are always defined before they are referenced across Zoo versions. Adds a regression test that patches in a Zoo source without the license header. --- tests/utils/test_packing.py | 43 ++++++++++++++++ unsloth/models/rl_replacements.py | 83 +++++++++++++++++-------------- 2 files changed, 88 insertions(+), 38 deletions(-) diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py index a79c638b787..98c29d9f0fc 100644 --- a/tests/utils/test_packing.py +++ b/tests/utils/test_packing.py @@ -486,6 +486,49 @@ def legacy_pack_dataset( assert all(len(input_ids) <= args.max_length for input_ids in packed_ids) +# Named to match the unsloth_zoo helper: sft_trainer_prepare_dataset sources it by +# name and renames "def sft_prepare_dataset" -> "def _prepare_dataset". This fixture +# deliberately omits the "All Unsloth Zoo code licensed under LGPLv3" header to emulate +# a newer, compatible Zoo whose header moved (the dependency is only lower-bounded). +def sft_prepare_dataset( + self, dataset, processing_class, args, packing, formatting_func, dataset_text_field +): + do_truncation = True + # Mirror the Zoo call so the "truncation = do_truncation," injection anchor + # survives formatting (a bare tuple assignment gets rewritten to a paren form). + dataset = processing_class( + dataset, + truncation = do_truncation, + ) + return dataset + + +def test_wrapped_packing_setup_survives_missing_zoo_header(monkeypatch): + # Regression: the wrapped-packing setup used to anchor on the Zoo license comment, + # so a header change made it a no-op while the truncation reference still landed, + # NameError-ing every SFT dataset preparation. It must now install via the + # signature and always precede the reference. + import ast + import textwrap + import unsloth.models.rl_replacements as rlr + + monkeypatch.setitem(rlr.RL_REPLACEMENTS, "sft_prepare_dataset", sft_prepare_dataset) + + source = ( + "def _prepare_dataset(self, dataset, processing_class, args, packing, " + "formatting_func, dataset_text_field):\n return dataset\n" + ) + patched = rlr.sft_trainer_prepare_dataset("_prepare_dataset", source) + + assert "_unsloth_wrapped_packing = packing" in patched + assert "import inspect as _inspect" in patched + assert "not _unsloth_wrapped_packing" in patched + assert patched.index("_unsloth_wrapped_packing = packing") < patched.index( + "truncation = do_truncation and not _unsloth_wrapped_packing" + ) + ast.parse(textwrap.dedent(patched)) + + class _DummyChild(torch.nn.Module): def __init__(self): super().__init__() diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index d9792a8db8e..088737bc320 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -276,17 +276,13 @@ def dpo_trainer_vision_signature_columns(function_name, function): _extra_columns = "".join(f' "{_k}",\n' for _k in _DPO_VISION_KEYS) new_function = function.replace( ' "image_sizes",\n "token_type_ids",\n', - f' "image_sizes",\n' - f"{_extra_columns}" - f' "token_type_ids",\n', + f' "image_sizes",\n{_extra_columns} "token_type_ids",\n', ) if new_function != function: return new_function return function.replace( ' "image_sizes",\n "ref_chosen_logps",\n', - f' "image_sizes",\n' - f"{_extra_columns}" - f' "ref_chosen_logps",\n', + f' "image_sizes",\n{_extra_columns} "ref_chosen_logps",\n', ) @@ -458,21 +454,36 @@ def sft_trainer_prepare_dataset(function_name, function): if matched: # Use fast version! function = inspect.getsource(fast_sft_prepare_dataset) - function = function.replace( - " # All Unsloth Zoo code licensed under LGPLv3\n", - """ # All Unsloth Zoo code licensed under LGPLv3 - import inspect as _inspect - try: - _unsloth_pack_has_strategy = "strategy" in _inspect.signature(pack_dataset).parameters - except Exception: - _unsloth_pack_has_strategy = True - _unsloth_wrapped_packing = packing and ( - getattr(args, "packing_strategy", None) == "wrapped" - or not _unsloth_pack_has_strategy - ) -""", - 1, + # why: install the wrapped-packing setup (and the `_inspect` import the + # truncation / pack_dataset rewrites below depend on) at the function + # signature, a structural anchor that always exists, rather than the + # unsloth_zoo license-comment line. That header is only lower-bounded, so a + # newer Zoo may move or drop it; anchoring there let the setup silently + # no-op while the references still landed, NameError-ing every SFT dataset + # preparation. Fail loudly if even the signature cannot be located. + _wrapped_packing_setup = ( + " import inspect as _inspect\n" + " try:\n" + ' _unsloth_pack_has_strategy = "strategy" in _inspect.signature(pack_dataset).parameters\n' + " except Exception:\n" + " _unsloth_pack_has_strategy = True\n" + " _unsloth_wrapped_packing = packing and (\n" + ' getattr(args, "packing_strategy", None) == "wrapped"\n' + " or not _unsloth_pack_has_strategy\n" + " )\n" ) + function, _n_setup = re.subn( + r"(def sft_prepare_dataset\s*\(.*?\)\s*(?:->[^:\n]*)?:[ \t]*\n)", + lambda match: match.group(1) + _wrapped_packing_setup, + function, + count = 1, + flags = re.DOTALL, + ) + if _n_setup != 1: + raise RuntimeError( + "Unsloth: failed to install wrapped-packing support into " + "sft_prepare_dataset (signature not found); please file a bug report." + ) function = function.replace( "truncation = do_truncation,", "truncation = do_truncation and not _unsloth_wrapped_packing,", @@ -2153,25 +2164,21 @@ def compute_loss( _logits_to_keep = logits_to_keep get_logps_func = ( - lambda model, - input_ids, - attention_mask, - logits_to_keep, - batch_size = None, - compute_entropy = False, - compute_efficient = False: self._get_per_token_logps( - model, input_ids, attention_mask, logits_to_keep, compute_efficient + lambda model, input_ids, attention_mask, logits_to_keep, batch_size = None, compute_entropy = False, compute_efficient = False: ( + self._get_per_token_logps( + model, input_ids, attention_mask, logits_to_keep, compute_efficient + ) + if hasattr(self, "_get_per_token_logps") + else self._get_per_token_logps_and_entropies( + model, + input_ids, + attention_mask, + logits_to_keep, + batch_size, + compute_entropy, + compute_efficient, + )[0] ) - if hasattr(self, "_get_per_token_logps") - else self._get_per_token_logps_and_entropies( - model, - input_ids, - attention_mask, - logits_to_keep, - batch_size, - compute_entropy, - compute_efficient, - )[0] ) # logps per_token_logps = get_logps_func( From 07fe3bd02e77f8ae2c69d5ad89005e9d1c3ada71 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 04:33:43 +0000 Subject: [PATCH 15/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl_replacements.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 088737bc320..b0709f73760 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -2164,7 +2164,13 @@ def compute_loss( _logits_to_keep = logits_to_keep get_logps_func = ( - lambda model, input_ids, attention_mask, logits_to_keep, batch_size = None, compute_entropy = False, compute_efficient = False: ( + lambda model, + input_ids, + attention_mask, + logits_to_keep, + batch_size = None, + compute_entropy = False, + compute_efficient = False: ( self._get_per_token_logps( model, input_ids, attention_mask, logits_to_keep, compute_efficient )