Torch 2.2

pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 ]
 
 [tool.setuptools.dynamic]
-version = {attr = "unsloth.__version__"}
+version = {attr = "unsloth.models._utils.__version__"}
 
 [tool.setuptools]
 include-package-data = false
@@ -62,6 +62,16 @@ cu121onlytorch211 = [
     "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
     "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
 ]
+cu118onlytorch220 = [
+    "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
+    "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
+    "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
+]
+cu121onlytorch220 = [
+    "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
+    "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
+    "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
+]
 cu118 = [
     "unsloth[huggingface]",
     "bitsandbytes",
@@ -82,6 +92,16 @@ cu121_torch211 = [
     "bitsandbytes",
     "unsloth[cu121onlytorch211]",
 ]
+cu118_torch220 = [
+    "unsloth[huggingface]",
+    "bitsandbytes",
+    "unsloth[cu118onlytorch220]",
+]
+cu121_torch220 = [
+    "unsloth[huggingface]",
+    "bitsandbytes",
+    "unsloth[cu121onlytorch220]",
+]
 kaggle = [
     "unsloth[huggingface]",
 ]
@@ -110,6 +130,19 @@ colab_ampere_torch211 = [
     "ninja",
     "flash-attn",
 ]
+colab_torch220 = [
+    "unsloth[huggingface]",
+    "bitsandbytes",
+    "unsloth[cu121onlytorch220]",
+]
+colab_ampere_torch220 = [
+    "unsloth[huggingface]",
+    "bitsandbytes",
+    "unsloth[cu121onlytorch220]",
+    "packaging",
+    "ninja",
+    "flash-attn",
+]
 cu118_ampere = [
     "unsloth[huggingface]",
     "bitsandbytes",
@@ -142,6 +175,22 @@ cu121_ampere_torch211 = [
     "ninja",
     "flash-attn",
 ]
+cu118_ampere_torch220 = [
+    "unsloth[huggingface]",
+    "bitsandbytes",
+    "unsloth[cu118onlytorch220]",
+    "packaging",
+    "ninja",
+    "flash-attn",
+]
+cu121_ampere_torch220 = [
+    "unsloth[huggingface]",
+    "bitsandbytes",
+    "unsloth[cu121onlytorch220]",
+    "packaging",
+    "ninja",
+    "flash-attn",
+]
 
 [project.urls]
 homepage = "http://www.unsloth.ai"

unsloth/__init__.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-__version__ = "2024.1"
 import os
 import warnings
 import importlib

unsloth/models/llama.py

@@ -171,15 +171,28 @@ def LlamaAttention_fast_forward_inference(
     Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
     Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)
 
+    # Handle sliding windows
+    sliding_window = getattr(self.config, "sliding_window", None)
+    if sliding_window is not None and kv_seq_len > sliding_window:
+        # From https://github.com/huggingface/transformers/blob/main/src/transformers/models/mistral/modeling_mistral.py#L193
+        slicing_tokens = 1 - sliding_window
+        Knn = Kn[:, :, slicing_tokens:, :]#.contiguous()
+        Vnn = Vn[:, :, slicing_tokens:, :]#.contiguous()
+    else:
+        Knn, Vnn = Kn, Vn
+    pass
+
     # Grouped query attention
     if n_groups != 1:
-        _, _, cached_len, _ = Kn.shape
-        Knn = Kn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
-        Vnn = Vn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
+        _, _, cached_len, _ = Knn.shape
+        Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
+        Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
         Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
         Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)
-    else:
-        Knn, Vnn = Kn, Vn
+    pass
+    # else:
+    #     Knn, Vnn = Knn, Vnn
+    # pass
 
     # Attention
     A = torch.matmul(Qn, Knn.transpose(2, 3), out = self.attention[:,:,:,:kv_seq_len])

unsloth/models/loader.py

@@ -128,7 +128,7 @@ def from_pretrained(
                 "bnb_4bit_use_double_quant"        : True,
                 "llm_int8_enable_fp32_cpu_offload" : False,
                 "llm_int8_has_fp16_weight"         : False,
-                "llm_int8_skip_modules"            : "null",
+                "llm_int8_skip_modules"            : None,
                 "llm_int8_threshold"               : 6.0,
                 "load_in_4bit"                     : True,
                 "load_in_8bit"                     : False,

unsloth/save.py

@@ -72,6 +72,7 @@ def print_quantization_methods():
 pass
 
 
+
 def _merge_lora(layer, name):
 
     if isinstance(layer, (Bnb_Linear4bit, Peft_Linear4bit, Peft_Linear)):
@@ -85,9 +86,12 @@ def _merge_lora(layer, name):
         W = W.to(torch.float32).t()
 
         if A is not None:
-            sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32)))
-            W += sAB
-            if not torch.isfinite(W).all():
+            # sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32)))
+            # W += sAB
+            W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s)
+            # if not torch.isfinite(W).all():
+            maximum_element = torch.max(W.min().abs(), W.max())
+            if not torch.isfinite(maximum_element).item():
                 raise ValueError(f"Unsloth: Merge failed.\n{name} has some elements = infinity.")
         pass
         W = W.t().to(dtype)
@@ -373,7 +377,7 @@ def unsloth_save_model(
             # elif (max_ram - W.nbytes) > 0:
             #     # Save to CPU memory
             #     logger.warning_once(f"We will save to RAM and not VRAM now.")
-            #     state_dict[name] = W.to("cpu", non_blocking = True)
+            #     state_dict[name] = W.to("cpu", non_blocking = True, copy = True)
             #     max_ram = max(max_ram - W.nbytes, 0)
             else:
                 # Save to Disk
@@ -579,9 +583,11 @@ def save_to_gguf(
         f"--outfile {final_location} "\
         f"--outtype {first_conversion} --concurrency {n_cpus}"
 
-    with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, bufsize = 1) as sp:
+    with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE, bufsize = 1) as sp:
         for line in sp.stdout:
             print(line.decode("utf-8"), flush = True, end = "")
+        if sp.returncode is not None and sp.returncode != 0:
+            raise subprocess.CalledProcessError(sp.returncode, sp.args)
     pass
 
     # Check if quantization succeeded!
@@ -609,6 +615,8 @@ def save_to_gguf(
         with subprocess.Popen(command, shell = True, stderr = subprocess.PIPE, bufsize = 1) as sp:
             for line in sp.stderr:
                 print(line.decode("utf-8"), flush = True, end = "")
+            if sp.returncode is not None and sp.returncode != 0:
+                raise subprocess.CalledProcessError(sp.returncode, sp.args)
         pass
 
         # Check if quantization succeeded!