Merge pull request #57 from nixified-ai/bitsandbytes-support-old-gpus

support bitsandbytes on older Nvidia GPUs (CC < 7.5)

Author: MatthewCroughan

overlays/python/default.nix

@@ -63,4 +63,10 @@ lib: {
       rocmSupport = false;
     };
   };
+
+  bitsAndBytesOldGpu = final: prev: {
+    bitsandbytes = prev.bitsandbytes.overridePythonAttrs (old: {
+      preBuild = old.preBuild + " cuda${final.torch.cudaPackages.cudaMajorVersion}x_nomatmul";
+    });
+  };
 }

projects/textgen/default.nix

@@ -30,6 +30,7 @@ in
       ]);
       nvidia = l.overlays.applyOverlays pkgs.python3Packages (commonOverlays ++ [
         overlays.python-torchCuda
+        overlays.python-bitsAndBytesOldGpu
       ]);
     };