From d131bde18349af16e9a3797ca140b2dde6565642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E9=BE=8D=E8=81=96=E8=80=85=40bdsqlsz?= <865105819@qq.com> Date: Sat, 22 Jul 2023 18:45:32 +0800 Subject: [PATCH] Support for bitsandbytes 0.39.1 with Paged Optimizer(AdamW8bit and Lion8bit) (#631) * ADD libbitsandbytes.dll for 0.38.1 * Delete libbitsandbytes_cuda116.dll * Delete cextension.py * add main.py * Update requirements.txt for bitsandbytes 0.38.1 * Update README.md for bitsandbytes-windows * Update README-ja.md for bitsandbytes 0.38.1 * Update main.py for return cuda118 * Update train_util.py for lion8bit * Update train_README-ja.md for lion8bit * Update train_util.py for add DAdaptAdan and DAdaptSGD * Update train_util.py for DAdaptadam * Update train_network.py for dadapt * Update train_README-ja.md for DAdapt * Update train_util.py for DAdapt * Update train_network.py for DAdaptAdaGrad * Update train_db.py for DAdapt * Update fine_tune.py for DAdapt * Update train_textual_inversion.py for DAdapt * Update train_textual_inversion_XTI.py for DAdapt * Revert "Merge branch 'qinglong' into main" This reverts commit b65c023083d6d1e8a30eb42eddd603d1aac97650, reversing changes made to f6fda20caf5e773d56bcfb5c4575c650bb85362b. * Revert "Update requirements.txt for bitsandbytes 0.38.1" This reverts commit 83abc60dfaddb26845f54228425b98dd67997528. * Revert "Delete cextension.py" This reverts commit 3ba4dfe046874393f2a022a4cbef3628ada35391. * Revert "Update README.md for bitsandbytes-windows" This reverts commit 4642c52086b5e9791233007e2fdfd97f832cd897. * Revert "Update README-ja.md for bitsandbytes 0.38.1" This reverts commit fa6d7485ac067ebc49e6f381afdb8dd2f12caa8f. * Update train_util.py * Update requirements.txt * support PagedAdamW8bit/PagedLion8bit * Update requirements.txt * update for PageAdamW8bit and PagedLion8bit * Revert * revert main --- README-ja.md | 10 ++++++++++ README.md | 10 ++++++++++ docs/train_README-ja.md | 2 ++ docs/train_README-zh.md | 5 ++++- library/train_util.py | 39 ++++++++++++++++++++++++++++----------- 5 files changed, 54 insertions(+), 12 deletions(-) diff --git a/README-ja.md b/README-ja.md index 6f0e574e2..865e0d35a 100644 --- a/README-ja.md +++ b/README-ja.md @@ -125,6 +125,16 @@ pip install https://github.com/jllllll/bitsandbytes-windows-webui/raw/main/bitsa アップグレード時には`pip install .`でこのリポジトリを更新し、必要に応じて他のパッケージもアップグレードしてください。 +### オプション：PagedAdamW8bitとPagedLion8bitを使う + +PagedAdamW8bitとPagedLion8bitを使う場合には`bitsandbytes`を0.39.0以降にアップグレードする必要があります。`bitsandbytes`をアンインストールし、Windows環境では例えば[こちら](https://github.com/jllllll/bitsandbytes-windows-webui)などからWindows版のwhlファイルをインストールしてください。たとえば以下のような手順になります。 + +```powershell +pip install https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.39.1-py3-none-win_amd64.whl +``` + +アップグレード時には`pip install .`でこのリポジトリを更新し、必要に応じて他のパッケージもアップグレードしてください。 + ## アップグレード 新しいリリースがあった場合、以下のコマンドで更新できます。 diff --git a/README.md b/README.md index e6202baec..898c78683 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,16 @@ pip install https://github.com/jllllll/bitsandbytes-windows-webui/raw/main/bitsa For upgrading, upgrade this repo with `pip install .`, and upgrade necessary packages manually. +### Optional: Use PagedAdamW8bit and PagedLion8bit + +For PagedAdamW8bit and PagedLion8bit, you need to upgrade `bitsandbytes` to 0.39.0 or later. Uninstall `bitsandbytes`, and for Windows, install the Windows version whl file from [here](https://github.com/jllllll/bitsandbytes-windows-webui) or other sources, like: + +```powershell +pip install https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.39.1-py3-none-win_amd64.whl +``` + +For upgrading, upgrade this repo with `pip install .`, and upgrade necessary packages manually. + ## Upgrade When a new release comes out you can upgrade your repo with the following command: diff --git a/docs/train_README-ja.md b/docs/train_README-ja.md index 158363b39..37da7591c 100644 --- a/docs/train_README-ja.md +++ b/docs/train_README-ja.md @@ -609,10 +609,12 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b - AdamW : [torch.optim.AdamW](https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html) - 過去のバージョンのオプション未指定時と同じ - AdamW8bit : 引数は同上 + - PagedAdamW8bit : 引数は同上 - 過去のバージョンの--use_8bit_adam指定時と同じ - Lion : https://github.com/lucidrains/lion-pytorch - 過去のバージョンの--use_lion_optimizer指定時と同じ - Lion8bit : 引数は同上 + - PagedLion8bit : 引数は同上 - SGDNesterov : [torch.optim.SGD](https://pytorch.org/docs/stable/generated/torch.optim.SGD.html), nesterov=True - SGDNesterov8bit : 引数は同上 - DAdaptation(DAdaptAdamPreprint) : https://github.com/facebookresearch/dadaptation diff --git a/docs/train_README-zh.md b/docs/train_README-zh.md index 454d54561..ee4798cab 100644 --- a/docs/train_README-zh.md +++ b/docs/train_README-zh.md @@ -545,9 +545,12 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b -- 指定优化器类型。您可以指定 - AdamW : [torch.optim.AdamW](https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html) - 与过去版本中未指定选项时相同 - - AdamW8bit : 同上 + - AdamW8bit : 参数同上 + - PagedAdamW8bit : 参数同上 - 与过去版本中指定的 --use_8bit_adam 相同 - Lion : https://github.com/lucidrains/lion-pytorch + - Lion8bit : 参数同上 + - PagedLion8bit : 参数同上 - 与过去版本中指定的 --use_lion_optimizer 相同 - SGDNesterov : [torch.optim.SGD](https://pytorch.org/docs/stable/generated/torch.optim.SGD.html), nesterov=True - SGDNesterov8bit : 参数同上 diff --git a/library/train_util.py b/library/train_util.py index acfb503be..3a68bed95 100644 --- a/library/train_util.py +++ b/library/train_util.py @@ -2033,7 +2033,7 @@ def add_optimizer_arguments(parser: argparse.ArgumentParser): "--optimizer_type", type=str, default="", - help="Optimizer to use / オプティマイザの種類: AdamW (default), AdamW8bit, Lion8bit, Lion, SGDNesterov, SGDNesterov8bit, DAdaptation(DAdaptAdamPreprint), DAdaptAdaGrad, DAdaptAdam, DAdaptAdan, DAdaptAdanIP, DAdaptLion, DAdaptSGD, AdaFactor", + help="Optimizer to use / オプティマイザの種類: AdamW (default), AdamW8bit, PagedAdamW8bit, Lion8bit, PagedLion8bit, Lion, SGDNesterov, SGDNesterov8bit, DAdaptation(DAdaptAdamPreprint), DAdaptAdaGrad, DAdaptAdam, DAdaptAdan, DAdaptAdanIP, DAdaptLion, DAdaptSGD, AdaFactor", ) # backward compatibility @@ -2649,7 +2649,7 @@ def task(): def get_optimizer(args, trainable_params): - # "Optimizer to use: AdamW, AdamW8bit, Lion, SGDNesterov, SGDNesterov8bit, Lion8bit, DAdaptation(DAdaptAdamPreprint), DAdaptAdaGrad, DAdaptAdam, DAdaptAdan, DAdaptAdanIP, DAdaptLion, DAdaptSGD, Adafactor" + # "Optimizer to use: AdamW, AdamW8bit, Lion, SGDNesterov, SGDNesterov8bit, PagedAdamW8bit, Lion8bit, PagedLion8bit, DAdaptation(DAdaptAdamPreprint), DAdaptAdaGrad, DAdaptAdam, DAdaptAdan, DAdaptAdanIP, DAdaptLion, DAdaptSGD, Adafactor" optimizer_type = args.optimizer_type if args.use_8bit_adam: @@ -2726,20 +2726,37 @@ def get_optimizer(args, trainable_params): print(f"use Lion optimizer | {optimizer_kwargs}") optimizer_class = lion_pytorch.Lion optimizer = optimizer_class(trainable_params, lr=lr, **optimizer_kwargs) - - elif optimizer_type == "Lion8bit".lower(): + + elif optimizer_type.endswith("8bit".lower()): try: import bitsandbytes as bnb except ImportError: raise ImportError("No bitsandbytes / bitsandbytesがインストールされていないようです") - print(f"use 8-bit Lion optimizer | {optimizer_kwargs}") - try: - optimizer_class = bnb.optim.Lion8bit - except AttributeError: - raise AttributeError( - "No Lion8bit. The version of bitsandbytes installed seems to be old. Please install 0.38.0 or later. / Lion8bitが定義されていません。インストールされているbitsandbytesのバージョンが古いようです。0.38.0以上をインストールしてください" - ) + if optimizer_type == "Lion8bit".lower(): + print(f"use 8-bit Lion optimizer | {optimizer_kwargs}") + try: + optimizer_class = bnb.optim.Lion8bit + except AttributeError: + raise AttributeError( + "No Lion8bit. The version of bitsandbytes installed seems to be old. Please install 0.38.0 or later. / Lion8bitが定義されていません。インストールされているbitsandbytesのバージョンが古いようです。0.38.0以上をインストールしてください" + ) + elif optimizer_type == "PagedAdamW8bit".lower(): + print(f"use 8-bit PagedAdamW optimizer | {optimizer_kwargs}") + try: + optimizer_class = bnb.optim.PagedAdamW8bit + except AttributeError: + raise AttributeError( + "No PagedAdamW8bit. The version of bitsandbytes installed seems to be old. Please install 0.39.0 or later. / PagedAdamW8bitが定義されていません。インストールされているbitsandbytesのバージョンが古いようです。0.39.0以上をインストールしてください" + ) + elif optimizer_type == "PagedLion8bit".lower(): + print(f"use 8-bit Paged Lion optimizer | {optimizer_kwargs}") + try: + optimizer_class = bnb.optim.PagedLion8bit + except AttributeError: + raise AttributeError( + "No PagedLion8bit. The version of bitsandbytes installed seems to be old. Please install 0.39.0 or later. / PagedLion8bitが定義されていません。インストールされているbitsandbytesのバージョンが古いようです。0.39.0以上をインストールしてください" + ) optimizer = optimizer_class(trainable_params, lr=lr, **optimizer_kwargs)