Add transformers pyproject.toml to test requirements (#5657)

This `pyproject.toml`, translated from https://github.com/huggingface/transformers/blob/0e4a1c3401421d3342938a183c7b021cdac6fd79/setup.py, has found a number of bugs in blue jay, so i'm adding it to uv as a development test case. (The file is too large to make it a test by itself.)
astral-sh · Aug 1, 2024 · 9a1a211 · 9a1a211
1 parent b9866b3
commit 9a1a211
Show file tree

Hide file tree

Showing 2 changed files with 298 additions and 0 deletions.
diff --git a/scripts/requirements/.gitignore b/scripts/requirements/.gitignore
@@ -0,0 +1 @@
+uv.lock
diff --git a/scripts/requirements/transformers/pyproject.toml b/scripts/requirements/transformers/pyproject.toml
@@ -0,0 +1,297 @@
+# This file is a translation of
+# https://github.com/huggingface/transformers/blob/0e4a1c3401421d3342938a183c7b021cdac6fd79/setup.py
+# to PEP 621 and `tool.uv` sources.
+
+[project]
+name = "transformers"
+version = "4.39.0.dev0"
+requires-python = ">=3.9.0"
+
+dependencies = [
+    "filelock",
+    "huggingface-hub>=0.19.3,<1.0",
+    "numpy>=1.17",
+    "packaging>=20.0",
+    "pyyaml>=5.1",
+    "regex!=2019.12.17",
+    "requests",
+    "tokenizers>=0.14,<0.19",
+    "safetensors>=0.4.1",
+    "tqdm>=4.27",
+]
+
+[project.optional-dependencies]
+ja = [
+    "fugashi>=1.0",
+    "ipadic>=1.0.0,<2.0",
+    "unidic_lite>=1.0.7",
+    "unidic>=1.0.2",
+    "sudachipy>=0.6.6",
+    "sudachidict_core>=20220729",
+    "rhoknp>=1.1.0,<1.3.1"
+]
+sklearn = ["scikit-learn"]
+tf = [
+    "tensorflow>=2.6,<2.16",
+    "onnxconverter-common",
+    "tf2onnx",
+    "tensorflow-text<2.16",
+    "keras-nlp>=0.3.1"
+]
+tf-cpu = [
+    "tensorflow-cpu>=2.6,<2.16",
+    "onnxconverter-common",
+    "tf2onnx",
+    "tensorflow-text<2.16",
+    "keras-nlp>=0.3.1"
+]
+torch = ["torch", "accelerate>=0.21.0"]
+accelerate = ["accelerate>=0.21.0"]
+retrieval = ["faiss-cpu", "datasets!=2.5.0"]
+flax = [
+    "jax>=0.4.1,<=0.4.13",
+    "jaxlib>=0.4.1,<=0.4.13",
+    "flax>=0.4.1,<=0.7.0",
+    "optax>=0.0.8,<=0.1.4"
+]
+tokenizers = ["tokenizers>=0.14,<0.19"]
+ftfy = ["ftfy"]
+onnxruntime = ["onnxruntime>=1.4.0", "onnxruntime-tools>=1.4.2"]
+onnx = [
+    "onnxconverter-common",
+    "tf2onnx",
+    "onnxruntime>=1.4.0",
+    "onnxruntime-tools>=1.4.2"
+]
+modelcreation = ["cookiecutter==1.7.3"]
+sagemaker = ["sagemaker==2.226.1"] # sagemaker>=2.31.0 # TODO(konsti)
+deepspeed = ["deepspeed>=0.9.3", "accelerate>=0.21.0"]
+optuna = ["optuna"]
+ray = ["ray[tune]>=2.7.0"]
+sigopt = ["sigopt"]
+integrations = ["optuna", "ray[tune]>=2.7.0", "sigopt"]
+serving = ["pydantic", "uvicorn", "fastapi", "starlette"]
+audio = ["librosa", "pyctcdecode>=0.4.0", "phonemizer", "kenlm"]
+speech = [
+    "torchaudio",
+    "librosa",
+    "pyctcdecode>=0.4.0",
+    "phonemizer",
+    "kenlm"
+]
+torch-speech = [
+    "torchaudio",
+    "librosa",
+    "pyctcdecode>=0.4.0",
+    "phonemizer",
+    "kenlm"
+]
+tf-speech = ["librosa", "pyctcdecode>=0.4.0", "phonemizer", "kenlm"]
+flax-speech = ["librosa", "pyctcdecode>=0.4.0", "phonemizer", "kenlm"]
+vision = ["Pillow>=10.0.1,<=15.0"]
+timm = ["timm"]
+torch-vision = ["torchvision", "Pillow>=10.0.1,<=15.0"]
+# natten = ["natten>=0.14.6,<0.15.0"] # TODO(konsti)
+codecarbon = ["codecarbon==1.2.0"]
+video = ["decord==0.6.0", "av==9.2.0"]
+sentencepiece = ["sentencepiece>=0.1.91,!=0.1.92", "protobuf"]
+
+deepspeed-testing = [
+    "deepspeed>=0.9.3",
+    "accelerate>=0.21.0",
+    "pytest>=7.2.0,<8.0.0",
+    "pytest-xdist",
+    "timeout-decorator",
+    "parameterized",
+    "psutil",
+    "datasets!=2.5.0",
+    "dill<0.3.5",
+    "evaluate>=0.2.0",
+    "pytest-timeout",
+    "ruff==0.1.5",
+    "sacrebleu>=1.4.12,<2.0.0",
+    "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
+    "nltk",
+    "GitPython<3.1.19",
+    "hf-doc-builder>=0.3.0",
+    "protobuf",
+    "sacremoses",
+    "rjieba",
+    "beautifulsoup4",
+    "tensorboard",
+    "pydantic",
+    "faiss-cpu",
+    "datasets!=2.5.0",
+    "cookiecutter==1.7.3",
+    "optuna",
+    "sentencepiece>=0.1.91,!=0.1.92",
+    "protobuf"
+]
+quality = [
+    "datasets!=2.5.0",
+    "isort>=5.5.4",
+    "ruff==0.1.5",
+    "GitPython<3.1.19",
+    "hf-doc-builder>=0.3.0",
+    "urllib3<2.0.0"
+]
+all = [
+    "tensorflow>=2.6,<2.16",
+    "onnxconverter-common",
+    "tf2onnx",
+    "tensorflow-text<2.16",
+    "keras-nlp>=0.3.1",
+    "torch",
+    "accelerate>=0.21.0",
+    "jax>=0.4.1,<=0.4.13",
+    "jaxlib>=0.4.1,<=0.4.13",
+    "flax>=0.4.1,<=0.7.0",
+    "optax>=0.0.8,<=0.1.4",
+    "sentencepiece>=0.1.91,!=0.1.92",
+    "protobuf",
+    "tokenizers>=0.14,<0.19",
+    "torchaudio",
+    "librosa",
+    "pyctcdecode>=0.4.0",
+    "phonemizer",
+    "kenlm",
+    "Pillow>=10.0.1,<=15.0",
+    "optuna",
+    "ray[tune]>=2.7.0",
+    "sigopt",
+    "timm",
+    "torchvision",
+    "Pillow>=10.0.1,<=15.0",
+    "codecarbon==1.2.0",
+    "accelerate>=0.21.0",
+    "decord==0.6.0",
+    "av==9.2.0"
+]
+docs_specific = ["hf-doc-builder"]
+docs = [
+    "tensorflow>=2.6,<2.16",
+    "onnxconverter-common",
+    "tf2onnx",
+    "tensorflow-text<2.16",
+    "keras-nlp>=0.3.1",
+    "torch",
+    "accelerate>=0.21.0",
+    "jax>=0.4.1,<=0.4.13",
+    "jaxlib>=0.4.1,<=0.4.13",
+    "flax>=0.4.1,<=0.7.0",
+    "optax>=0.0.8,<=0.1.4",
+    "sentencepiece>=0.1.91,!=0.1.92",
+    "protobuf",
+    "tokenizers>=0.14,<0.19",
+    "torchaudio",
+    "librosa",
+    "pyctcdecode>=0.4.0",
+    "phonemizer",
+    "kenlm",
+    "Pillow>=10.0.1,<=15.0",
+    "optuna",
+    "ray[tune]>=2.7.0",
+    "sigopt",
+    "timm",
+    "torchvision",
+    "Pillow>=10.0.1,<=15.0",
+    "codecarbon==1.2.0",
+    "accelerate>=0.21.0",
+    "decord==0.6.0",
+    "av==9.2.0",
+    "hf-doc-builder"
+]
+torchhub = [
+    "filelock",
+    "huggingface-hub>=0.19.3,<1.0",
+    "importlib_metadata",
+    "numpy>=1.17",
+    "packaging>=20.0",
+    "protobuf",
+    "regex!=2019.12.17",
+    "requests",
+    "sentencepiece>=0.1.91,!=0.1.92",
+    "torch",
+    "tokenizers>=0.14,<0.19",
+    "tqdm>=4.27"
+]
+agents = [
+    "diffusers",
+    "accelerate>=0.21.0",
+    "datasets!=2.5.0",
+    "torch",
+    "sentencepiece>=0.1.91,!=0.1.92",
+    "opencv-python",
+    "Pillow>=10.0.1,<=15.0"
+]
+
+dev-dependencies = [
+    "GitPython<3.1.19",
+    "Pillow>=10.0.1,<=15.0",
+    "accelerate>=0.21.0",
+    "av==9.2.0",
+    "beautifulsoup4",
+    "codecarbon==1.2.0",
+    "codecarbon==1.2.0",
+    "cookiecutter==1.7.3",
+    "datasets!=2.5.0",
+    "decord==0.6.0",
+    "dill<0.3.5",
+    "evaluate>=0.2.0",
+    "faiss-cpu",
+    "flax>=0.4.1,<=0.7.0",
+    "fugashi>=1.0",
+    "hf-doc-builder",
+    "hf-doc-builder>=0.3.0",
+    "ipadic>=1.0.0,<2.0",
+    "isort>=5.5.4",
+    "jax>=0.4.1,<=0.4.13",
+    "jaxlib>=0.4.1,<=0.4.13",
+    "kenlm",
+    "keras-nlp>=0.3.1",
+    "librosa",
+    "nltk",
+    "onnxconverter-common",
+    "onnxruntime-tools>=1.4.2",
+    "onnxruntime>=1.4.0",
+    "optax>=0.0.8,<=0.1.4",
+    "optuna",
+    "parameterized",
+    "phonemizer",
+    "protobuf",
+    "psutil",
+    "pyctcdecode>=0.4.0",
+    "pydantic",
+    "pytest-timeout",
+    "pytest-xdist",
+    "pytest>=7.2.0,<8.0.0",
+    "ray[tune]>=2.7.0",
+    "rhoknp>=1.1.0,<1.3.1",
+    "rjieba",
+    "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
+    "ruff==0.1.5",
+    "sacrebleu>=1.4.12,<2.0.0",
+    "sacremoses",
+    "scikit-learn",
+    "sentencepiece>=0.1.91,!=0.1.92",
+    "sigopt",
+    "sudachidict_core>=20220729",
+    "sudachipy>=0.6.6",
+    "tensorboard",
+    "tensorflow-text<2.16",
+    "tensorflow>=2.6,<2.16",
+    "tf2onnx",
+    "timeout-decorator",
+    "timm",
+    "tokenizers>=0.14,<0.19",
+    "torch",
+    "torchaudio",
+    "torchvision",
+    "unidic>=1.0.2",
+    "unidic_lite>=1.0.7",
+    "urllib3<2.0.0"
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"