diff --git a/pkgs/development/python-modules/huggingface-hub/default.nix b/pkgs/development/python-modules/huggingface-hub/default.nix index 2f448cef5b47f..2eaed71208641 100644 --- a/pkgs/development/python-modules/huggingface-hub/default.nix +++ b/pkgs/development/python-modules/huggingface-hub/default.nix @@ -35,14 +35,14 @@ buildPythonPackage (finalAttrs: { pname = "huggingface-hub"; - version = "1.3.4"; + version = "1.4.0"; pyproject = true; src = fetchFromGitHub { owner = "huggingface"; repo = "huggingface_hub"; tag = "v${finalAttrs.version}"; - hash = "sha256-q6BTpNawUPZ9LWSxJkq/GaLNLdmhAeJehyWVMvdtH24="; + hash = "sha256-jXXwjGJoPUSDb1ptgX3SvMKRz65vtho5XkV+QuowV0s="; }; build-system = [ setuptools ]; diff --git a/pkgs/development/python-modules/transformers/default.nix b/pkgs/development/python-modules/transformers/default.nix index c0679aa8779fc..1efce3f23db19 100644 --- a/pkgs/development/python-modules/transformers/default.nix +++ b/pkgs/development/python-modules/transformers/default.nix @@ -25,23 +25,28 @@ # torch torch, accelerate, - # hf_xet - hf-xet, + # deepspeed + # deepspeed, + # codecarbon + # codecarbon, # retrieval faiss, datasets, - # tokenizers - # ftfy - ftfy, - # modelcreation - cookiecutter, + # ja + fugashi, + ipadic, + sudachipy, + # sudachidict_core, + # unidic_lite, + unidic, + # rhoknp, # sagemaker sagemaker, # optuna optuna, # ray ray, - # hub-kernels + # kernels kernels, # serving openai, @@ -52,8 +57,9 @@ rich, # audio librosa, + # pyctcdecode, phonemizer, - # speech + # kenlm, torchaudio, # vision pillow, @@ -80,21 +86,22 @@ gitpython, urllib3, libcst, - pandas, - # torchhub - importlib-metadata, + # opentelemetry + opentelemetry-api, + opentelemetry-exporter-otlp, + opentelemetry-sdk, }: buildPythonPackage (finalAttrs: { pname = "transformers"; - version = "5.0.0"; + version = "5.1.0"; pyproject = true; src = fetchFromGitHub { owner = "huggingface"; repo = "transformers"; tag = "v${finalAttrs.version}"; - hash = "sha256-ART1ARd+hfC0GQNDa225SWF0zTFUKE4eDxFYbWFaTl8="; + hash = "sha256-DMm85M47hMWhqbwY3k3F5nbkbctM23K6wnmIUa2O43g="; }; build-system = [ setuptools ]; @@ -114,13 +121,13 @@ buildPythonPackage (finalAttrs: { optional-dependencies = lib.fix (self: { ja = [ - # fugashi - # ipadic + fugashi + ipadic # unidic_lite - # unidic - # sudachipy - # sudachidict_core + unidic # rhoknp + sudachipy + # sudachidict_core ]; sklearn = [ scikit-learn ]; torch = [ @@ -128,14 +135,10 @@ buildPythonPackage (finalAttrs: { accelerate ]; accelerate = [ accelerate ]; - hf_xet = [ hf-xet ]; retrieval = [ faiss datasets ]; - tokenizers = [ tokenizers ]; - ftfy = [ ftfy ]; - modelcreation = [ cookiecutter ]; sagemaker = [ sagemaker ]; deepspeed = [ # deepspeed @@ -143,8 +146,11 @@ buildPythonPackage (finalAttrs: { ++ self.accelerate; optuna = [ optuna ]; ray = [ ray ] ++ ray.optional-dependencies.tune; - hub-kernels = [ kernels ]; - integrations = self.hub-kernels ++ self.optuna ++ self.ray; + kernels = [ kernels ]; + codecarbon = [ + # codecarbon + ]; + integrations = self.kernels ++ self.optuna ++ self.codecarbon ++ self.ray; serving = [ openai pydantic @@ -155,22 +161,17 @@ buildPythonPackage (finalAttrs: { ] ++ self.torch; audio = [ + torchaudio librosa # pyctcdecode phonemizer # kenlm ]; - speech = [ torchaudio ] ++ self.audio; - torch-speech = [ torchaudio ] ++ self.audio; - vision = [ pillow ]; - timm = [ timm ]; - torch-vision = [ torchvision ] ++ self.vision; - natten = [ - # natten - ]; - codecarbon = [ - # codecarbon + vision = [ + torchvision + pillow ]; + timm = [ timm ]; video = [ av ]; num2words = [ num2words ]; sentencepiece = [ @@ -193,24 +194,15 @@ buildPythonPackage (finalAttrs: { urllib3 libcst rich - pandas - ]; - torchhub = [ - filelock - huggingface-hub - importlib-metadata - numpy - packaging - protobuf - regex - sentencepiece - torch - tokenizers - tqdm ]; benchmark = [ # optimum-benchmark ]; + open-telemetry = [ + opentelemetry-api + opentelemetry-exporter-otlp + opentelemetry-sdk + ]; }); # Many tests require internet access. @@ -226,6 +218,7 @@ buildPythonPackage (finalAttrs: { license = lib.licenses.asl20; platforms = lib.platforms.unix; maintainers = with lib.maintainers; [ + GaetanLepage pashashocky happysalada ];