python3Packages.huggingface-hub: 1.3.4 -> 1.4.0; python3Packages.transformers: 5.0.0 -> 5.1.0

pkgs/development/python-modules/huggingface-hub/default.nix

@@ -35,14 +35,14 @@
 
 buildPythonPackage (finalAttrs: {
   pname = "huggingface-hub";
-  version = "1.3.4";
+  version = "1.4.0";
   pyproject = true;
 
   src = fetchFromGitHub {
     owner = "huggingface";
     repo = "huggingface_hub";
     tag = "v${finalAttrs.version}";
-    hash = "sha256-q6BTpNawUPZ9LWSxJkq/GaLNLdmhAeJehyWVMvdtH24=";
+    hash = "sha256-jXXwjGJoPUSDb1ptgX3SvMKRz65vtho5XkV+QuowV0s=";
   };
 
   build-system = [ setuptools ];

pkgs/development/python-modules/transformers/default.nix

@@ -25,23 +25,28 @@
   # torch
   torch,
   accelerate,
-  # hf_xet
-  hf-xet,
+  # deepspeed
+  # deepspeed,
+  # codecarbon
+  # codecarbon,
   # retrieval
   faiss,
   datasets,
-  # tokenizers
-  # ftfy
-  ftfy,
-  # modelcreation
-  cookiecutter,
+  # ja
+  fugashi,
+  ipadic,
+  sudachipy,
+  # sudachidict_core,
+  # unidic_lite,
+  unidic,
+  # rhoknp,
   # sagemaker
   sagemaker,
   # optuna
   optuna,
   # ray
   ray,
-  # hub-kernels
+  # kernels
   kernels,
   # serving
   openai,
@@ -52,8 +57,9 @@
   rich,
   # audio
   librosa,
+  # pyctcdecode,
   phonemizer,
-  # speech
+  # kenlm,
   torchaudio,
   # vision
   pillow,
@@ -80,21 +86,22 @@
   gitpython,
   urllib3,
   libcst,
-  pandas,
-  # torchhub
-  importlib-metadata,
+  # opentelemetry
+  opentelemetry-api,
+  opentelemetry-exporter-otlp,
+  opentelemetry-sdk,
 }:
 
 buildPythonPackage (finalAttrs: {
   pname = "transformers";
-  version = "5.0.0";
+  version = "5.1.0";
   pyproject = true;
 
   src = fetchFromGitHub {
     owner = "huggingface";
     repo = "transformers";
     tag = "v${finalAttrs.version}";
-    hash = "sha256-ART1ARd+hfC0GQNDa225SWF0zTFUKE4eDxFYbWFaTl8=";
+    hash = "sha256-DMm85M47hMWhqbwY3k3F5nbkbctM23K6wnmIUa2O43g=";
   };
 
   build-system = [ setuptools ];
@@ -114,37 +121,36 @@ buildPythonPackage (finalAttrs: {
 
   optional-dependencies = lib.fix (self: {
     ja = [
-      # fugashi
-      # ipadic
+      fugashi
+      ipadic
       # unidic_lite
-      # unidic
-      # sudachipy
-      # sudachidict_core
+      unidic
       # rhoknp
+      sudachipy
+      # sudachidict_core
     ];
     sklearn = [ scikit-learn ];
     torch = [
       torch
       accelerate
     ];
     accelerate = [ accelerate ];
-    hf_xet = [ hf-xet ];
     retrieval = [
       faiss
       datasets
     ];
-    tokenizers = [ tokenizers ];
-    ftfy = [ ftfy ];
-    modelcreation = [ cookiecutter ];
     sagemaker = [ sagemaker ];
     deepspeed = [
       # deepspeed
     ]
     ++ self.accelerate;
     optuna = [ optuna ];
     ray = [ ray ] ++ ray.optional-dependencies.tune;
-    hub-kernels = [ kernels ];
-    integrations = self.hub-kernels ++ self.optuna ++ self.ray;
+    kernels = [ kernels ];
+    codecarbon = [
+      # codecarbon
+    ];
+    integrations = self.kernels ++ self.optuna ++ self.codecarbon ++ self.ray;
     serving = [
       openai
       pydantic
@@ -155,22 +161,17 @@ buildPythonPackage (finalAttrs: {
     ]
     ++ self.torch;
     audio = [
+      torchaudio
       librosa
       # pyctcdecode
       phonemizer
       # kenlm
     ];
-    speech = [ torchaudio ] ++ self.audio;
-    torch-speech = [ torchaudio ] ++ self.audio;
-    vision = [ pillow ];
-    timm = [ timm ];
-    torch-vision = [ torchvision ] ++ self.vision;
-    natten = [
-      # natten
-    ];
-    codecarbon = [
-      # codecarbon
+    vision = [
+      torchvision
+      pillow
     ];
+    timm = [ timm ];
     video = [ av ];
     num2words = [ num2words ];
     sentencepiece = [
@@ -193,24 +194,15 @@ buildPythonPackage (finalAttrs: {
       urllib3
       libcst
       rich
-      pandas
-    ];
-    torchhub = [
-      filelock
-      huggingface-hub
-      importlib-metadata
-      numpy
-      packaging
-      protobuf
-      regex
-      sentencepiece
-      torch
-      tokenizers
-      tqdm
     ];
     benchmark = [
       # optimum-benchmark
     ];
+    open-telemetry = [
+      opentelemetry-api
+      opentelemetry-exporter-otlp
+      opentelemetry-sdk
+    ];
   });
 
   # Many tests require internet access.
@@ -226,6 +218,7 @@ buildPythonPackage (finalAttrs: {
     license = lib.licenses.asl20;
     platforms = lib.platforms.unix;
     maintainers = with lib.maintainers; [
+      GaetanLepage
       pashashocky
       happysalada
     ];