Test datasets@main for datasets-2.19.0 release

.circleci/create_circleci_config.py

@@ -128,6 +128,7 @@ def to_dict(self):
         steps.extend([{"run": l} for l in self.install_steps])
         steps.extend([{"run": 'pip install "fsspec>=2023.5.0,<2023.10.0"'}])
         steps.extend([{"run": "pip install pytest-subtests"}])
+        steps.append({"run": {"name": "Install `datasets@main`", "command": 'pip uninstall datasets -y && pip install "datasets @ git+https://github.com/huggingface/datasets@main#egg=datasets"'}})
         steps.append({"run": {"name": "Show installed libraries and their versions", "command": "pip freeze | tee installed.txt"}})
         steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}})
 

setup.py

@@ -102,7 +102,7 @@
     "codecarbon==1.2.0",
     "cookiecutter==1.7.3",
     "dataclasses",
-    "datasets!=2.5.0",
+    "datasets!=2.5.0",  # pinned to datasets@main in create_circleci_config.py
     "decord==0.6.0",
     "deepspeed>=0.9.3",
     "diffusers",