From f22d1cbdafb05ebe6f6dcb37cf456779cd98a89b Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Fri, 18 Nov 2022 12:01:27 +0100
Subject: [PATCH 01/19] First draft for a contrib test suite + test for timm
 contrib

---
 .github/workflows/contrib-tests.yml | 32 +++++++++++++++
 Makefile                            |  2 +-
 contrib/README.md                   | 26 ++++++++++++
 contrib/conftest.py                 | 61 +++++++++++++++++++++++++++++
 contrib/requirements.txt            |  6 +++
 contrib/test_timm.py                | 10 +++++
 6 files changed, 136 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/contrib-tests.yml
 create mode 100644 contrib/README.md
 create mode 100644 contrib/conftest.py
 create mode 100644 contrib/requirements.txt
 create mode 100644 contrib/test_timm.py

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
new file mode 100644
index 0000000000..607559ba99
--- /dev/null
+++ b/.github/workflows/contrib-tests.yml
@@ -0,0 +1,32 @@
+name: Contrib tests
+
+on: workflow_dispatch
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.7", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # Install huggingface_hub
+      - name: Install `huggingface_hub`
+        run: |
+          pip install --upgrade pip
+          pip install .
+
+      # Install downstream libraries
+      - name: Install downstream libraries
+        run: pip install -r contrib/requirements.txt
+
+      # Run tests
+      - name: Run tests
+        run: pytest contrib/ -n 4
diff --git a/Makefile b/Makefile
index 5ba1b48f22..5550598c63 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 .PHONY: quality style test
 
 
-check_dirs := tests src utils setup.py
+check_dirs := contrib src tests utils setup.py
 
 
 quality:
diff --git a/contrib/README.md b/contrib/README.md
new file mode 100644
index 0000000000..e641495ad8
--- /dev/null
+++ b/contrib/README.md
@@ -0,0 +1,26 @@
+# Contrib test suite
+
+The contrib folder contains simple end-to-end scripts to test integration of `huggingface_hub` in downstream libraries. The main goal is to proactively notice breaking changes and deprecation warnings.
+
+## Run contrib tests on CI
+
+Contrib tests can be [manually triggered in github](https://github.com/huggingface/huggingface_hub/actions) with the `Contrib tests` workflow.
+
+Tests are not run in the default test suite (for each PR) as this would slow down development process. The goal is to notice breaking changes, not to avoid them. In particular, it is interesting to trigger it before a release to make sure it will not cause too much friction.
+
+## Run contrib tests locally
+
+### Install dependencies
+
+```sh
+# Create a separate contrib environment
+python3 -m venv .venv_contrib
+source .venv_contrib/bin/activate
+
+# Install requirements
+pip install . # huggingface_hub
+pip install -r contrib/requirements.txt
+
+# Run tests !
+pytest contrib -n 4
+```
\ No newline at end of file
diff --git a/contrib/conftest.py b/contrib/conftest.py
new file mode 100644
index 0000000000..02a979900e
--- /dev/null
+++ b/contrib/conftest.py
@@ -0,0 +1,61 @@
+import time
+import uuid
+from typing import Generator
+
+import pytest
+
+from huggingface_hub import HfFolder, delete_repo
+
+
+@pytest.fixture(scope="session")
+def token() -> str:
+    # Not critical, only usable on the sandboxed CI instance.
+    return "hf_94wBhPGp6KrrTH3KDchhKpRxZwd6dmHWLL"
+
+
+@pytest.fixture(scope="session")
+def user() -> str:
+    return "__DUMMY_TRANSFORMERS_USER__"
+
+
+@pytest.fixture(autouse=True, scope="session")
+def login_as_dummy_user(token: str) -> Generator:
+    """Login with dummy user token on machine
+
+    Once all tests are completed, set back previous token."""
+    # Remove registered token
+    old_token = HfFolder().get_token()
+    HfFolder().save_token(token)
+
+    yield  # Run all tests
+
+    # Set back token once all tests have passed
+    if old_token is not None:
+        HfFolder().save_token(old_token)
+
+
+@pytest.fixture
+def repo_name(request) -> None:
+    """
+    Return a readable pseudo-unique repository name for tests.
+
+    Example: "repo-2fe93f-16599646671840"
+    """
+    prefix = request.module.__name__  # example: `test_timm`
+    id = uuid.uuid4().hex[:6]
+    ts = int(time.time() * 10e3)
+    return f"repo-{prefix}-{id}-{ts}"
+
+
+@pytest.fixture
+def cleanup_repo(user: str, repo_name: str) -> None:
+    """Delete the repo at the end of the tests.
+
+    TODO: Adapt to handle `repo_type` as well
+    """
+    yield  # run test
+    delete_repo(repo_id=f"{user}/{repo_name}")
+
+
+# ENDPOINT_PRODUCTION = "https://huggingface.co"
+# ENDPOINT_STAGING = "https://hub-ci.huggingface.co"
diff --git a/contrib/requirements.txt b/contrib/requirements.txt
new file mode 100644
index 0000000000..c73f25fc2b
--- /dev/null
+++ b/contrib/requirements.txt
@@ -0,0 +1,6 @@
+pytest
+pytest-env
+pytest-xdist
+
+# Timm
+git+https://github.com/rwightman/pytorch-image-models.git#egg=timm
\ No newline at end of file
diff --git a/contrib/test_timm.py b/contrib/test_timm.py
new file mode 100644
index 0000000000..d50a8061ec
--- /dev/null
+++ b/contrib/test_timm.py
@@ -0,0 +1,10 @@
+def test_push_to_hub(repo_name: str, cleanup_repo: None) -> None:
+    import timm
+
+    # Build a model 🔧
+    model = timm.create_model("resnet18", pretrained=True, num_classes=4)
+
+    # Push it to the 🤗 hub
+    timm.models.hub.push_to_hf_hub(
+        model, repo_name, model_config=dict(labels=["a", "b", "c", "d"])
+    )

From fc1da547297c75b63e23055bc40f0a6ac6b962dc Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Fri, 18 Nov 2022 12:02:55 +0100
Subject: [PATCH 02/19] run only Python 3.8

---
 .github/workflows/contrib-tests.yml | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index 607559ba99..ca05ba8baa 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -5,17 +5,13 @@ on: workflow_dispatch
 jobs:
   build:
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.7", "3.11"]
 
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: 3.8
 
       # Install huggingface_hub
       - name: Install `huggingface_hub`

From 64185aae99f46f9169f834b0c907c57c29f592ff Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Fri, 18 Nov 2022 12:11:54 +0100
Subject: [PATCH 03/19] remove commented code

---
 contrib/conftest.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/contrib/conftest.py b/contrib/conftest.py
index 02a979900e..f438cba61c 100644
--- a/contrib/conftest.py
+++ b/contrib/conftest.py
@@ -55,7 +55,3 @@ def cleanup_repo(user: str, repo_name: str) -> None:
     """
     yield  # run test
     delete_repo(repo_id=f"{user}/{repo_name}")
-
-
-# ENDPOINT_PRODUCTION = "https://huggingface.co"
-# ENDPOINT_STAGING = "https://hub-ci.huggingface.co"

From af45cdb55509fdf89b6fdcbcf00c460f0cda6628 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Mon, 21 Nov 2022 18:26:23 +0100
Subject: [PATCH 04/19] Run contrib tests in separate environments

---
 .github/workflows/contrib-tests.yml         | 31 +++++++++++++++------
 contrib/README.md                           | 17 +++++++----
 contrib/reqs_common.txt                     |  2 ++
 contrib/{requirements.txt => reqs_timm.txt} |  4 ---
 4 files changed, 37 insertions(+), 17 deletions(-)
 create mode 100644 contrib/reqs_common.txt
 rename contrib/{requirements.txt => reqs_timm.txt} (69%)

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index ca05ba8baa..96a18cca5a 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -1,10 +1,21 @@
 name: Contrib tests
 
-on: workflow_dispatch
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - ci_contrib_*
+      - 1190-rfc-add-contrib-test-suite
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        contrib: [
+          "timm",
+        ]
 
     steps:
       - uses: actions/checkout@v2
@@ -13,16 +24,20 @@ jobs:
         with:
           python-version: 3.8
 
-      # Install huggingface_hub
+      # Install pip
+      - name: Install pip
+          pip install --upgrade pip
+
+      # Install downstream library
+      - name: Install ${{ matrix.python-version }}
+        run: pip install -r contrib/reqs_${{ matrix.python-version }}.txt
+
+      # Install huggingface_hub as last from source code
       - name: Install `huggingface_hub`
         run: |
-          pip install --upgrade pip
+          pip uninstall huggingface_hub
           pip install .
 
-      # Install downstream libraries
-      - name: Install downstream libraries
-        run: pip install -r contrib/requirements.txt
-
       # Run tests
       - name: Run tests
-        run: pytest contrib/ -n 4
+        run: pytest contrib/${{ matrix.python-version }}
diff --git a/contrib/README.md b/contrib/README.md
index e641495ad8..9176ad95a5 100644
--- a/contrib/README.md
+++ b/contrib/README.md
@@ -10,17 +10,24 @@ Tests are not run in the default test suite (for each PR) as this would slow dow
 
 ## Run contrib tests locally
 
+Tests must be ran individually for each dependent library. Here is an example to run
+`timm` tests. Tests are separated to avoid conflicts between version dependencies.
+
 ### Install dependencies
 
 ```sh
 # Create a separate contrib environment
-python3 -m venv .venv_contrib
-source .venv_contrib/bin/activate
+python3 -m venv .venv_contrib_timm
+source .venv_contrib_timm/bin/activate
 
 # Install requirements
+pip install -r contrib/reqs_common.txt
+pip install -r contrib/reqs_timm.txt
 pip install . # huggingface_hub
-pip install -r contrib/requirements.txt
+```
+
+### Run tests !
 
-# Run tests !
-pytest contrib -n 4
+```
+pytest contrib/test_timm.py
 ```
\ No newline at end of file
diff --git a/contrib/reqs_common.txt b/contrib/reqs_common.txt
new file mode 100644
index 0000000000..93d33ed304
--- /dev/null
+++ b/contrib/reqs_common.txt
@@ -0,0 +1,2 @@
+pytest
+pytest-env
\ No newline at end of file
diff --git a/contrib/requirements.txt b/contrib/reqs_timm.txt
similarity index 69%
rename from contrib/requirements.txt
rename to contrib/reqs_timm.txt
index c73f25fc2b..8455ddbef6 100644
--- a/contrib/requirements.txt
+++ b/contrib/reqs_timm.txt
@@ -1,6 +1,2 @@
-pytest
-pytest-env
-pytest-xdist
-
 # Timm
 git+https://github.com/rwightman/pytorch-image-models.git#egg=timm
\ No newline at end of file

From 821aa04ae66e7aed47e8e4a66310b85918a3629a Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Mon, 21 Nov 2022 18:27:45 +0100
Subject: [PATCH 05/19] fix ci

---
 .github/workflows/contrib-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index 96a18cca5a..09883ce77a 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -26,7 +26,7 @@ jobs:
 
       # Install pip
       - name: Install pip
-          pip install --upgrade pip
+        run: pip install --upgrade pip
 
       # Install downstream library
       - name: Install ${{ matrix.python-version }}

From 2a0d7c6ea92b3deb2b7d6d0a8a15659376fc80ab Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Mon, 21 Nov 2022 18:33:49 +0100
Subject: [PATCH 06/19] fix ci again

---
 .github/workflows/contrib-tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index 09883ce77a..7e168be6be 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -29,8 +29,8 @@ jobs:
         run: pip install --upgrade pip
 
       # Install downstream library
-      - name: Install ${{ matrix.python-version }}
-        run: pip install -r contrib/reqs_${{ matrix.python-version }}.txt
+      - name: Install ${{ matrix.contrib }}
+        run: pip install -r contrib/reqs_${{ matrix.contrib }}.txt
 
       # Install huggingface_hub as last from source code
       - name: Install `huggingface_hub`
@@ -40,4 +40,4 @@ jobs:
 
       # Run tests
       - name: Run tests
-        run: pytest contrib/${{ matrix.python-version }}
+        run: pytest contrib/${{ matrix.contrib }}

From 339af7f711a4a0ad0e9b9a3c81a6a4358389df15 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Mon, 21 Nov 2022 18:42:56 +0100
Subject: [PATCH 07/19] and now ?

---
 .github/workflows/contrib-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index 7e168be6be..73af93b55a 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -36,7 +36,7 @@ jobs:
       - name: Install `huggingface_hub`
         run: |
           pip uninstall huggingface_hub
-          pip install .
+          pip install .[testing]
 
       # Run tests
       - name: Run tests

From 68c26b291521a626e70025992b24b225b339d274 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Mon, 21 Nov 2022 18:45:57 +0100
Subject: [PATCH 08/19] stupid me

---
 .github/workflows/contrib-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index 73af93b55a..93cf93b4a3 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -35,8 +35,8 @@ jobs:
       # Install huggingface_hub as last from source code
       - name: Install `huggingface_hub`
         run: |
-          pip uninstall huggingface_hub
-          pip install .[testing]
+          pip uninstall -y huggingface_hub
+          pip install .
 
       # Run tests
       - name: Run tests

From 1eb0a09119c84e106249562f88ed3893663d45a3 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Mon, 21 Nov 2022 18:50:23 +0100
Subject: [PATCH 09/19] this time ?

---
 .github/workflows/contrib-tests.yml            |  6 +++++-
 Makefile                                       | 11 ++++++++++-
 contrib/README.md                              | 18 +++++-------------
 contrib/{reqs_common.txt => requirements.txt}  |  0
 .../{reqs_timm.txt => timm/requirements.txt}   |  0
 contrib/{ => timm}/test_timm.py                |  0
 6 files changed, 20 insertions(+), 15 deletions(-)
 rename contrib/{reqs_common.txt => requirements.txt} (100%)
 rename contrib/{reqs_timm.txt => timm/requirements.txt} (100%)
 rename contrib/{ => timm}/test_timm.py (100%)

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index 93cf93b4a3..8b308c6158 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -28,9 +28,13 @@ jobs:
       - name: Install pip
         run: pip install --upgrade pip
 
+      # Install common dependencies
+      - name: Install common dependencies
+        run: pip install -r contrib/requirements.txt
+
       # Install downstream library
       - name: Install ${{ matrix.contrib }}
-        run: pip install -r contrib/reqs_${{ matrix.contrib }}.txt
+        run: pip install -r contrib/${{ matrix.contrib }}/requirements.txt
 
       # Install huggingface_hub as last from source code
       - name: Install `huggingface_hub`
diff --git a/Makefile b/Makefile
index 5550598c63..5eabc722cf 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: quality style test
+.PHONY: contrib quality style test
 
 
 check_dirs := contrib src tests utils setup.py
@@ -18,3 +18,12 @@ style:
 
 test:
 	pytest ./tests/
+
+contrib:
+	python3 -m venv contrib/timm/.venv
+	. contrib/timm/.venv/bin/activate
+	pip install -r contrib/requirements.txt
+	pip install -r contrib/timm/requirements.txt
+	pip uninstall -y huggingface_hub
+	pip install -e .
+	pytest contrib/timm
\ No newline at end of file
diff --git a/contrib/README.md b/contrib/README.md
index 9176ad95a5..2af3153f6f 100644
--- a/contrib/README.md
+++ b/contrib/README.md
@@ -13,21 +13,13 @@ Tests are not run in the default test suite (for each PR) as this would slow dow
 Tests must be ran individually for each dependent library. Here is an example to run
 `timm` tests. Tests are separated to avoid conflicts between version dependencies.
 
-### Install dependencies
+### Using make command
 
-```sh
-# Create a separate contrib environment
-python3 -m venv .venv_contrib_timm
-source .venv_contrib_timm/bin/activate
+The command will take care of installing dependencies in separated virtual envs and to
+run tests independently.
 
-# Install requirements
-pip install -r contrib/reqs_common.txt
-pip install -r contrib/reqs_timm.txt
-pip install . # huggingface_hub
-```
-
-### Run tests !
+TODO: accept multiple contrib libs
 
 ```
-pytest contrib/test_timm.py
+make contrib
 ```
\ No newline at end of file
diff --git a/contrib/reqs_common.txt b/contrib/requirements.txt
similarity index 100%
rename from contrib/reqs_common.txt
rename to contrib/requirements.txt
diff --git a/contrib/reqs_timm.txt b/contrib/timm/requirements.txt
similarity index 100%
rename from contrib/reqs_timm.txt
rename to contrib/timm/requirements.txt
diff --git a/contrib/test_timm.py b/contrib/timm/test_timm.py
similarity index 100%
rename from contrib/test_timm.py
rename to contrib/timm/test_timm.py

From d4a72e70a5146fac69cc5ddf63e9f33b0add5846 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Tue, 22 Nov 2022 10:58:14 +0100
Subject: [PATCH 10/19] Refactor how to run contrib tests locally

---
 Makefile          | 47 +++++++++++++++++++++++++++++++-------
 contrib/README.md | 58 ++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 91 insertions(+), 14 deletions(-)

diff --git a/Makefile b/Makefile
index 5eabc722cf..3b89a8f1d8 100644
--- a/Makefile
+++ b/Makefile
@@ -19,11 +19,42 @@ style:
 test:
 	pytest ./tests/
 
-contrib:
-	python3 -m venv contrib/timm/.venv
-	. contrib/timm/.venv/bin/activate
-	pip install -r contrib/requirements.txt
-	pip install -r contrib/timm/requirements.txt
-	pip uninstall -y huggingface_hub
-	pip install -e .
-	pytest contrib/timm
\ No newline at end of file
+# Taken from https://stackoverflow.com/a/12110773
+# Commands:
+#	make contrib_setup_timm : setup tests for timm
+#	make contrib_test_timm  : run tests for timm
+#	make contrib_timm       : setup and run tests for timm
+#	make contrib_clear_timm : delete timm virtual env
+#
+#	make contrib_setup      : setup ALL tests
+#	make contrib_test       : run ALL tests
+#	make contrib            : setup and run ALL tests
+#	make contrib_clear      : delete all virtual envs
+# Use -j4 flag to run jobs in parallel.
+CONTRIB_LIBS := timm
+CONTRIB_JOBS := $(addprefix contrib_,${CONTRIB_LIBS})
+CONTRIB_CLEAR_JOBS := $(addprefix contrib_clear_,${CONTRIB_LIBS})
+CONTRIB_SETUP_JOBS := $(addprefix contrib_setup_,${CONTRIB_LIBS})
+CONTRIB_TEST_JOBS := $(addprefix contrib_test_,${CONTRIB_LIBS})
+
+contrib_clear_%:
+	rm -rf contrib/$*/.venv
+
+contrib_setup_%:
+	python3 -m venv contrib/$*/.venv
+	./contrib/$*/.venv/bin/pip install -r contrib/requirements.txt
+	./contrib/$*/.venv/bin/pip install -r contrib/$*/requirements.txt
+	./contrib/$*/.venv/bin/pip uninstall -y huggingface_hub
+	./contrib/$*/.venv/bin/pip install -e .
+
+contrib_test_%:
+	./contrib/$*/.venv/bin/python -m pytest contrib/$*
+
+contrib_%:
+	make contrib_setup_$*
+	make contrib_test_$*
+
+contrib: ${CONTRIB_JOBS};
+contrib_clear: ${CONTRIB_CLEAR_JOBS}; echo "Successful contrib tests."
+contrib_setup: ${CONTRIB_SETUP_JOBS}; echo "Successful contrib setup."
+contrib_test: ${CONTRIB_TEST_JOBS}; echo "Successful contrib tests."
\ No newline at end of file
diff --git a/contrib/README.md b/contrib/README.md
index 2af3153f6f..dcd757b611 100644
--- a/contrib/README.md
+++ b/contrib/README.md
@@ -2,6 +2,15 @@
 
 The contrib folder contains simple end-to-end scripts to test integration of `huggingface_hub` in downstream libraries. The main goal is to proactively notice breaking changes and deprecation warnings.
 
+## Add tests for a new library
+
+To add another contrib lib, one must:
+1. Create a subfolder with the lib name. Example: `./contrib/transformers`
+2. Create a `requirements.txt` file specific to this lib. Example `./contrib/transformers/requirements.txt`
+3. Implements tests for this lib. Example: `./contrib/transformers/test_push_to_hub.py`
+4. Edit `makefile` to add the lib to `CONTRIB_LIBS` variable. Example: `CONTRIB_LIBS := timm transformers`
+5. Edit `.github/workflows/contrib-tests.yml` to add the lib to `matrix.contrib` list. Example: `contrib: ["timm", "transformers"]`
+
 ## Run contrib tests on CI
 
 Contrib tests can be [manually triggered in github](https://github.com/huggingface/huggingface_hub/actions) with the `Contrib tests` workflow.
@@ -13,13 +22,50 @@ Tests are not run in the default test suite (for each PR) as this would slow dow
 Tests must be ran individually for each dependent library. Here is an example to run
 `timm` tests. Tests are separated to avoid conflicts between version dependencies.
 
-### Using make command
+### Run all contrib tests
+
+Before running tests, a virtual env must be setup for each contrib library. To do so, run:
+
+```sh
+# Run setup in parallel to save time 
+make contrib_setup -j4
+```
+
+Then tests can be run
+
+```sh
+# Optional: -j4 to run in parallel. Output will be messy in that case.
+make contrib_test -j4
+```
+
+Optionally, it is possible to setup and run all tests in a single command. However this
+take more time as you don't need to setup the venv each time you run tests.
+
+```sh
+make contrib -j4
+```
+
+Finally, it is possible to delete all virtual envs to get a fresh start for contrib tests.
+After running this command, `contrib_setup` will have to re-download/re-install all dependencies.
+
+```
+make contrib_clear
+```
+
+### Run contrib tests for a single lib
+
+Instead of running tests for all contrib libraries, you can run a specific lib:
+
+```sh
+# Setup timm tests
+make contrib_setup_timm
 
-The command will take care of installing dependencies in separated virtual envs and to
-run tests independently.
+# Run timm tests
+make contrib_test_timm
 
-TODO: accept multiple contrib libs
+# (or) Setup and run timm tests at once
+make contrib_timm
 
+# Delete timm virtualenv if corrupted
+make contrib_clear_timm
 ```
-make contrib
-```
\ No newline at end of file

From aa1ffad73745b81433f0b11978c9881af6a026d9 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Tue, 22 Nov 2022 12:14:41 +0100
Subject: [PATCH 11/19] add tests for sentence_transformers

---
 .github/workflows/contrib-tests.yml           |  1 +
 Makefile                                      |  2 +-
 contrib/__init__.py                           |  0
 contrib/sentence_transformers/__init__.py     |  0
 .../sentence_transformers/requirements.txt    |  1 +
 .../test_sentence_transformers.py             | 29 ++++++++++
 contrib/timm/__init__.py                      |  0
 contrib/timm/test_timm.py                     |  5 +-
 contrib/utils.py                              | 58 +++++++++++++++++++
 9 files changed, 93 insertions(+), 3 deletions(-)
 create mode 100644 contrib/__init__.py
 create mode 100644 contrib/sentence_transformers/__init__.py
 create mode 100644 contrib/sentence_transformers/requirements.txt
 create mode 100644 contrib/sentence_transformers/test_sentence_transformers.py
 create mode 100644 contrib/timm/__init__.py
 create mode 100644 contrib/utils.py

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index 8b308c6158..4c3bfd8998 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -14,6 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         contrib: [
+          "sentence_transformers",
           "timm",
         ]
 
diff --git a/Makefile b/Makefile
index 3b89a8f1d8..e5576580b7 100644
--- a/Makefile
+++ b/Makefile
@@ -31,7 +31,7 @@ test:
 #	make contrib            : setup and run ALL tests
 #	make contrib_clear      : delete all virtual envs
 # Use -j4 flag to run jobs in parallel.
-CONTRIB_LIBS := timm
+CONTRIB_LIBS := sentence_transformers timm
 CONTRIB_JOBS := $(addprefix contrib_,${CONTRIB_LIBS})
 CONTRIB_CLEAR_JOBS := $(addprefix contrib_clear_,${CONTRIB_LIBS})
 CONTRIB_SETUP_JOBS := $(addprefix contrib_setup_,${CONTRIB_LIBS})
diff --git a/contrib/__init__.py b/contrib/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/contrib/sentence_transformers/__init__.py b/contrib/sentence_transformers/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/contrib/sentence_transformers/requirements.txt b/contrib/sentence_transformers/requirements.txt
new file mode 100644
index 0000000000..128abfc4be
--- /dev/null
+++ b/contrib/sentence_transformers/requirements.txt
@@ -0,0 +1 @@
+git+https://github.com/UKPLab/sentence-transformers.git#egg=sentence-transformers
\ No newline at end of file
diff --git a/contrib/sentence_transformers/test_sentence_transformers.py b/contrib/sentence_transformers/test_sentence_transformers.py
new file mode 100644
index 0000000000..90751d26d6
--- /dev/null
+++ b/contrib/sentence_transformers/test_sentence_transformers.py
@@ -0,0 +1,29 @@
+from sentence_transformers import SentenceTransformer, util
+
+import pytest
+from ..utils import production_endpoint
+
+
+@pytest.fixture(scope="module")
+def multi_qa_model() -> SentenceTransformer:
+    with production_endpoint():
+        return SentenceTransformer("multi-qa-MiniLM-L6-cos-v1")
+
+
+def test_from_pretrained(multi_qa_model: SentenceTransformer) -> None:
+    # Example taken from https://www.sbert.net/docs/hugging_face.html#using-hugging-face-models.
+    query_embedding = multi_qa_model.encode("How big is London")
+    passage_embedding = multi_qa_model.encode(
+        [
+            "London has 9,787,426 inhabitants at the 2011 census",
+            "London is known for its financial district",
+        ]
+    )
+    print("Similarity:", util.dot_score(query_embedding, passage_embedding))
+
+
+@pytest.mark.xfail(reason="Production endpoint is hardcoded in sentence_transformers when pushing to Hub.")
+def test_push_to_hub(
+    multi_qa_model: SentenceTransformer, repo_name: str, cleanup_repo: None
+) -> None:
+    multi_qa_model.save_to_hub(repo_name)
diff --git a/contrib/timm/__init__.py b/contrib/timm/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/contrib/timm/test_timm.py b/contrib/timm/test_timm.py
index d50a8061ec..9d0cd69f10 100644
--- a/contrib/timm/test_timm.py
+++ b/contrib/timm/test_timm.py
@@ -1,6 +1,7 @@
-def test_push_to_hub(repo_name: str, cleanup_repo: None) -> None:
-    import timm
+import timm
+
 
+def test_push_to_hub(repo_name: str, cleanup_repo: None) -> None:
     # Build a model 🔧
     model = timm.create_model("resnet18", pretrained=True, num_classes=4)
 
diff --git a/contrib/utils.py b/contrib/utils.py
new file mode 100644
index 0000000000..2c3c45cb34
--- /dev/null
+++ b/contrib/utils.py
@@ -0,0 +1,58 @@
+import contextlib
+from typing import Generator
+from unittest.mock import patch
+
+
+@contextlib.contextmanager
+def production_endpoint() -> Generator:
+    """Patch huggingface_hub to connect to production server in a context manager.
+
+    Ugly way to patch all constants at once.
+    TODO: refactor when https://github.com/huggingface/huggingface_hub/issues/1172 is fixed.
+
+    Example:
+    ```py
+    def test_push_to_hub():
+        # Pull from production Hub
+        with production_endpoint():
+            model = ...from_pretrained("modelname")
+
+        # Push to staging Hub
+        model.push_to_hub()
+    ```
+    """
+    PROD_ENDPOINT = "https://huggingface.co"
+    ENDPOINT_TARGETS = [
+        "huggingface_hub.constants",
+        "huggingface_hub._commit_api",
+        "huggingface_hub.hf_api",
+        "huggingface_hub.lfs",
+        "huggingface_hub.commands.user",
+        "huggingface_hub.utils._git_credential",
+    ]
+
+    PROD_URL_TEMPLATE = PROD_ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}"
+    URL_TEMPLATE_TARGETS = [
+        "huggingface_hub.constants",
+        "huggingface_hub.file_download",
+    ]
+
+    from huggingface_hub.hf_api import api
+    patchers = (
+        [patch(target + ".ENDPOINT", PROD_ENDPOINT) for target in ENDPOINT_TARGETS]
+        + [
+            patch(target + ".HUGGINGFACE_CO_URL_TEMPLATE", PROD_URL_TEMPLATE)
+            for target in URL_TEMPLATE_TARGETS
+        ]
+        + [patch.object(api, "endpoint", PROD_URL_TEMPLATE)]
+    )
+
+    # Start all patches
+    for patcher in patchers:
+        patcher.start()
+
+    yield
+
+    # Stop all patches
+    for patcher in patchers:
+        patcher.stop()

From 930c29ebaafbd9594b78bc45c55ef3b6f462af31 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Tue, 22 Nov 2022 13:19:46 +0100
Subject: [PATCH 12/19] amke style

---
 .../sentence_transformers/test_sentence_transformers.py  | 9 +++++++--
 contrib/utils.py                                         | 1 +
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/contrib/sentence_transformers/test_sentence_transformers.py b/contrib/sentence_transformers/test_sentence_transformers.py
index 90751d26d6..62d1593b5f 100644
--- a/contrib/sentence_transformers/test_sentence_transformers.py
+++ b/contrib/sentence_transformers/test_sentence_transformers.py
@@ -1,6 +1,7 @@
+import pytest
+
 from sentence_transformers import SentenceTransformer, util
 
-import pytest
 from ..utils import production_endpoint
 
 
@@ -22,7 +23,11 @@ def test_from_pretrained(multi_qa_model: SentenceTransformer) -> None:
     print("Similarity:", util.dot_score(query_embedding, passage_embedding))
 
 
-@pytest.mark.xfail(reason="Production endpoint is hardcoded in sentence_transformers when pushing to Hub.")
+@pytest.mark.xfail(
+    reason=(
+        "Production endpoint is hardcoded in sentence_transformers when pushing to Hub."
+    )
+)
 def test_push_to_hub(
     multi_qa_model: SentenceTransformer, repo_name: str, cleanup_repo: None
 ) -> None:
diff --git a/contrib/utils.py b/contrib/utils.py
index 2c3c45cb34..396c87e206 100644
--- a/contrib/utils.py
+++ b/contrib/utils.py
@@ -38,6 +38,7 @@ def test_push_to_hub():
     ]
 
     from huggingface_hub.hf_api import api
+
     patchers = (
         [patch(target + ".ENDPOINT", PROD_ENDPOINT) for target in ENDPOINT_TARGETS]
         + [

From 625768d08f46166026821d652c380343b8639f66 Mon Sep 17 00:00:00 2001
From: Lucain <lucainp@gmail.com>
Date: Wed, 23 Nov 2022 09:10:40 +0100
Subject: [PATCH 13/19] Update contrib/README.md

Co-authored-by: Omar Sanseviero <osanseviero@gmail.com>
---
 contrib/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/README.md b/contrib/README.md
index dcd757b611..b24d64e7ee 100644
--- a/contrib/README.md
+++ b/contrib/README.md
@@ -13,7 +13,7 @@ To add another contrib lib, one must:
 
 ## Run contrib tests on CI
 
-Contrib tests can be [manually triggered in github](https://github.com/huggingface/huggingface_hub/actions) with the `Contrib tests` workflow.
+Contrib tests can be [manually triggered in GitHub](https://github.com/huggingface/huggingface_hub/actions) with the `Contrib tests` workflow.
 
 Tests are not run in the default test suite (for each PR) as this would slow down development process. The goal is to notice breaking changes, not to avoid them. In particular, it is interesting to trigger it before a release to make sure it will not cause too much friction.
 

From 12aa4bac343ebf2f55e0f83cc66e8fd685a29cd3 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Thu, 24 Nov 2022 15:51:06 +0100
Subject: [PATCH 14/19] ADapt timm tests

---
 contrib/timm/test_timm.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/contrib/timm/test_timm.py b/contrib/timm/test_timm.py
index 9d0cd69f10..f8e0210af4 100644
--- a/contrib/timm/test_timm.py
+++ b/contrib/timm/test_timm.py
@@ -1,11 +1,19 @@
 import timm
 
+from ..utils import production_endpoint
 
-def test_push_to_hub(repo_name: str, cleanup_repo: None) -> None:
-    # Build a model 🔧
-    model = timm.create_model("resnet18", pretrained=True, num_classes=4)
 
-    # Push it to the 🤗 hub
-    timm.models.hub.push_to_hf_hub(
-        model, repo_name, model_config=dict(labels=["a", "b", "c", "d"])
-    )
+MODEL_ID = "nateraw/timm-resnet50-beans"
+
+
+def test_load_and_push_to_hub(repo_name: str, cleanup_repo: None) -> None:
+    # Test load only config
+    with production_endpoint():
+        _ = timm.models.hub.load_model_config_from_hf(MODEL_ID)
+
+    # Load entire model from Hub
+    with production_endpoint():
+        model = timm.create_model("hf_hub:" + MODEL_ID, pretrained=True)
+
+    # Push model to Hub
+    timm.models.hub.push_to_hf_hub(model, repo_name)

From 173aff85bd26254a9a2d0d2cbc14f0d7c1f355f0 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Fri, 25 Nov 2022 13:19:27 +0100
Subject: [PATCH 15/19] Include feedback form osanseviero

---
 .github/workflows/contrib-tests.yml |  3 +--
 contrib/timm/test_timm.py           | 13 +++++++------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index 4c3bfd8998..2bbe926bec 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -5,7 +5,6 @@ on:
   push:
     branches:
       - ci_contrib_*
-      - 1190-rfc-add-contrib-test-suite
 
 jobs:
   build:
@@ -33,7 +32,7 @@ jobs:
       - name: Install common dependencies
         run: pip install -r contrib/requirements.txt
 
-      # Install downstream library
+      # Install downstream library and its specific dependencies
       - name: Install ${{ matrix.contrib }}
         run: pip install -r contrib/${{ matrix.contrib }}/requirements.txt
 
diff --git a/contrib/timm/test_timm.py b/contrib/timm/test_timm.py
index f8e0210af4..85e65d8dde 100644
--- a/contrib/timm/test_timm.py
+++ b/contrib/timm/test_timm.py
@@ -6,14 +6,15 @@
 MODEL_ID = "nateraw/timm-resnet50-beans"
 
 
-def test_load_and_push_to_hub(repo_name: str, cleanup_repo: None) -> None:
+@production_endpoint()
+def test_load_from_hub() -> None:
     # Test load only config
-    with production_endpoint():
-        _ = timm.models.hub.load_model_config_from_hf(MODEL_ID)
+    _ = timm.models.hub.load_model_config_from_hf(MODEL_ID)
 
     # Load entire model from Hub
-    with production_endpoint():
-        model = timm.create_model("hf_hub:" + MODEL_ID, pretrained=True)
+    _ = timm.create_model("hf_hub:" + MODEL_ID, pretrained=True)
 
-    # Push model to Hub
+
+def test_push_to_hub(repo_name: str, cleanup_repo: None) -> None:
+    model = timm.create_model("resnet18")
     timm.models.hub.push_to_hf_hub(model, repo_name)

From f20ab77919533fa1254d4e89df3884956644c259 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Mon, 28 Nov 2022 16:03:12 +0100
Subject: [PATCH 16/19] script to check contrib list is accurate

---
 .github/workflows/contrib-tests.yml |   2 +-
 Makefile                            |   4 +-
 contrib/README.md                   |   3 +-
 utils/check_contrib_list.py         | 114 ++++++++++++++++++++++++++++
 utils/check_static_imports.py       |  12 +--
 5 files changed, 125 insertions(+), 10 deletions(-)
 create mode 100644 utils/check_contrib_list.py

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index 2bbe926bec..4ffd60715c 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -13,8 +13,8 @@ jobs:
       fail-fast: false
       matrix:
         contrib: [
-          "sentence_transformers",
           "timm",
+          "sentence_transformers",
         ]
 
     steps:
diff --git a/Makefile b/Makefile
index e5576580b7..e65e4567a7 100644
--- a/Makefile
+++ b/Makefile
@@ -9,12 +9,14 @@ quality:
 	isort --check-only $(check_dirs)
 	flake8 $(check_dirs)
 	mypy src
+	python utils/check_contrib_list.py
 	python utils/check_static_imports.py
 
 style:
 	black $(check_dirs)
 	isort $(check_dirs)
-	python utils/check_static_imports.py --update-file
+	python utils/check_contrib_list.py --update
+	python utils/check_static_imports.py --update
 
 test:
 	pytest ./tests/
diff --git a/contrib/README.md b/contrib/README.md
index b24d64e7ee..2b051bc341 100644
--- a/contrib/README.md
+++ b/contrib/README.md
@@ -8,8 +8,7 @@ To add another contrib lib, one must:
 1. Create a subfolder with the lib name. Example: `./contrib/transformers`
 2. Create a `requirements.txt` file specific to this lib. Example `./contrib/transformers/requirements.txt`
 3. Implements tests for this lib. Example: `./contrib/transformers/test_push_to_hub.py`
-4. Edit `makefile` to add the lib to `CONTRIB_LIBS` variable. Example: `CONTRIB_LIBS := timm transformers`
-5. Edit `.github/workflows/contrib-tests.yml` to add the lib to `matrix.contrib` list. Example: `contrib: ["timm", "transformers"]`
+4. Run `make style`. This will edit both `makefile` and `.github/workflows/contrib-tests.yml` to add the lib to list of libs to test. Make sure changes are accurate before committing.
 
 ## Run contrib tests on CI
 
diff --git a/utils/check_contrib_list.py b/utils/check_contrib_list.py
new file mode 100644
index 0000000000..5e82e5c273
--- /dev/null
+++ b/utils/check_contrib_list.py
@@ -0,0 +1,114 @@
+# coding=utf-8
+# Copyright 2022-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains a tool to list contrib test suites automatically."""
+import argparse
+import re
+from pathlib import Path
+from typing import NoReturn
+
+
+ROOT_DIR = Path(__file__).parent.parent
+CONTRIB_PATH = ROOT_DIR / "contrib"
+MAKEFILE_PATH = ROOT_DIR / "Makefile"
+WORKFLOW_PATH = ROOT_DIR / ".github" / "workflows" / "contrib-tests.yml"
+
+MAKEFILE_REGEX = re.compile(r"^CONTRIB_LIBS := .*$", flags=re.MULTILINE)
+WORKFLOW_REGEX = re.compile(
+    r"""
+    # First: match "contrib: ["
+    (?P<before>^\s{8}contrib:\s\[\n)
+    # Match list of libs
+    (\s{10}\".*\",\n)*
+    # Finally: match trailing "]"
+    (?P<after>^\s{8}\])
+    """,
+    flags=re.MULTILINE | re.VERBOSE,
+)
+
+
+def check_contrib_list(update: bool) -> NoReturn:
+    """List `contrib` test suites.
+
+    Make sure `Makefile` and `.github/workflows/contrib-tests.yml` are consistent with
+    the list."""
+    # List contrib test suites
+    contrib_list = sorted(
+        path.name
+        for path in CONTRIB_PATH.glob("*")
+        if path.is_dir() and not path.name.startswith("_")
+    )
+
+    # Check Makefile is consistent with list
+    makefile_content = MAKEFILE_PATH.read_text()
+    makefile_expected_content = MAKEFILE_REGEX.sub(
+        f"CONTRIB_LIBS := {' '.join(contrib_list)}", makefile_content
+    )
+
+    # Check workflow is consistent with list
+    workflow_content = WORKFLOW_PATH.read_text()
+    _substitute = "\n".join(f'{" "*10}"{lib}",' for lib in contrib_list)
+    workflow_content_expected = WORKFLOW_REGEX.sub(
+        rf"\g<before>{_substitute}\n\g<after>", workflow_content
+    )
+
+    #
+    failed = False
+    if makefile_content != makefile_expected_content:
+        if update:
+            print(
+                "✅ Contrib libs have been updated in `Makefile`."
+                "\n   Please make sure the changes are accurate and commit them."
+            )
+            MAKEFILE_PATH.write_text(makefile_expected_content)
+        else:
+            print(
+                "❌ Expected content mismatch in `Makefile`.\n   It is most likely that"
+                " you added a contrib test and did not update the Makefile.\n   Please"
+                " run `make style` or `python utils/check_contrib_list.py --update`."
+            )
+            failed = True
+
+    if workflow_content != workflow_content_expected:
+        if update:
+            print(
+                f"✅ Contrib libs have been updated in `{WORKFLOW_PATH}`."
+                "\n   Please make sure the changes are accurate and commit them."
+            )
+            MAKEFILE_PATH.write_text(makefile_expected_content)
+        else:
+            print(
+                f"❌ Expected content mismatch in `{WORKFLOW_PATH}`.\n   It is most"
+                " likely that you added a contrib test and did not update the github"
+                " workflow file.\n   Please run `make style` or `python"
+                " utils/check_contrib_list.py --update`."
+            )
+            failed = True
+
+    if failed:
+        exit(1)
+    print("✅ All good! (contrib list)")
+    exit(0)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--update",
+        action="store_true",
+        help="Whether to fix Makefile and github workflow if a new lib is detected.",
+    )
+    args = parser.parse_args()
+
+    check_contrib_list(update=args.update)
diff --git a/utils/check_static_imports.py b/utils/check_static_imports.py
index 5107ad47d5..2d9a10e3bf 100644
--- a/utils/check_static_imports.py
+++ b/utils/check_static_imports.py
@@ -29,7 +29,7 @@
 SUBMOD_ATTRS_PATTERN = re.compile("_SUBMOD_ATTRS = {[^}]+}")  # match the all dict
 
 
-def check_static_imports(update_file: bool) -> NoReturn:
+def check_static_imports(update: bool) -> NoReturn:
     """Check all imports are made twice (1 in lazy-loading and 1 in static checks).
 
     For more explanations, see `./src/huggingface_hub/__init__.py`.
@@ -85,7 +85,7 @@ def check_static_imports(update_file: bool) -> NoReturn:
     # If expected `__init__.py` content is different, test fails. If '--update-init-file'
     # is used, `__init__.py` file is updated before the test fails.
     if init_content != expected_init_content:
-        if update_file:
+        if update:
             with INIT_FILE_PATH.open("w") as f:
                 f.write(expected_init_content)
 
@@ -100,18 +100,18 @@ def check_static_imports(update_file: bool) -> NoReturn:
                 " `./src/huggingface_hub/__init__.py`.\n   It is most likely that you"
                 " added a module/function to `_SUBMOD_ATTRS` and did not update the"
                 " 'static import'-part.\n   Please run `make style` or `python"
-                " utils/check_static_imports.py --update-file`."
+                " utils/check_static_imports.py --update`."
             )
             exit(1)
 
-    print("✅ All good!")
+    print("✅ All good! (static imports)")
     exit(0)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--update-file",
+        "--update",
         action="store_true",
         help=(
             "Whether to fix `./src/huggingface_hub/__init__.py` if a change is"
@@ -120,4 +120,4 @@ def check_static_imports(update_file: bool) -> NoReturn:
     )
     args = parser.parse_args()
 
-    check_static_imports(update_file=args.update_file)
+    check_static_imports(update=args.update)

From d5949fab6872348e5bc39004a36631f7e08b092b Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Mon, 28 Nov 2022 16:09:00 +0100
Subject: [PATCH 17/19] Use [testing] requirements as contrib common
 dependencies

---
 .github/workflows/contrib-tests.yml | 10 ++++------
 Makefile                            |  3 +--
 contrib/requirements.txt            |  2 --
 3 files changed, 5 insertions(+), 10 deletions(-)
 delete mode 100644 contrib/requirements.txt

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index 4ffd60715c..7e70c5f58e 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -2,6 +2,8 @@ name: Contrib tests
 
 on:
   workflow_dispatch:
+  schedule:
+  - cron:  '0 0 * * 6' # Run once a week, Saturday midnight
   push:
     branches:
       - ci_contrib_*
@@ -28,19 +30,15 @@ jobs:
       - name: Install pip
         run: pip install --upgrade pip
 
-      # Install common dependencies
-      - name: Install common dependencies
-        run: pip install -r contrib/requirements.txt
-
       # Install downstream library and its specific dependencies
       - name: Install ${{ matrix.contrib }}
         run: pip install -r contrib/${{ matrix.contrib }}/requirements.txt
 
-      # Install huggingface_hub as last from source code
+      # Install huggingface_hub from source code + testing extras
       - name: Install `huggingface_hub`
         run: |
           pip uninstall -y huggingface_hub
-          pip install .
+          pip install .[testing]
 
       # Run tests
       - name: Run tests
diff --git a/Makefile b/Makefile
index e65e4567a7..9da02e4346 100644
--- a/Makefile
+++ b/Makefile
@@ -44,10 +44,9 @@ contrib_clear_%:
 
 contrib_setup_%:
 	python3 -m venv contrib/$*/.venv
-	./contrib/$*/.venv/bin/pip install -r contrib/requirements.txt
 	./contrib/$*/.venv/bin/pip install -r contrib/$*/requirements.txt
 	./contrib/$*/.venv/bin/pip uninstall -y huggingface_hub
-	./contrib/$*/.venv/bin/pip install -e .
+	./contrib/$*/.venv/bin/pip install -e .[testing]
 
 contrib_test_%:
 	./contrib/$*/.venv/bin/python -m pytest contrib/$*
diff --git a/contrib/requirements.txt b/contrib/requirements.txt
deleted file mode 100644
index 93d33ed304..0000000000
--- a/contrib/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-pytest
-pytest-env
\ No newline at end of file

From 6cb96e7658c72922dfb5cf9b7b7773e0df5fcf53 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Mon, 28 Nov 2022 16:11:27 +0100
Subject: [PATCH 18/19] add check_contrib_list in github workflow

---
 .github/workflows/python-quality.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/python-quality.yml b/.github/workflows/python-quality.yml
index 7586153929..69a1b58729 100644
--- a/.github/workflows/python-quality.yml
+++ b/.github/workflows/python-quality.yml
@@ -30,6 +30,7 @@ jobs:
       - run: black --check tests src
       - run: isort --check-only tests src
       - run: flake8 tests src
+      - run: python utils/check_contrib_list.py
       - run: python utils/check_static_imports.py
 
       # Run type checking at least on huggingface_hub root file to check all modules

From b29907741525d2c05b125319ad254fe9d72592d5 Mon Sep 17 00:00:00 2001
From: Lucain Pouget <lucainp@gmail.com>
Date: Mon, 28 Nov 2022 16:20:36 +0100
Subject: [PATCH 19/19] code qualiry

---
 .github/workflows/contrib-tests.yml | 2 +-
 utils/check_contrib_list.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml
index 7e70c5f58e..22d7762bc8 100644
--- a/.github/workflows/contrib-tests.yml
+++ b/.github/workflows/contrib-tests.yml
@@ -15,8 +15,8 @@ jobs:
       fail-fast: false
       matrix:
         contrib: [
-          "timm",
           "sentence_transformers",
+          "timm",
         ]
 
     steps:
diff --git a/utils/check_contrib_list.py b/utils/check_contrib_list.py
index 5e82e5c273..ea3d0a73f1 100644
--- a/utils/check_contrib_list.py
+++ b/utils/check_contrib_list.py
@@ -86,7 +86,7 @@ def check_contrib_list(update: bool) -> NoReturn:
                 f"✅ Contrib libs have been updated in `{WORKFLOW_PATH}`."
                 "\n   Please make sure the changes are accurate and commit them."
             )
-            MAKEFILE_PATH.write_text(makefile_expected_content)
+            WORKFLOW_PATH.write_text(workflow_content_expected)
         else:
             print(
                 f"❌ Expected content mismatch in `{WORKFLOW_PATH}`.\n   It is most"