[RFC] Contrib test suite + tests for timm and sentence_transformers

.github/workflows/contrib-tests.yml

@@ -0,0 +1,28 @@
+name: Contrib tests
+
+on: workflow_dispatch
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      # Install huggingface_hub
+      - name: Install `huggingface_hub`
+        run: |
+          pip install --upgrade pip
+          pip install .
+
+      # Install downstream libraries
+      - name: Install downstream libraries
+        run: pip install -r contrib/requirements.txt
+
+      # Run tests
+      - name: Run tests
+        run: pytest contrib/ -n 4

Makefile

@@ -1,7 +1,7 @@
 .PHONY: quality style test
 
 
-check_dirs := tests src utils setup.py
+check_dirs := contrib src tests utils setup.py
 
 
 quality:

contrib/README.md

@@ -0,0 +1,26 @@
+# Contrib test suite
+
+The contrib folder contains simple end-to-end scripts to test integration of `huggingface_hub` in downstream libraries. The main goal is to proactively notice breaking changes and deprecation warnings.
+
+## Run contrib tests on CI
+
+Contrib tests can be [manually triggered in github](https://github.com/huggingface/huggingface_hub/actions) with the `Contrib tests` workflow.
+
+Tests are not run in the default test suite (for each PR) as this would slow down development process. The goal is to notice breaking changes, not to avoid them. In particular, it is interesting to trigger it before a release to make sure it will not cause too much friction.
+
+## Run contrib tests locally
+
+### Install dependencies
+
+```sh
+# Create a separate contrib environment
+python3 -m venv .venv_contrib
+source .venv_contrib/bin/activate
+
+# Install requirements
+pip install . # huggingface_hub
+pip install -r contrib/requirements.txt
+
+# Run tests !
+pytest contrib -n 4
+```

contrib/conftest.py

@@ -0,0 +1,57 @@
+import time
+import uuid
+from typing import Generator
+
+import pytest
+
+from huggingface_hub import HfFolder, delete_repo
+
+
+@pytest.fixture(scope="session")
+def token() -> str:
+    # Not critical, only usable on the sandboxed CI instance.
+    return "hf_94wBhPGp6KrrTH3KDchhKpRxZwd6dmHWLL"
+
+
+@pytest.fixture(scope="session")
+def user() -> str:
+    return "__DUMMY_TRANSFORMERS_USER__"
+
+
+@pytest.fixture(autouse=True, scope="session")
+def login_as_dummy_user(token: str) -> Generator:
+    """Login with dummy user token on machine
+
+    Once all tests are completed, set back previous token."""
+    # Remove registered token
+    old_token = HfFolder().get_token()
+    HfFolder().save_token(token)
+
+    yield  # Run all tests
+
+    # Set back token once all tests have passed
+    if old_token is not None:
+        HfFolder().save_token(old_token)
+
+
+@pytest.fixture
+def repo_name(request) -> None:
+    """
+    Return a readable pseudo-unique repository name for tests.
+
+    Example: "repo-2fe93f-16599646671840"
+    """
+    prefix = request.module.__name__  # example: `test_timm`
+    id = uuid.uuid4().hex[:6]
+    ts = int(time.time() * 10e3)
+    return f"repo-{prefix}-{id}-{ts}"
+
+
+@pytest.fixture
+def cleanup_repo(user: str, repo_name: str) -> None:
+    """Delete the repo at the end of the tests.
+
+    TODO: Adapt to handle `repo_type` as well
+    """
+    yield  # run test
+    delete_repo(repo_id=f"{user}/{repo_name}")

contrib/requirements.txt

@@ -0,0 +1,6 @@
+pytest
+pytest-env
+pytest-xdist
+
+# Timm
+git+https://github.com/rwightman/pytorch-image-models.git#egg=timm

contrib/test_timm.py

@@ -0,0 +1,10 @@
+def test_push_to_hub(repo_name: str, cleanup_repo: None) -> None:
+    import timm
+
+    # Build a model 🔧
+    model = timm.create_model("resnet18", pretrained=True, num_classes=4)
+
+    # Push it to the 🤗 hub
+    timm.models.hub.push_to_hf_hub(
+        model, repo_name, model_config=dict(labels=["a", "b", "c", "d"])
+    )