diff --git a/.github/workflows/python-api-superb.yaml b/.github/workflows/python-api-superb.yaml deleted file mode 100644 index fc409ae0..00000000 --- a/.github/workflows/python-api-superb.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# TODO: Merge this with allenNLP to have a single workflow for all docker images. -name: spacy-docker - -on: - pull_request: - paths: - - "docker_images/superb/**" -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - name: Checkout - uses: actions/checkout@v2 - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Install dependencies - run: | - pip install --upgrade pip - pip install pytest pillow httpx - pip install -e . - - run: RUN_DOCKER_TESTS=1 pytest -sv tests/test_dockers.py::DockerImageTests::test_superb diff --git a/docker_images/superb/Dockerfile b/docker_images/superb/Dockerfile deleted file mode 100644 index ec2dc32d..00000000 --- a/docker_images/superb/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM tiangolo/uvicorn-gunicorn:python3.8 -LABEL maintainer="Omar osanseviero@gmail.com" - -# Add any system dependency here -# RUN apt-get update -y && apt-get install libXXX -y -RUN apt-get update -y -RUN apt-get install git -y -RUN apt-get install libsndfile1 -y -RUN apt-get install ffmpeg -y - -RUN pip install -U pip -RUN pip install --no-cache-dir torch==1.9 -COPY ./requirements.txt /app -# Speeding up requirements -RUN pip install --no-cache-dir -r requirements.txt -COPY ./prestart.sh /app/ - - -# Most DL models are quite large in terms of memory, using workers is a HUGE -# slowdown because of the fork and GIL with python. -# Using multiple pods seems like a better default strategy. -# Feel free to override if it does not make sense for your library. -ARG max_workers=1 -ENV MAX_WORKERS=$max_workers -ENV HUGGINGFACE_HUB_CACHE=/data -ENV TORCH_HOME=/data - -# Necessary on GPU environment docker. -# TIMEOUT env variable is used by nvcr.io/nvidia/pytorch:xx for another purpose -# rendering TIMEOUT defined by uvicorn impossible to use correctly -# We're overriding it to be renamed UVICORN_TIMEOUT -# UVICORN_TIMEOUT is a useful variable for very large models that take more -# than 30s (the default) to load in memory. -# If UVICORN_TIMEOUT is too low, uvicorn will simply never loads as it will -# kill workers all the time before they finish. -RUN sed -i 's/TIMEOUT/UVICORN_TIMEOUT/g' /gunicorn_conf.py -COPY ./app /app/app diff --git a/docker_images/superb/app/__init__.py b/docker_images/superb/app/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/docker_images/superb/app/main.py b/docker_images/superb/app/main.py deleted file mode 100644 index c736f699..00000000 --- a/docker_images/superb/app/main.py +++ /dev/null @@ -1,96 +0,0 @@ -import functools -import logging -import os -from typing import Dict, Type - -from api_inference_community.routes import pipeline_route, status_ok -from app.pipelines import ( - AutomaticSpeechRecognitionPipeline, - Pipeline, - SpeechSegmentationPipeline, -) -from starlette.applications import Starlette -from starlette.middleware import Middleware -from starlette.middleware.gzip import GZipMiddleware -from starlette.routing import Route - - -TASK = os.getenv("TASK") -MODEL_ID = os.getenv("MODEL_ID") - - -logger = logging.getLogger(__name__) - - -# Add the allowed tasks -# Supported tasks are: -# - text-generation -# - text-classification -# - token-classification -# - translation -# - summarization -# - automatic-speech-recognition -# - ... -# For instance -# from app.pipelines import AutomaticSpeechRecognitionPipeline -# ALLOWED_TASKS = {"automatic-speech-recognition": AutomaticSpeechRecognitionPipeline} -# You can check the requirements and expectations of each pipelines in their respective -# directories. Implement directly within the directories. -ALLOWED_TASKS: Dict[str, Type[Pipeline]] = { - "automatic-speech-recognition": AutomaticSpeechRecognitionPipeline, - "speech-segmentation": SpeechSegmentationPipeline, -} - - -@functools.lru_cache() -def get_pipeline() -> Pipeline: - task = os.environ["TASK"] - model_id = os.environ["MODEL_ID"] - if task not in ALLOWED_TASKS: - raise EnvironmentError(f"{task} is not a valid pipeline for model : {model_id}") - return ALLOWED_TASKS[task](model_id) - - -routes = [ - Route("/{whatever:path}", status_ok), - Route("/{whatever:path}", pipeline_route, methods=["POST"]), -] - -middleware = [Middleware(GZipMiddleware, minimum_size=1000)] -if os.environ.get("DEBUG", "") == "1": - from starlette.middleware.cors import CORSMiddleware - - middleware.append( - Middleware( - CORSMiddleware, - allow_origins=["*"], - allow_headers=["*"], - allow_methods=["*"], - ) - ) - -app = Starlette(routes=routes, middleware=middleware) - - -@app.on_event("startup") -async def startup_event(): - logger = logging.getLogger("uvicorn.access") - handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")) - logger.handlers = [handler] - - # Link between `api-inference-community` and framework code. - app.get_pipeline = get_pipeline - try: - get_pipeline() - except Exception: - # We can fail so we can show exception later. - pass - - -if __name__ == "__main__": - try: - get_pipeline() - except Exception: - # We can fail so we can show exception later. - pass diff --git a/docker_images/superb/app/pipelines/__init__.py b/docker_images/superb/app/pipelines/__init__.py deleted file mode 100644 index 8e8c4c8f..00000000 --- a/docker_images/superb/app/pipelines/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from app.pipelines.base import Pipeline, PipelineException # isort:skip - -from app.pipelines.automatic_speech_recognition import ( - AutomaticSpeechRecognitionPipeline, -) -from app.pipelines.speech_segmentation import SpeechSegmentationPipeline diff --git a/docker_images/superb/app/pipelines/automatic_speech_recognition.py b/docker_images/superb/app/pipelines/automatic_speech_recognition.py deleted file mode 100644 index fcabf3fb..00000000 --- a/docker_images/superb/app/pipelines/automatic_speech_recognition.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import subprocess -import sys -from typing import Dict - -import numpy as np -from app.pipelines import Pipeline -from huggingface_hub import snapshot_download - - -class AutomaticSpeechRecognitionPipeline(Pipeline): - def __init__(self, model_id: str): - # IMPLEMENT_THIS - # Preload all the elements you are going to need at inference. - # For instance your model, processors, tokenizer that might be needed. - # This function is only called once, so do all the heavy processing I/O here - # IMPLEMENT_THIS : Please define a `self.sampling_rate` for this pipeline - # to automatically read the input correctly - filepath = snapshot_download(model_id) - sys.path.append(filepath) - if "requirements.txt" in os.listdir(filepath): - subprocess.check_call( - [ - sys.executable, - "-m", - "pip", - "install", - "-r", - os.path.join(filepath, "requirements.txt"), - ] - ) - - from model import PreTrainedModel - - self.model = PreTrainedModel(filepath) - self.sampling_rate = 16000 - - def __call__(self, inputs: np.array) -> Dict[str, str]: - """ - Args: - inputs (:obj:`np.array`): - The raw waveform of audio received. By default at 16KHz. - Check `app.validation` if a different sample rate is required - or if it depends on the model - Return: - A :obj:`dict`:. The object return should be liked {"text": "XXX"} containing - the detected langage from the input audio - """ - return self.model(inputs) diff --git a/docker_images/superb/app/pipelines/base.py b/docker_images/superb/app/pipelines/base.py deleted file mode 100644 index 2d846eb4..00000000 --- a/docker_images/superb/app/pipelines/base.py +++ /dev/null @@ -1,16 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any - - -class Pipeline(ABC): - @abstractmethod - def __init__(self, model_id: str): - raise NotImplementedError("Pipelines should implement an __init__ method") - - @abstractmethod - def __call__(self, inputs: Any) -> Any: - raise NotImplementedError("Pipelines should implement a __call__ method") - - -class PipelineException(Exception): - pass diff --git a/docker_images/superb/app/pipelines/speech_segmentation.py b/docker_images/superb/app/pipelines/speech_segmentation.py deleted file mode 100644 index ac7f172b..00000000 --- a/docker_images/superb/app/pipelines/speech_segmentation.py +++ /dev/null @@ -1,56 +0,0 @@ -import os -import subprocess -import sys -from typing import Dict, List, Union - -import numpy as np -from api_inference_community.normalizers import speaker_diarization_normalize -from app.pipelines import Pipeline -from huggingface_hub import snapshot_download - - -class SpeechSegmentationPipeline(Pipeline): - def __init__(self, model_id: str): - # IMPLEMENT_THIS - # Preload all the elements you are going to need at inference. - # For instance your model, processors, tokenizer that might be needed. - # This function is only called once, so do all the heavy processing I/O here - # IMPLEMENT_THIS : Please define a `self.sampling_rate` for this pipeline - # to automatically read the input correctly - filepath = snapshot_download(model_id) - sys.path.append(filepath) - if "requirements.txt" in os.listdir(filepath): - subprocess.check_call( - [ - sys.executable, - "-m", - "pip", - "install", - "-r", - os.path.join(filepath, "requirements.txt"), - ] - ) - - from model import PreTrainedModel - - self.model = PreTrainedModel(filepath) - self.sampling_rate = 16000 - - def __call__(self, inputs: np.array) -> List[Dict[str, Union[str, float]]]: - """ - Args: - inputs (:obj:`np.array`): - The raw waveform of audio received. By default at self.sampling_rate, otherwise 16KHz. - Return: - A :obj:`list`:. Each item in the list is like {"class": "XXX", "start": float, "end": float} - "class" is the associated class of the audio segment, "start" and "end" are markers expressed in seconds - within the audio file. Segments can overlap in any way the want. - """ - # S x N boolean tensor - # S : sequence_length - # N : Number of expected speakers - # Filled with ones where speaker-n is speaking - outputs = self.model(inputs) - return speaker_diarization_normalize( - outputs, self.sampling_rate, ["SPEAKER_0", "SPEAKER_1"] - ) diff --git a/docker_images/superb/prestart.sh b/docker_images/superb/prestart.sh deleted file mode 100644 index 67cbe056..00000000 --- a/docker_images/superb/prestart.sh +++ /dev/null @@ -1,2 +0,0 @@ -git-lfs install -python app/main.py diff --git a/docker_images/superb/requirements.txt b/docker_images/superb/requirements.txt deleted file mode 100644 index b9a83140..00000000 --- a/docker_images/superb/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -starlette==0.14.2 -api-inference-community==0.0.23 -git+https://github.com/pytorch/fairseq.git@f2146bd#egg=fairseq -git+https://github.com/huggingface/s3prl.git@huggingface2#egg=s3prl -hf-lfs -huggingface_hub==0.5.1 diff --git a/docker_images/superb/tests/__init__.py b/docker_images/superb/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/docker_images/superb/tests/samples/malformed.flac b/docker_images/superb/tests/samples/malformed.flac deleted file mode 100644 index 06d74050..00000000 Binary files a/docker_images/superb/tests/samples/malformed.flac and /dev/null differ diff --git a/docker_images/superb/tests/samples/plane.jpg b/docker_images/superb/tests/samples/plane.jpg deleted file mode 100644 index 9e0cc585..00000000 Binary files a/docker_images/superb/tests/samples/plane.jpg and /dev/null differ diff --git a/docker_images/superb/tests/samples/plane2.jpg b/docker_images/superb/tests/samples/plane2.jpg deleted file mode 100644 index b1f076e3..00000000 Binary files a/docker_images/superb/tests/samples/plane2.jpg and /dev/null differ diff --git a/docker_images/superb/tests/samples/sample1.flac b/docker_images/superb/tests/samples/sample1.flac deleted file mode 100644 index 837cd984..00000000 Binary files a/docker_images/superb/tests/samples/sample1.flac and /dev/null differ diff --git a/docker_images/superb/tests/samples/sample1.webm b/docker_images/superb/tests/samples/sample1.webm deleted file mode 100644 index 7c449a27..00000000 Binary files a/docker_images/superb/tests/samples/sample1.webm and /dev/null differ diff --git a/docker_images/superb/tests/samples/sample1_dual.ogg b/docker_images/superb/tests/samples/sample1_dual.ogg deleted file mode 100644 index 1e8524e0..00000000 Binary files a/docker_images/superb/tests/samples/sample1_dual.ogg and /dev/null differ diff --git a/docker_images/superb/tests/test_api.py b/docker_images/superb/tests/test_api.py deleted file mode 100644 index ceb691ac..00000000 --- a/docker_images/superb/tests/test_api.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -from typing import Dict, List -from unittest import TestCase, skipIf - -from app.main import ALLOWED_TASKS, get_pipeline - - -# Must contain at least one example of each implemented pipeline -# Tests do not check the actual values of the model output, so small dummy -# models are recommended for faster tests. -TESTABLE_MODELS: Dict[str, List[str]] = { - "automatic-speech-recognition": [ - # Repo using transformers library. - "osanseviero/asr-with-transformers-wav2vec2", - # Repo using s3prl. - "osanseviero/hubert_s3prl", - # Repo using s3prl and installing a custom dependency. - "osanseviero/hubert_s3prl_req", - ], - "speech-segmentation": "osanseviero/hubert-sd", -} - - -ALL_TASKS = { - "automatic-speech-recognition", - "feature-extraction", - "image-classification", - "question-answering", - "sentence-similarity", - "text-generation", - "text-to-speech", - "token-classification", -} - - -class PipelineTestCase(TestCase): - @skipIf( - os.path.dirname(os.path.dirname(__file__)).endswith("common"), - "common is a special case", - ) - def test_has_at_least_one_task_enabled(self): - self.assertGreater( - len(ALLOWED_TASKS.keys()), 0, "You need to implement at least one task" - ) - - def test_unsupported_tasks(self): - unsupported_tasks = ALL_TASKS - ALLOWED_TASKS.keys() - for unsupported_task in unsupported_tasks: - with self.subTest(msg=unsupported_task, task=unsupported_task): - os.environ["TASK"] = unsupported_task - os.environ["MODEL_ID"] = "XX" - with self.assertRaises(EnvironmentError): - get_pipeline() diff --git a/docker_images/superb/tests/test_api_automatic_speech_recognition.py b/docker_images/superb/tests/test_api_automatic_speech_recognition.py deleted file mode 100644 index 941bec2f..00000000 --- a/docker_images/superb/tests/test_api_automatic_speech_recognition.py +++ /dev/null @@ -1,100 +0,0 @@ -import json -import os -from unittest import TestCase, skipIf - -from app.main import ALLOWED_TASKS -from parameterized import parameterized_class -from starlette.testclient import TestClient -from tests.test_api import TESTABLE_MODELS - - -@skipIf( - "automatic-speech-recognition" not in ALLOWED_TASKS, - "automatic-speech-recognition not implemented", -) -@parameterized_class( - [ - {"model_id": model_id} - for model_id in TESTABLE_MODELS["automatic-speech-recognition"] - ] -) -class AutomaticSpeecRecognitionTestCase(TestCase): - def setUp(self): - self.old_model_id = os.getenv("MODEL_ID") - self.old_task = os.getenv("TASK") - os.environ["MODEL_ID"] = self.model_id - os.environ["TASK"] = "automatic-speech-recognition" - - from app.main import app, get_pipeline - - get_pipeline.cache_clear() - - self.app = app - - def tearDown(self): - if self.old_model_id is not None: - os.environ["MODEL_ID"] = self.old_model_id - else: - del os.environ["MODEL_ID"] - if self.old_task is not None: - os.environ["TASK"] = self.old_task - else: - del os.environ["TASK"] - - def read(self, filename: str) -> bytes: - dirname = os.path.dirname(os.path.abspath(__file__)) - filename = os.path.join(dirname, "samples", filename) - with open(filename, "rb") as f: - bpayload = f.read() - return bpayload - - def test_original_audiofile(self): - bpayload = self.read("sample1.flac") - - with TestClient(self.app) as client: - response = client.post("/", data=bpayload) - - self.assertEqual( - response.status_code, - 200, - ) - content = json.loads(response.content) - self.assertEqual(set(content.keys()), {"text"}) - - def test_malformed_audio(self): - bpayload = self.read("malformed.flac") - - with TestClient(self.app) as client: - response = client.post("/", data=bpayload) - - self.assertEqual( - response.status_code, - 400, - ) - self.assertEqual(response.content, b'{"error":"Malformed soundfile"}') - - def test_dual_channel_audiofile(self): - bpayload = self.read("sample1_dual.ogg") - - with TestClient(self.app) as client: - response = client.post("/", data=bpayload) - - self.assertEqual( - response.status_code, - 200, - ) - content = json.loads(response.content) - self.assertEqual(set(content.keys()), {"text"}) - - def test_webm_audiofile(self): - bpayload = self.read("sample1.webm") - - with TestClient(self.app) as client: - response = client.post("/", data=bpayload) - - self.assertEqual( - response.status_code, - 200, - ) - content = json.loads(response.content) - self.assertEqual(set(content.keys()), {"text"}) diff --git a/docker_images/superb/tests/test_api_speech_segmentation.py b/docker_images/superb/tests/test_api_speech_segmentation.py deleted file mode 100644 index c88292f1..00000000 --- a/docker_images/superb/tests/test_api_speech_segmentation.py +++ /dev/null @@ -1,112 +0,0 @@ -import json -import os -from unittest import TestCase, skipIf - -from app.main import ALLOWED_TASKS -from starlette.testclient import TestClient -from tests.test_api import TESTABLE_MODELS - - -@skipIf( - "speech-segmentation" not in ALLOWED_TASKS, - "speech-segmentation not implemented", -) -class SpeechSegmentationTestCase(TestCase): - def setUp(self): - model_id = TESTABLE_MODELS["speech-segmentation"] - self.old_model_id = os.getenv("MODEL_ID") - self.old_task = os.getenv("TASK") - os.environ["MODEL_ID"] = model_id - os.environ["TASK"] = "speech-segmentation" - from app.main import app - - self.app = app - - @classmethod - def setUpClass(cls): - from app.main import get_pipeline - - get_pipeline.cache_clear() - - def tearDown(self): - if self.old_model_id is not None: - os.environ["MODEL_ID"] = self.old_model_id - else: - del os.environ["MODEL_ID"] - if self.old_task is not None: - os.environ["TASK"] = self.old_task - else: - del os.environ["TASK"] - - def read(self, filename: str) -> bytes: - dirname = os.path.dirname(os.path.abspath(__file__)) - filename = os.path.join(dirname, "samples", filename) - with open(filename, "rb") as f: - bpayload = f.read() - return bpayload - - def test_original_audiofile(self): - bpayload = self.read("sample1.flac") - - with TestClient(self.app) as client: - response = client.post("/", data=bpayload) - - self.assertEqual( - response.status_code, - 200, - ) - content = json.loads(response.content) - self.assertIsInstance(content, list) - for c in content: - self.assertEqual(set(c.keys()), {"class", "start", "end"}) - self.assertIsInstance(c["class"], str) - self.assertIsInstance(c["start"], float) - self.assertIsInstance(c["end"], float) - - def test_malformed_audio(self): - bpayload = self.read("malformed.flac") - - with TestClient(self.app) as client: - response = client.post("/", data=bpayload) - - self.assertEqual( - response.status_code, - 400, - ) - self.assertEqual(response.content, b'{"error":"Malformed soundfile"}') - - def test_dual_channel_audiofile(self): - bpayload = self.read("sample1_dual.ogg") - - with TestClient(self.app) as client: - response = client.post("/", data=bpayload) - - self.assertEqual( - response.status_code, - 200, - ) - content = json.loads(response.content) - self.assertIsInstance(content, list) - for c in content: - self.assertEqual(set(c.keys()), {"class", "start", "end"}) - self.assertIsInstance(c["class"], str) - self.assertIsInstance(c["start"], float) - self.assertIsInstance(c["end"], float) - - def test_webm_audiofile(self): - bpayload = self.read("sample1.webm") - - with TestClient(self.app) as client: - response = client.post("/", data=bpayload) - - self.assertEqual( - response.status_code, - 200, - ) - content = json.loads(response.content) - self.assertIsInstance(content, list) - for c in content: - self.assertEqual(set(c.keys()), {"class", "start", "end"}) - self.assertIsInstance(c["class"], str) - self.assertIsInstance(c["start"], float) - self.assertIsInstance(c["end"], float) diff --git a/docker_images/superb/tests/test_docker_build.py b/docker_images/superb/tests/test_docker_build.py deleted file mode 100644 index ca5dd65d..00000000 --- a/docker_images/superb/tests/test_docker_build.py +++ /dev/null @@ -1,23 +0,0 @@ -import os -import subprocess -from unittest import TestCase - - -class cd: - """Context manager for changing the current working directory""" - - def __init__(self, newPath): - self.newPath = os.path.expanduser(newPath) - - def __enter__(self): - self.savedPath = os.getcwd() - os.chdir(self.newPath) - - def __exit__(self, etype, value, traceback): - os.chdir(self.savedPath) - - -class DockerBuildTestCase(TestCase): - def test_can_build_docker_image(self): - with cd(os.path.dirname(os.path.dirname(__file__))): - subprocess.check_output(["docker", "build", "."]) diff --git a/tests/test_dockers.py b/tests/test_dockers.py index 2ac43f17..4f726fa5 100644 --- a/tests/test_dockers.py +++ b/tests/test_dockers.py @@ -327,33 +327,6 @@ def test_doctr(self): ) self.framework_invalid_test("doctr") - def test_superb(self): - # Very basic repo just using transformers. - self.framework_docker_test( - "superb", - "automatic-speech-recognition", - "osanseviero/asr-with-transformers-wav2vec2", - ) - self.framework_docker_test( - "superb", - "speech-segmentation", - "osanseviero/hubert-sd", - ) - # # Too slow, requires downloading the upstream model from PyTorch Hub which is quite heavy - # # self.framework_docker_test( - # # "superb", "automatic-speech-recognition", "osanseviero/hubert_s3prl_req" - # # ) - self.framework_invalid_test("superb") - self.framework_docker_batch( - "superb", - "automatic-speech-recognition", - "osanseviero/asr-with-transformers-wav2vec2", - dataset_name="Narsil/asr_dummy", - dataset_config="asr", - dataset_split="test", - dataset_column="file", - ) - def test_nemo(self): self.framework_docker_test( "nemo", "automatic-speech-recognition", "nvidia/stt_en_conformer_ctc_medium"