feat: make ragas pip more lighter (explodinggradients#283)

*merge explodinggradients#261 before merging this*

- removed all the optional dependencies that is making ragas bloated
- add option dependencies for sentence-transformer in `ragas[all]`

Author: jjmachan

.dockerignore

@@ -0,0 +1 @@
+Dockerfile

Makefile

@@ -28,6 +28,11 @@ run-ci: format lint type ## Running all CI checks
 run-benchmarks: ## Run benchmarks
 	@echo "Running benchmarks..."
 	@cd $(GIT_ROOT)/tests/benchmarks && python benchmark_eval.py
+run-benchmarks-in-docker: ## Run benchmarks in docker
+	@echo "Running benchmarks in docker..."
+	@cd $(GIT_ROOT)
+	docker buildx build --build-arg OPENAI_API_KEY=$(OPENAI_API_KEY) -t ragas-benchmark -f $(GIT_ROOT)/tests/benchmarks/Dockerfile . 
+	docker inspect ragas-benchmark:latest | jq ".[0].Size" | numfmt --to=si
 test: ## Run tests
 	@echo "Running tests..."
 	@pytest tests/unit $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)

pyproject.toml

@@ -2,8 +2,6 @@
 name = "ragas"
 dependencies = [
     "numpy",
-    "transformers",
-    "sentence-transformers",
     "datasets",
     "tiktoken",
     "langchain",
@@ -13,6 +11,11 @@ dependencies = [
 ]
 dynamic = ["version", "readme"]
 
+[project.optional-dependencies]
+all = [
+    "sentence-transformers",
+]
+
 [tool.setuptools]
 package-dir = {"" = "src"}
 

src/ragas/metrics/base.py

@@ -16,12 +16,12 @@
 from langchain.callbacks.manager import CallbackManager, trace_as_chain_group
 from tqdm import tqdm
 
+from ragas.embeddings.base import RagasEmbeddings
 from ragas.llms import llm_factory
 
 if t.TYPE_CHECKING:
     from langchain.callbacks.base import Callbacks
 
-    from ragas.embeddings.base import RagasEmbeddings
     from ragas.llms import RagasLLM
 
 

src/ragas/utils.py

@@ -2,31 +2,13 @@
 
 import logging
 import os
-import typing as t
 from functools import lru_cache
-from warnings import warn
 
-import torch
-from torch import device as Device
-
-DEVICES = ["cpu", "cuda"]
 DEBUG_ENV_VAR = "RAGAS_DEBUG"
 # constant to tell us that there is no key passed to the llm/embeddings
 NO_KEY = "no-key"
 
 
-def device_check(device: t.Literal["cpu", "cuda"] | Device) -> torch.device:
-    if isinstance(device, Device):
-        return device
-    if device not in DEVICES:
-        raise ValueError(f"Invalid device {device}")
-    if device == "cuda" and not torch.cuda.is_available():
-        warn("cuda not available, using cpu")
-        device = "cpu"
-
-    return torch.device(device)
-
-
 @lru_cache(maxsize=1)
 def get_debug_mode() -> bool:
     if os.environ.get(DEBUG_ENV_VAR, str(False)).lower() == "true":

tests/benchmarks/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.9-slim
+RUN apt-get update && apt-get install -y git make
+COPY . /app
+WORKDIR /app
+RUN pip install -e /app/
+ARG OPENAI_API_KEY
+ENV OPENAI_API_KEY=$OPENAI_API_KEY
+RUN make run-benchmarks

tests/benchmarks/benchmark_eval.py

@@ -1,7 +1,6 @@
 import time
 
 from datasets import DatasetDict, load_dataset
-from torch.cuda import is_available
 
 from ragas import evaluate
 from ragas.metrics import (
@@ -12,8 +11,6 @@
 )
 from ragas.metrics.critique import harmfulness
 
-DEVICE = "cuda" if is_available() else "cpu"
-
 # data
 ds = load_dataset("explodinggradients/fiqa", "ragas_eval")
 assert isinstance(ds, DatasetDict)