diff --git a/docs/usage/loading_results.md b/docs/usage/loading_results.md index 18e39b38d2..612bd369d5 100644 --- a/docs/usage/loading_results.md +++ b/docs/usage/loading_results.md @@ -7,12 +7,11 @@ For instance, if you are selecting the best model for semantic text similarity ( ```python import mteb -from mteb.cache import ResultCache tasks = mteb.get_tasks(tasks=["STS12"]) model_names = ["intfloat/multilingual-e5-large"] -cache = ResultCache("~/.cache/mteb") +cache = mteb.ResultCache("~/.cache/mteb") results = cache.load_results(models=model_names, tasks=tasks) ``` @@ -36,16 +35,16 @@ All previously submitted results are available results [repository](https://gith You can download this using: ```python -from mteb.cache import ResultCache +import mteb -cache = ResultCache() +cache = mteb.ResultCache() cache.download_from_remote() # download results from the remote repository ``` From here, you can work with the cache as usual. For instance, if you are selecting the best model for your French and English retrieval task on legal documents, you could fetch the relevant tasks and create a dataframe of the results using the following code: ```python -from mteb.cache import ResultCache +import mteb # select your tasks tasks = mteb.get_tasks(task_types=["Retrieval"], languages=["eng", "fra"], domains=["Legal"]) @@ -56,7 +55,7 @@ model_names = [ ] -cache = ResultCache() +cache = mteb.ResultCache() cache.download_from_remote() # download results from the remote repository. Might take a while the first time. results = cache.load_results( @@ -88,11 +87,10 @@ If you loaded results for a specific benchmark, you can get the aggregated bench ```python import mteb -from mteb.cache import ResultCache # Load results for a specific benchmark benchmark = mteb.get_benchmark("MTEB(eng, v2)") -cache = ResultCache() +cache = mteb.ResultCache() cache.download_from_remote() # download results from the remote repository results = cache.load_results( models=["intfloat/e5-small", "intfloat/multilingual-e5-small"], diff --git a/docs/whats_new.md b/docs/whats_new.md index 2352a2967e..b5a1b951ed 100644 --- a/docs/whats_new.md +++ b/docs/whats_new.md @@ -37,12 +37,12 @@ results = mteb.evaluate(model, tasks) ### Better local and online caching The new [`mteb.ResultCache`][mteb.cache.ResultCache] makes managing the cache notably easier: ```py -from mteb.cache import ResultCache +import mteb model = ... tasks = ... -cache = ResultCache(cache_path="~/.cache/mteb") # default +cache = mteb.ResultCache(cache_path="~/.cache/mteb") # default # simple evaluate with cache results = mteb.evaluate(model, tasks, cache=cache) # only runs if results not in cache @@ -169,9 +169,9 @@ We've added a lot of new documentation to make it easier to get started with MTE The new `ResultCache` also makes it easier to load, inspect and compare both local and online results: ```py -from mteb.cache import ResultCache +import mteb -cache = ResultCache(cache_path="~/.cache/mteb") # default +cache = mteb.ResultCache(cache_path="~/.cache/mteb") # default cache.download_from_remote() # download the latest results from the remote repository # load both local and online results diff --git a/mteb/__init__.py b/mteb/__init__.py index b6f4142bc7..a2952f5f95 100644 --- a/mteb/__init__.py +++ b/mteb/__init__.py @@ -3,6 +3,7 @@ from mteb import types from mteb.abstasks import AbsTask from mteb.abstasks.task_metadata import TaskMetadata +from mteb.cache import ResultCache from mteb.deprecated_evaluator import MTEB from mteb.evaluate import evaluate from mteb.filter_tasks import filter_tasks @@ -33,6 +34,7 @@ "CrossEncoderProtocol", "EncoderProtocol", "IndexEncoderSearchProtocol", + "ResultCache", "SearchProtocol", "SentenceTransformerEncoderWrapper", "TaskMetadata", diff --git a/mteb/cache.py b/mteb/cache.py index 2cfb327741..c37cd561d9 100644 --- a/mteb/cache.py +++ b/mteb/cache.py @@ -27,8 +27,8 @@ class ResultCache: """Class to handle the local cache of MTEB results. Examples: - >>> from mteb.cache import ResultCache - >>> cache = ResultCache(cache_path="~/.cache/mteb") # default + >>> import mteb + >>> cache = mteb.ResultCache(cache_path="~/.cache/mteb") # default >>> cache.download_from_remote() # download the latest results from the remote repository >>> result = cache.load_results("task_name", "model_name") """ @@ -320,8 +320,8 @@ def _download_cached_results_from_branch( OSError: On other file system errors Examples: - >>> from mteb.cache import ResultCache - >>> cache = ResultCache() + >>> import mteb + >>> cache = mteb.ResultCache() >>> # Download optimized cached results >>> cache_file = cache._download_cached_results_from_branch() >>> # Use custom output path @@ -460,8 +460,8 @@ def get_cache_paths( A list of paths in the cache directory. Examples: - >>> from mteb.cache import ResultCache - >>> cache = ResultCache() + >>> import mteb + >>> cache = mteb.ResultCache() >>> >>> # Get all cache paths >>> paths = cache.get_cache_paths() @@ -642,8 +642,8 @@ def load_results( A BenchmarkResults object containing the results for the specified models and tasks. Examples: - >>> from mteb.cache import ResultCache - >>> cache = ResultCache() + >>> import mteb + >>> cache = mteb.ResultCache() >>> >>> # Load results for specific models and tasks >>> results = cache.load_results(