Skip to content

Commit

Permalink
remove chroma dependency; add local storage that doesn't have retrieval
Browse files Browse the repository at this point in the history
  • Loading branch information
astaff committed Jul 28, 2024
1 parent 58e7d13 commit 32f767a
Show file tree
Hide file tree
Showing 5 changed files with 122 additions and 1,705 deletions.
6 changes: 6 additions & 0 deletions platogram/library/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,9 @@ def get_keyword_local_bm25(home_dir: Path = Path("./my_library")) -> Library:
from .keyword_local_bm25 import LocalBM25Library

return LocalBM25Library(home_dir)


def get_local_dumb(home_dir: Path = Path("./my_library")) -> Library:
from .local_dumb import LocalDumbLibrary

return LocalDumbLibrary(home_dir)
45 changes: 45 additions & 0 deletions platogram/library/local_dumb.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import json
from pathlib import Path

from platogram.types import Content
from platogram.utils import make_filesystem_safe


class LocalDumbLibrary:
def __init__(self, home_dir: Path):
if not home_dir.exists():
home_dir.mkdir(parents=True)
self.home_dir = home_dir

@property
def home(self) -> Path:
return self.home_dir

def ls(self) -> list[str]:
return [f.stem for f in self.home.glob("*.json")]

def exists(self, id: str) -> bool:
return (self.home / f"{make_filesystem_safe(id)}.json").exists()

def put(self, id: str, content: Content) -> None:
file = self.home / f"{make_filesystem_safe(id)}.json"
with open(file, "w") as f:
json.dump(content.model_dump(mode="json"), f)

def get_content(self, id: str) -> Content:
file = self.home_dir / f"{make_filesystem_safe(id)}.json"
with open(file, "r") as f:
content = Content(**json.load(f))
return content

def delete(self, id: str) -> None:
file = self.home_dir / f"{make_filesystem_safe(id)}.json"
file.unlink()

def retrieve(
self,
query: str,
n_results: int,
filter_keys: list[str],
) -> tuple[list[Content], list[float]]:
raise NotImplementedError("Dumb local storage does not support retrieval. Use get_content().")
11 changes: 9 additions & 2 deletions platogram/library/semantic_local_chroma.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,15 @@
import os
from pathlib import Path

import chromadb
from chromadb.utils import embedding_functions
try:
import chromadb
except ImportError:
pass

try:
from chromadb.utils import embedding_functions
except ImportError:
pass

from platogram.ops import remove_markers
from platogram.types import Content
Expand Down
Loading

0 comments on commit 32f767a

Please sign in to comment.