feat: implement Gutenberg books scraping and resumable data ingestion…

app/services/embedding.py

@@ -1,26 +1,28 @@
-from sentence_transformers import SentenceTransformer
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from app.core.config import settings
 
-# Load a pre-trained HuggingFace model for generating embeddings.
-# all-MiniLM-L6-v2 is a good balance between speed and performance (384 dims).
-# If 1536 dims is strictly required to match existing pgvector tables or 
-# another embedding model config, use a compatible model or pad vectors.
-# For demonstration and efficiency, we use all-MiniLM-L6-v2.
+# Google's text-embedding-004 is a powerful model.
+# By default it produces 768 dimensions.
+# Since the database was initialized with 1536 dimensions, 
+# we should ideally match it or update the database schema.
+# For now, we'll use Gemini embeddings and recommend a schema update if necessary.
 
-MODEL_NAME = 'all-MiniLM-L6-v2' 
+MODEL_NAME = "models/gemini-embedding-001"
 
 class EmbeddingService:
     def __init__(self):
-        # The model is loaded into memory when the service is instantiated.
-        # This occurs on application startup if instantiated globally.
-        self.model = SentenceTransformer(MODEL_NAME)
+        # Using Gemini API for embeddings avoids the heavy local torch dependency
+        self.embeddings = GoogleGenerativeAIEmbeddings(
+            model=MODEL_NAME,
+            google_api_key=settings.GEMINI_API_KEY
+        )
 
     def generate_embedding(self, text: str) -> list[float]:
         """
-        Generates a vector embedding for the given text.
+        Generates a vector embedding for the given text using Gemini.
         """
-        embedding = self.model.encode(text)
-        # Convert numpy array to list of floats for pgvector/JSON
-        return embedding.tolist()
+        # The invoke method returns a list of floats
+        return self.embeddings.embed_query(text)
 
 # Singleton instance
 embedding_service = EmbeddingService()

app/services/llm.py

@@ -10,7 +10,7 @@
 # Configure LangChain model
 # We use gemini-1.5-pro or gemini-pro depending on availability and needs
 llm = ChatGoogleGenerativeAI(
-    model="gemini-1.5-pro", 
+    model="gemini-2.5-flash", 
     google_api_key=settings.GEMINI_API_KEY,
     temperature=0.7
 )

check_progress.py

@@ -0,0 +1,33 @@
+import os
+import sys
+
+# Ensure we can import app modules
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from app.services.database import supabase_client
+
+def check_progress():
+    print("Querying Supabase to check ingestion progress...")
+    try:
+        res = supabase_client.table('documents').select('metadata').execute()
+
+        books = {}
+        for row in res.data:
+            meta = row.get('metadata', {})
+            title = meta.get('book_info', {}).get('title', 'Unknown Title')
+            books[title] = books.get(title, 0) + 1
+
+        print("\n--- Ingestion Progress ---")
+        if not books:
+            print("No documents found in the database.")
+        else:
+            for title, count in books.items():
+                print(f"- {title}: {count} chunks")
+            print(f"\nTotal Chunks: {len(res.data)}")
+            print(f"Total Books Started: {len(books)}")
+
+    except Exception as e:
+        print(f"Error querying database: {e}")
+
+if __name__ == "__main__":
+    check_progress()