Fix issue 88

.hydra_config/config.yaml

@@ -69,6 +69,18 @@ loader:
   image_captioning: true
   save_markdown: false
   audio_model: ${oc.env:WHISPER_MODEL, base} # tiny, base, small, medium, large-v1, large-v2, large-v3
+  mimetypes:
+    text/plain: .txt
+    text/markdown: .md
+    application/pdf: .pdf
+    message/rfc822: .eml
+    application/vnd.openxmlformats-officedocument.wordprocessingml.document: .docx
+    application/vnd.openxmlformats-officedocument.presentationml.presentation: .pptx
+    application/msword: .doc
+    image/png: .png
+    image/jpeg: .jpeg
+    audio/vnd.wav: .wav
+    audio/mpeg: .mp3
   file_loaders:
     txt: TextLoader
     pdf: ${oc.env:PDFLoader, MarkerLoader}  # DoclingLoader # MarkerLoader # PyMuPDFLoader # Custompymupdf4llm

docker-compose.yaml

@@ -4,15 +4,15 @@ include:
   - extern/infinity.yaml
   - ${INDEXERUI_COMPOSE_FILE:-extern/dummy.yaml}
 
-x-openrag: &openrag_template
+x-ragondin: &ragondin_template
   build:
     context: .
     dockerfile: Dockerfile
   volumes:
     - ${CONFIG_VOLUME:-./.hydra_config}:/app/.hydra_config
     - ${DATA_VOLUME:-./data}:/app/data
     - ${MODEL_WEIGHTS_VOLUME:-~/.cache/huggingface}:/app/model_weights # Model weights for RAG
-    - ./openrag:/app/openrag # For dev mode
+    - ./ragondin:/app/ragondin # For dev mode
     - ${DB_VOLUME:-./db}:/app/db
     - /$SHARED_ENV:/ray_mount/.env # Shared environment variables
     - /ray_mount/logs:/app/logs
@@ -22,7 +22,7 @@ x-openrag: &openrag_template
   networks:
     default:
       aliases:
-        - openrag
+        - ragondin
   env_file:
     - ${SHARED_ENV:-.env}
   shm_size: 10.24gb
@@ -45,11 +45,10 @@ x-vllm: &vllm_template
     --trust-remote-code
   ports:
     - ${VLLM_PORT:-8000}:8000
-
 services:
   # GPU - default 
-  openrag:
-    <<: *openrag_template
+  ragondin:
+    <<: *ragondin_template
     deploy:
       resources:
         reservations:
@@ -61,8 +60,8 @@ services:
       - ''
 
   # No GPU
-  openrag-cpu:
-    <<: *openrag_template
+  ragondin-cpu:
+    <<: *ragondin_template
     deploy: {}
     profiles:
       - 'cpu'
@@ -87,7 +86,7 @@ services:
       context: extern/vllm
       dockerfile: Dockerfile.cpu
       target: vllm-openai
-    image: openrag-vllm-openai-cpu
+    image: ragondin-vllm-openai-cpu
     deploy: {}
     profiles:
       - 'cpu'

openrag/components/indexer/loaders/serializer.py

@@ -21,6 +21,7 @@
 POOL_SIZE = config.ray.get("pool_size")
 MAX_TASKS_PER_WORKER = config.ray.get("max_tasks_per_worker")
 
+DICT_MIMETYPES = dict(config.loader["mimetypes"])
 
 @ray.remote(num_gpus=NUM_GPUS)
 class DocSerializer:
@@ -57,9 +58,15 @@ async def serialize_document(
 
         p = Path(path)
         file_ext = p.suffix
-
+        mimetype = metadata.get("mimetype", None)
         # Get appropriate loader for the file type
-        loader_cls = self.loader_classes.get(file_ext)
+        if mimetype is None:
+            loader_cls = self.loader_classes.get(file_ext)
+        else:
+            loader_cls = self.loader_classes.get(
+                DICT_MIMETYPES.get(mimetype)
+            )
+
         if loader_cls is None:
             log.warning(f"No loader available for {p.name}")
             raise ValueError(f"No loader available for file type {file_ext}.")