feat: add pdf ingestion lane

pmoves/.env.example

@@ -48,6 +48,11 @@ MINIO_SECURE=false
 AWS_DEFAULT_REGION=us-east-1
 ALLOWED_BUCKETS=assets,outputs
 PRESIGN_SHARED_SECRET=change_me
+# PDF ingest defaults
+PDF_DEFAULT_BUCKET=assets
+PDF_DEFAULT_NAMESPACE=pmoves
+PDF_MAX_PAGES=0
+PDF_INGEST_EXTRACT_URL=http://extract-worker:8083/ingest
 
 # Supabase / PostgREST
 POSTGRES_DB=pmoves

pmoves/Makefile

@@ -1,7 +1,7 @@
 .PHONY: up down clean
 up:
 	docker compose --profile data up -d qdrant neo4j minio meilisearch postgres postgrest presign
-	docker compose --profile workers up -d hi-rag-gateway-v2 retrieval-eval render-webhook langextract extract-worker
+        docker compose --profile workers up -d hi-rag-gateway-v2 retrieval-eval render-webhook langextract extract-worker pdf-ingest
 
 down:
 	docker compose down
@@ -48,8 +48,32 @@ eval-jsonl:
 smoke-presign-put:
 	@which jq >/dev/null 2>&1 || (echo "jq is required" && exit 1)
 	@echo "Generating presign PUT and uploading a small text file..."
-	@URL=$$(curl -s -X POST http://localhost:8088/presign/put -H 'content-type: application/json' -H "Authorization: Bearer $${PRESIGN_SHARED_SECRET:-change_me}" -d '{"bucket":"outputs","key":"hello.txt","content_type":"text/plain","expires":300}') && \
-	 echo $$URL | jq -r '.url' | xargs -I {} sh -c "echo 'hello pmoves' | curl -s -X PUT -H 'Content-Type: text/plain' --data-binary @- '{}' >/dev/null" && echo OK || (echo FAIL && exit 1)
+	@URL=$$(curl -s -X POST http://localhost:8088/presign/put \
+	  -H 'content-type: application/json' \
+	  -H "Authorization: Bearer $${PRESIGN_SHARED_SECRET:-change_me}" \
+	  -d '{"bucket":"outputs","key":"hello.txt","content_type":"text/plain","expires":300}' ) && \
+	  echo $$URL | jq -r '.url' | xargs -I {} sh -c "echo 'hello pmoves' | curl -s -X PUT -H 'Content-Type: text/plain' --data-binary @- '{}' >/dev/null" && echo OK || (echo FAIL && exit 1)
+
+.PHONY: smoke-pdf
+smoke-pdf:
+	@which jq >/dev/null 2>&1 || (echo "jq is required for smoke-pdf" && exit 1)
+	@test -f pmoves/datasets/sample.pdf || (echo "datasets/sample.pdf missing" && exit 1)
+	@echo "[1/4] Create presign URL for sample.pdf" && \
+	URL=$$(curl -s -X POST http://localhost:8088/presign/put \
+	  -H 'content-type: application/json' \
+	  -H "Authorization: Bearer $${PRESIGN_SHARED_SECRET:-change_me}" \
+	  -d '{"bucket":"assets","key":"pdfs/sample.pdf","content_type":"application/pdf","expires":300}' ) && echo OK || (echo FAIL && exit 1)
+	@echo "[2/4] Upload sample.pdf to MinIO" && \
+	echo "$$URL" | jq -r .url | xargs -I {} curl -s -X PUT -H 'Content-Type: application/pdf' --data-binary @pmoves/datasets/sample.pdf '{}' >/dev/null && echo OK || (echo FAIL && exit 1)
+	@echo "[3/4] Trigger pdf-ingest" && \
+	curl -sf http://localhost:8092/pdf/ingest -H 'content-type: application/json' \
+	  -d '{"bucket":"assets","key":"pdfs/sample.pdf","namespace":"pmoves","title":"Sample PDF","publish_events":false}' \
+	  | jq -e '.ok == true and .chunks >= 1' >/dev/null && echo OK || (echo FAIL && exit 1)
+	@echo "[4/4] Verify chunks in extract-worker response" && \
+	curl -sf http://localhost:8092/pdf/ingest -H 'content-type: application/json' \
+	  -d '{"bucket":"assets","key":"pdfs/sample.pdf","namespace":"pmoves","doc_id":"pdf:sample","publish_events":false}' \
+	  | jq -e '.ingest.chunks >= 1' >/dev/null && echo OK || (echo FAIL && exit 1)
+	@echo "PDF ingest smoke passed."
 
 .PHONY: seed-data
 seed-data:
@@ -58,6 +82,7 @@ seed-data:
 	@echo "Seeding Qdrant + Meilisearch with small demo docs..."
 	docker compose run --rm --entrypoint python hi-rag-gateway-v2 /app/scripts/seed_local.py
 
+
 .PHONY: load-jsonl
 load-jsonl:
 	@if [ -z "$(FILE)" ]; then echo "Usage: make load-jsonl FILE=path/to/data.jsonl [NAMESPACE=pmoves]"; exit 1; fi
@@ -133,6 +158,7 @@ help:
 	@echo "  load-csv         Load a CSV file (FILE=/abs/path)"
 	@echo "  export-jsonl     Export namespace to JSONL (OUT=/abs/path)"
 	@echo "  smoke-presign-put Test presign PUT and upload"
+	@echo "  smoke-pdf        Upload and index a sample PDF via pdf-ingest"
 	@echo "  smoke-langextract Extract chunks via langextract and load"
 	@echo "  flight-check     Run environment preflight (deps, ports, .env)"
 	@echo "  flight-check-retro Retro-styled Rich CLI preflight"

pmoves/contracts/schemas/ingest/document-ready.v1.schema.json

@@ -0,0 +1,45 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Ingest Document Ready",
+  "type": "object",
+  "required": [
+    "doc_id",
+    "namespace",
+    "uri",
+    "chunk_count"
+  ],
+  "properties": {
+    "doc_id": {
+      "type": "string"
+    },
+    "namespace": {
+      "type": "string"
+    },
+    "uri": {
+      "type": "string"
+    },
+    "chunk_count": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "file_id": {
+      "type": "string"
+    },
+    "title": {
+      "type": ["string", "null"]
+    },
+    "checksum": {
+      "type": "string"
+    },
+    "size_bytes": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "preview": {
+      "type": "string"
+    },
+    "meta": {
+      "type": "object"
+    }
+  }
+}

pmoves/contracts/topics.json

@@ -4,9 +4,12 @@
     "ingest.transcript.ready.v1": {
       "schema": "schemas/ingest/transcript-ready.v1.schema.json"
     },
-    "ingest.file.added.v1": {
-      "schema": "schemas/ingest/file-added.v1.schema.json"
-    },
+    "ingest.file.added.v1": {
+      "schema": "schemas/ingest/file-added.v1.schema.json"
+    },
+    "ingest.document.ready.v1": {
+      "schema": "schemas/ingest/document-ready.v1.schema.json"
+    },
     "analysis.extract_topics.request.v1": {
       "schema": "schemas/analysis/extract-topics.request.v1.schema.json"
     },

pmoves/docker-compose.yml

@@ -163,6 +163,30 @@ services:
     networks: [pmoves]
     extra_hosts:
       - "host.docker.internal:host-gateway"
+  pdf-ingest:
+    build:
+      context: .
+      dockerfile: services/pdf-ingest/Dockerfile
+    restart: unless-stopped
+    env_file: [.env]
+    environment:
+      - MINIO_ENDPOINT=${MINIO_ENDPOINT:-minio:9000}
+      - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}
+      - MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
+      - MINIO_SECURE=${MINIO_SECURE:-false}
+      - PDF_DEFAULT_BUCKET=${PDF_DEFAULT_BUCKET:-assets}
+      - PDF_DEFAULT_NAMESPACE=${PDF_DEFAULT_NAMESPACE:-pmoves}
+      - PDF_MAX_PAGES=${PDF_MAX_PAGES:-0}
+      - PDF_INGEST_EXTRACT_URL=${PDF_INGEST_EXTRACT_URL:-http://extract-worker:8083/ingest}
+      - NATS_URL=${NATS_URL:-nats://nats:4222}
+    depends_on:
+      extract-worker:
+        condition: service_started
+      minio:
+        condition: service_started
+    ports: ["8092:8092"]
+    profiles: ["workers","orchestration"]
+    networks: [pmoves]
   langextract:
     build:
       context: .

pmoves/docs/DATA_IMPORT.md

@@ -20,6 +20,32 @@ The loader:
 - Embeds `text` via SentenceTransformers and upserts to Qdrant collection `${QDRANT_COLLECTION}`.
 - Indexes documents into Meilisearch index `${QDRANT_COLLECTION}` when available.
 
+## PDF ingestion from MinIO / S3
+The `pdf-ingest` service downloads PDFs stored in MinIO/S3, extracts paragraphs via `libs.langextract`, and forwards chunks to the extract worker so they land in Qdrant/Meilisearch.
+
+1. Ensure the core stack is running (`make up`). The workers profile now launches `pdf-ingest` alongside langextract and extract-worker.
+2. Upload a PDF to MinIO (or any S3-compatible bucket). A presign shortcut:
+   ```bash
+   curl -s -X POST http://localhost:8088/presign/put \
+     -H 'content-type: application/json' \
+     -H "Authorization: Bearer ${PRESIGN_SHARED_SECRET:-change_me}" \
+     -d '{"bucket":"assets","key":"pdfs/sample.pdf","content_type":"application/pdf","expires":300}' \
+     | jq -r '.url' \
+     | xargs -I {} curl -s -X PUT -H 'Content-Type: application/pdf' --data-binary @pmoves/datasets/sample.pdf '{}' >/dev/null
+   ```
+3. Trigger ingestion:
+   ```bash
+   curl -s http://localhost:8092/pdf/ingest \
+     -H 'content-type: application/json' \
+     -d '{"bucket":"assets","key":"pdfs/sample.pdf","namespace":"pmoves","title":"Sample PDF"}' | jq .
+   ```
+
+The service emits `ingest.file.added.v1` and `ingest.document.ready.v1` events (via NATS) and mirrors any langextract errors into Supabase through `extract-worker`.
+
 ## Notes
 - Ensure the stack is up (`make up`).
 - Large files: run with enough memory and expect model download on first use.
+- Control PDF fan-out via env vars:
+  - `PDF_DEFAULT_BUCKET`, `PDF_DEFAULT_NAMESPACE` – fallbacks for the service request body.
+  - `PDF_MAX_PAGES` – limit pages processed per document (0 = all pages).
+  - `PDF_INGEST_EXTRACT_URL` – override the extract-worker endpoint when running standalone.

pmoves/services/pdf-ingest/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.11-slim
+WORKDIR /app
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app
+RUN pip install --no-cache-dir --upgrade pip
+COPY services/pdf-ingest/requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+COPY libs /app/libs
+COPY services/common /app/services/common
+COPY contracts /app/contracts
+COPY services/pdf-ingest/ /app/
+ENV PMOVES_CONTRACTS_DIR=/app/contracts
+EXPOSE 8092
+CMD ["uvicorn","app:app","--host","0.0.0.0","--port","8092"]