diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 138a96f398..07dd969d28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,11 +144,62 @@ jobs: name: backend-coverage token: ${{ secrets.CODECOV_TOKEN }} - # Job 3: Docker Build Test + # Job 3: Multi-Platform Server Build (OCR Dependencies) + # Tests that tesseract-ocr and poppler-utils install correctly on both architectures + platform-compatibility: + name: Platform Compatibility (Server + OCR) + strategy: + matrix: + include: + - runner: ubuntu-latest + platform: linux/amd64 + arch: x86_64 + - runner: ubuntu-24.04-arm + platform: linux/arm64 + arch: ARM64 + runs-on: ${{ matrix.runner }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build server image (${{ matrix.arch }}) + run: | + docker build \ + --file python/Dockerfile.server \ + --tag archon-server:${{ matrix.arch }} \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + python/ + + - name: Verify OCR dependencies (${{ matrix.arch }}) + run: | + echo "🔍 Testing OCR dependencies on ${{ matrix.arch }}..." + + # Test tesseract is installed and working + docker run --rm archon-server:${{ matrix.arch }} tesseract --version + + # Test poppler-utils (pdftoppm) is installed + docker run --rm archon-server:${{ matrix.arch }} pdftoppm -v + + # Test Python imports work + docker run --rm archon-server:${{ matrix.arch }} python -c " + import pytesseract + import pdf2image + print('✅ pytesseract imported successfully') + print('✅ pdf2image imported successfully') + print(f'Tesseract version: {pytesseract.get_tesseract_version()}') + " + + echo "✅ All OCR dependencies verified on ${{ matrix.arch }}" + + # Job 4: Docker Build Test (all services) docker-build-test: name: Docker Build Tests runs-on: ubuntu-latest - + strategy: matrix: service: [server, mcp, agents, frontend] @@ -241,11 +292,11 @@ jobs: docker stop test-${{ matrix.service }} || true docker rm test-${{ matrix.service }} || true - # Job 4: Test Results Summary + # Job 5: Test Results Summary test-summary: name: Test Results Summary runs-on: ubuntu-latest - needs: [frontend-tests, backend-tests, docker-build-test] + needs: [frontend-tests, backend-tests, platform-compatibility, docker-build-test] if: always() steps: @@ -281,6 +332,13 @@ jobs: fi echo "" >> $GITHUB_STEP_SUMMARY + # Platform Compatibility Results + echo "## 🖥️ Platform Compatibility" >> $GITHUB_STEP_SUMMARY + echo "Server image built and OCR dependencies verified on:" >> $GITHUB_STEP_SUMMARY + echo "- **x86_64** (ubuntu-latest)" >> $GITHUB_STEP_SUMMARY + echo "- **ARM64** (ubuntu-24.04-arm)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + # Docker Build Results echo "## 🐳 Docker Build Tests" >> $GITHUB_STEP_SUMMARY echo "Docker build tests completed - check individual job results" >> $GITHUB_STEP_SUMMARY diff --git a/.gitignore b/.gitignore index 7a5f4d0eff..7df80f2cb4 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,7 @@ UAT/ # Local release notes testing release-notes-*.md + +# Supabase local data +supabase/volumes/ +supabase/.temp/ diff --git a/INFRASTRUCTURE.md b/INFRASTRUCTURE.md new file mode 100644 index 0000000000..11fcab1bb3 --- /dev/null +++ b/INFRASTRUCTURE.md @@ -0,0 +1,545 @@ +# Archon Infrastructure Setup + +> Dokumentation der lokalen Entwicklungsumgebung mit Supabase und Archon +> +> **Erstellt**: 20. November 2025 +> **Status**: ✅ Produktiv +> **Letzte Aktualisierung**: 20. November 2025 + +--- + +## 📋 Übersicht + +Diese Dokumentation beschreibt die vollständige lokale Entwicklungsumgebung für Archon mit Supabase als Backend-Datenbank. + +### Komponenten + +- **Supabase** (lokal): PostgreSQL 17.6 mit allen Services +- **Archon**: KI-gestütztes Knowledge Management System + - Backend Server (FastAPI) + - MCP Server (Model Context Protocol) + - Frontend (React) + +--- + +## 🏗️ Architektur + +``` +┌─────────────────────────────────────────────────────────┐ +│ Docker Desktop │ +├─────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────────────────────────────────────────┐ │ +│ │ Supabase Stack │ │ +│ │ (verwaltet durch supabase CLI) │ │ +│ ├──────────────────────────────────────────────────┤ │ +│ │ - PostgreSQL 17.6 (Port 54322) │ │ +│ │ - Kong API Gateway (Port 54321) │ │ +│ │ - Supabase Studio (Port 54323) │ │ +│ │ - GoTrue Auth (intern) │ │ +│ │ - Storage API (intern) │ │ +│ │ - Realtime (intern) │ │ +│ │ - Edge Functions (intern) │ │ +│ │ - Vector/Analytics (intern) │ │ +│ └──────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────┐ │ +│ │ Archon Stack │ │ +│ │ (verwaltet durch docker compose) │ │ +│ ├──────────────────────────────────────────────────┤ │ +│ │ - archon-server (Port 8181) │ │ +│ │ - archon-mcp (Port 8051) │ │ +│ │ - archon-ui (Port 3737) │ │ +│ └──────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────┘ +``` + +--- + +## 🚀 Installation & Konfiguration + +### Voraussetzungen + +- Docker Desktop für Mac (läuft bereits) +- Homebrew (installiert) +- Git (installiert) + +### 1. Supabase CLI Installation + +```bash +brew install supabase/tap/supabase +# Version: 2.58.5 +``` + +### 2. Supabase Initialisierung + +```bash +cd /Volumes/DATEN/Coding/archon/supabase +supabase start +``` + +**Wichtig**: Dies erstellt Container mit dem Suffix `_supabase` (z.B. `supabase_db_supabase`) + +### 3. Datenbank-Schema anwenden + +```bash +cd /Volumes/DATEN/Coding/archon +docker exec -i supabase_db_supabase psql -U postgres -d postgres < migration/complete_setup.sql +``` + +Erstellt folgende Tabellen: +- `archon_code_examples` +- `archon_crawled_pages` +- `archon_document_versions` +- `archon_migrations` +- `archon_page_metadata` +- `archon_project_sources` +- `archon_projects` +- `archon_prompts` +- `archon_settings` +- `archon_sources` +- `archon_tasks` + +### 4. Umgebungsvariablen konfigurieren + +**Datei**: `/Volumes/DATEN/Coding/archon/.env` + +```bash +# Supabase Connection (für Docker Container) +SUPABASE_URL=http://host.docker.internal:54321 + +# JWT Service Role Key (generiert mit lokalem JWT_SECRET) +# WICHTIG: Generiere dein eigenes Token mit dem Script in Abschnitt "JWT-Token-Problem und Lösung" +SUPABASE_SERVICE_KEY= + +# Service Ports +ARCHON_SERVER_PORT=8181 +ARCHON_MCP_PORT=8051 +ARCHON_AGENTS_PORT=8052 +ARCHON_UI_PORT=3737 +ARCHON_DOCS_PORT=3838 +``` + +### 5. Archon Services starten + +```bash +cd /Volumes/DATEN/Coding/archon +docker compose up -d +``` + +--- + +## 🔑 Wichtige Authentifizierung-Details + +### JWT-Token-Problem und Lösung + +**Problem**: Die Standard-JWT-Tokens aus `supabase/.env` funktionieren nicht mit der laufenden Supabase-Instanz. + +**Ursache**: Supabase CLI generiert beim Start ein neues JWT_SECRET (`super-secret-jwt-token-with-at-least-32-characters-long`), das nicht mit den vordefinierten Tokens übereinstimmt. + +**Lösung**: JWT-Token mit dem korrekten Secret generieren: + +```python +import jwt +from datetime import datetime, timedelta + +secret = "super-secret-jwt-token-with-at-least-32-characters-long" + +service_role_payload = { + "role": "service_role", + "iss": "supabase", + "iat": int(datetime.now().timestamp()), + "exp": int((datetime.now() + timedelta(days=365*10)).timestamp()) +} + +token = jwt.encode(service_role_payload, secret, algorithm="HS256") +print(token) +``` + +**Wichtig**: Supabase Python Client (v2.15.1) benötigt JWT-Format, NICHT das neue `sb_secret_*` Format! + +--- + +## 📍 Zugriffspunkte + +### Supabase + +| Service | URL | Credentials | +|---------|-----|-------------| +| **Studio UI** | http://localhost:54323 | - | +| **API Gateway** | http://localhost:54321 | Service Key (siehe .env) | +| **PostgreSQL** | `postgresql://postgres:postgres@localhost:54322/postgres` | postgres/postgres | + +### Archon + +| Service | URL | Beschreibung | +|---------|-----|--------------| +| **UI** | http://localhost:3737 | Hauptanwendung | +| **API Server** | http://localhost:8181 | Backend API | +| **MCP Server** | http://localhost:8051 | Model Context Protocol | +| **Health Check** | http://localhost:8181/health | Server-Status | + +--- + +## 🛠️ Wartung & Verwaltung + +### Supabase Status prüfen + +```bash +cd /Volumes/DATEN/Coding/archon/supabase +supabase status +``` + +Zeigt: +- API URL +- Database URL +- Studio URL +- Publishable/Secret Keys +- Gestoppte Services + +### Container Status + +```bash +# Alle Container anzeigen +docker ps --format "table {{.Names}}\t{{.Status}}" + +# Nur Archon +docker ps --format "table {{.Names}}\t{{.Status}}" | grep archon + +# Nur Supabase +docker ps --format "table {{.Names}}\t{{.Status}}" | grep supabase +``` + +### Services neu starten + +**Supabase**: +```bash +cd /Volumes/DATEN/Coding/archon/supabase +supabase stop +supabase start +``` + +**Archon**: +```bash +cd /Volumes/DATEN/Coding/archon +docker compose restart +# oder für kompletten Neustart: +docker compose down && docker compose up -d +``` + +### Logs einsehen + +**Supabase**: +```bash +docker logs supabase_db_supabase -f +docker logs supabase_kong_supabase -f +``` + +**Archon**: +```bash +docker logs archon-server -f +docker logs archon-mcp -f +docker logs archon-ui -f +``` + +--- + +## 🐛 Troubleshooting + +### Problem: Container mit Status "Created" oder "Restarting" + +**Symptom**: Container ohne `_supabase` Suffix existieren und starten nicht. + +**Ursache**: Docker Compose hat versehentlich Supabase-Container erstellt (sollte nur Archon verwalten). + +**Lösung**: +```bash +# Fehlerhafte Container entfernen +docker rm -f supabase-db supabase-kong supabase-auth supabase-storage \ + supabase-studio supabase-rest supabase-analytics supabase-meta \ + supabase-edge-functions supabase-pooler supabase-vector \ + supabase-imgproxy realtime-dev.supabase-realtime + +# Überflüssiges Netzwerk entfernen +docker network rm supabase_default +``` + +### Problem: "Invalid API key" Fehler + +**Symptom**: `SupabaseException: Invalid API key` beim Start von archon-server. + +**Ursache**: Falsches JWT-Token-Format oder falsches Secret. + +**Lösung**: JWT-Token mit dem tatsächlichen Secret neu generieren (siehe Abschnitt "JWT-Token-Problem"). + +### Problem: Port-Konflikte + +**Symptom**: "Port already in use" beim Start. + +**Lösung**: +```bash +# Belegte Ports prüfen +lsof -i :54321 # Supabase API +lsof -i :8181 # Archon Server +lsof -i :3737 # Archon UI + +# Container stoppen +supabase stop +docker compose down +``` + +### Problem: Container "unhealthy" + +**Symptom**: Container läuft, aber Status zeigt "unhealthy". + +**Diagnose**: +```bash +# Logs prüfen +docker logs --tail 50 + +# Healthcheck-Details +docker inspect | grep -A 10 Health +``` + +**Häufige Ursachen**: +- Datenbankverbindung fehlgeschlagen → JWT-Token prüfen +- Port nicht erreichbar → Netzwerk-Konfiguration prüfen +- Service noch nicht bereit → Warten und Status erneut prüfen + +--- + +## 🗄️ Datenbank-Management + +### Direkter Zugriff + +```bash +# Via Docker +docker exec -it supabase_db_supabase psql -U postgres -d postgres + +# Via lokaler psql (wenn installiert) +psql -h localhost -p 54322 -U postgres -d postgres +``` + +### Backup erstellen + +```bash +# Vollständiges Backup +docker exec supabase_db_supabase pg_dump -U postgres postgres > backup_$(date +%Y%m%d_%H%M%S).sql + +# Nur Schema +docker exec supabase_db_supabase pg_dump -U postgres -s postgres > schema_backup.sql + +# Nur Daten +docker exec supabase_db_supabase pg_dump -U postgres -a postgres > data_backup.sql +``` + +### Backup wiederherstellen + +```bash +docker exec -i supabase_db_supabase psql -U postgres -d postgres < backup.sql +``` + +### Migration hinzufügen + +1. Neue Migration erstellen: +```bash +cd /Volumes/DATEN/Coding/archon/supabase +supabase migration new +``` + +2. SQL-Befehle in generierte Datei einfügen + +3. Migration anwenden: +```bash +docker exec -i supabase_db_supabase psql -U postgres -d postgres < supabase/migrations/_.sql +``` + +--- + +## 🔒 Sicherheit + +### Produktionsumgebung + +**WICHTIG**: Diese Konfiguration ist NUR für lokale Entwicklung geeignet! + +Für Produktion ändern: + +1. **JWT Secret** in `supabase/.env` ändern: +```bash +JWT_SECRET= +``` + +2. **Neue JWT-Tokens** generieren mit neuem Secret + +3. **PostgreSQL Passwort** ändern: +```bash +POSTGRES_PASSWORD= +``` + +4. **Dashboard Credentials** ändern: +```bash +DASHBOARD_USERNAME= +DASHBOARD_PASSWORD= +``` + +5. **Firewall-Regeln** konfigurieren (nur notwendige Ports öffnen) + +### API Keys sicher speichern + +Archon speichert sensible API Keys verschlüsselt in der Datenbank: +- OpenAI API Key +- Google API Key +- Anthropic API Key +- etc. + +Konfiguration über UI: http://localhost:3737/settings + +--- + +## 📊 Ressourcen-Übersicht + +### Docker Images (ca. 15.7 GB) + +**Archon** (5.69 GB): +- `archon-archon-server`: 3.77 GB +- `archon-archon-frontend`: 1.54 GB +- `archon-archon-mcp`: 385 MB + +**Supabase** (~10 GB): +- `public.ecr.aws/supabase/postgres:17.6.1.043`: 4.33 GB +- `public.ecr.aws/supabase/studio`: 1.2 GB +- `public.ecr.aws/supabase/storage-api`: 1.11 GB +- `public.ecr.aws/supabase/edge-runtime`: 1.07 GB +- `public.ecr.aws/supabase/logflare`: 1.02 GB +- `public.ecr.aws/supabase/realtime`: 659 MB +- `public.ecr.aws/supabase/postgrest`: 585 MB +- `public.ecr.aws/supabase/postgres-meta`: 568 MB +- `public.ecr.aws/supabase/kong`: 212 MB +- `public.ecr.aws/supabase/vector`: 160 MB +- `public.ecr.aws/supabase/gotrue`: 74 MB +- `public.ecr.aws/supabase/mailpit`: 43 MB + +### Laufende Container (15) + +**Archon** (3): +- archon-server +- archon-mcp +- archon-ui + +**Supabase** (12): +- supabase_db_supabase +- supabase_kong_supabase +- supabase_studio_supabase +- supabase_auth_supabase +- supabase_storage_supabase +- supabase_realtime_supabase +- supabase_rest_supabase +- supabase_vector_supabase +- supabase_analytics_supabase +- supabase_pg_meta_supabase +- supabase_edge_runtime_supabase +- supabase_inbucket_supabase + +--- + +## 📚 Referenzen + +### Offizielle Dokumentation + +- **Archon**: https://github.com/coleam00/Archon +- **Supabase**: https://supabase.com/docs +- **Supabase CLI**: https://supabase.com/docs/guides/cli + +### Wichtige Konfigurationsdateien + +- `/Volumes/DATEN/Coding/archon/.env` - Archon Umgebungsvariablen +- `/Volumes/DATEN/Coding/archon/docker-compose.yml` - Archon Services +- `/Volumes/DATEN/Coding/archon/supabase/.env` - Supabase Konfiguration +- `/Volumes/DATEN/Coding/archon/supabase/volumes/api/kong.yml` - Kong API Gateway +- `/Volumes/DATEN/Coding/archon/migration/complete_setup.sql` - Datenbank-Schema + +### Nützliche Befehle (Schnellreferenz) + +```bash +# Status prüfen +cd /Volumes/DATEN/Coding/archon +docker ps | grep -E "archon|supabase" +curl http://localhost:8181/health + +# Alles neu starten +cd supabase && supabase stop && supabase start +cd .. && docker compose restart + +# Alles stoppen +docker compose down +cd supabase && supabase stop + +# Logs verfolgen +docker compose logs -f +docker logs archon-server -f + +# Datenbank abfragen +docker exec -it supabase_db_supabase psql -U postgres -d postgres +``` + +--- + +## 🎯 Nächste Schritte + +Nach erfolgreicher Installation: + +1. **API Keys konfigurieren** unter http://localhost:3737/settings + - OpenAI API Key (für Embeddings) + - Optional: Google, Anthropic, etc. + +2. **Knowledge Base befüllen**: + - Dokumente hochladen + - Websites crawlen + +3. **MCP Server mit IDE verbinden**: + - Claude Code: `claude mcp add --transport http archon http://localhost:8051/mcp` + - Cursor/Windsurf: Siehe http://localhost:3737/mcp + +4. **Projekte & Tasks erstellen** (optional, wenn Feature aktiviert) + +--- + +## ✅ Verifizierung + +Alle Services sollten diesen Status zeigen: + +```bash +$ docker ps --format "table {{.Names}}\t{{.Status}}" | grep -E "archon|supabase" + +archon-mcp Up X minutes (healthy) +archon-ui Up X minutes (healthy) +archon-server Up X minutes (healthy) +supabase_studio_supabase Up X minutes (healthy) +supabase_db_supabase Up X minutes (healthy) +supabase_kong_supabase Up X minutes (healthy) +# ... weitere Supabase Services (alle healthy) +``` + +Healthcheck-Endpunkte: +- ✅ http://localhost:8181/health → `{"status":"healthy",...}` +- ✅ http://localhost:3737 → Archon UI lädt +- ✅ http://localhost:54323 → Supabase Studio lädt + +--- + +## 📝 Änderungsprotokoll + +### 2025-11-20 - Initiale Einrichtung +- Supabase CLI 2.58.5 installiert +- PostgreSQL 17.6 mit Archon-Schema konfiguriert +- JWT-Token-Problem identifiziert und gelöst +- Alle Docker Container bereinigt (Duplikate entfernt) +- Playwright-Tests durchgeführt (alle erfolgreich) +- Status: ✅ Produktiv + +--- + +**Maintainer**: Mathias Boni +**Zuletzt getestet**: 20. November 2025 +**Archon Version**: 0.1.0 diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000000..4355a3277c --- /dev/null +++ b/PLAN.md @@ -0,0 +1,344 @@ +# Auth-Token Support für Ollama Chat & Embedding Instanzen - Implementierungsplan + +## Status: ✅ VOLLSTÄNDIG ABGESCHLOSSEN UND DEPLOYED + +## Kontext + +**Aktuelle Situation:** +- Frontend hat ZWEI separate Ollama-Konfigurationen: + - **LLM/Chat**: Gespeichert in `LLM_BASE_URL` (rag_strategy) + - **Embedding**: Gespeichert in `OLLAMA_EMBEDDING_URL` (rag_strategy) +- Backend liest diese URLs und erstellt OpenAI-kompatible Clients +- **PROBLEM GELÖST**: Auth-Token-Unterstützung für geschützte Ollama-Instanzen implementiert + +## Ziel ✅ ERREICHT + +Für BEIDE Ollama-Instanzen (Chat & Embedding) optionale Auth-Token-Felder hinzugefügt: +- ✅ Checkbox "Use Authentication" in jedem Modal +- ✅ Password-Input für Auth-Token (nur sichtbar wenn Checkbox aktiviert) +- ✅ Backend nutzt die korrekten Token basierend auf Operation (Chat vs. Embedding) + +## Implementierte Änderungen + +### ✅ 1. Frontend: RAGSettings.tsx erweitert + +**Datei**: `archon-ui-main/src/components/settings/RAGSettings.tsx` + +**State-Management (Zeile 207-219):** +```typescript +const [llmInstanceConfig, setLLMInstanceConfig] = useState({ + name: '', + url: ragSettings.LLM_BASE_URL || 'http://host.docker.internal:11434/v1', + useAuth: false, + authToken: '' +}); + +const [embeddingInstanceConfig, setEmbeddingInstanceConfig] = useState({ + name: '', + url: ragSettings.OLLAMA_EMBEDDING_URL || 'http://host.docker.internal:11434/v1', + useAuth: false, + authToken: '' +}); +``` + +**useEffect Hooks (Zeile 226-270):** +- ✅ Lädt `OLLAMA_CHAT_AUTH_TOKEN` aus ragSettings +- ✅ Lädt `OLLAMA_EMBEDDING_AUTH_TOKEN` aus ragSettings +- ✅ Setzt `useAuth` Checkbox automatisch basierend auf vorhandenem Token + +**Edit LLM Instance Modal (Zeile 2209-2232):** +- ✅ Checkbox "Use Authentication" +- ✅ Conditional Password-Input für Auth-Token +- ✅ Beim Speichern (Zeile 2244-2250): Speichert `OLLAMA_CHAT_AUTH_TOKEN` in ragSettings + +**Edit Embedding Instance Modal (Zeile 2299-2322):** +- ✅ Checkbox "Use Authentication" +- ✅ Conditional Password-Input für Auth-Token +- ✅ Beim Speichern (Zeile 2334-2340): Speichert `OLLAMA_EMBEDDING_AUTH_TOKEN` in ragSettings + +### ✅ 2. Backend: llm_provider_service.py angepasst + +**Datei**: `python/src/server/services/llm_provider_service.py` + +**Funktion `get_llm_client()` - Hauptimplementierung (Zeile 455-459):** +```python +# Get correct auth token based on operation type +if use_embedding_provider or instance_type == "embedding": + ollama_auth_token = rag_settings.get("OLLAMA_EMBEDDING_AUTH_TOKEN", "ollama") +else: + ollama_auth_token = rag_settings.get("OLLAMA_CHAT_AUTH_TOKEN", "ollama") +``` + +**Fallback-Code (Zeile 422-426):** +```python +# Get correct auth token based on operation type +if use_embedding_provider: + ollama_auth_token = rag_settings.get("OLLAMA_EMBEDDING_AUTH_TOKEN", "ollama") +else: + ollama_auth_token = rag_settings.get("OLLAMA_CHAT_AUTH_TOKEN", "ollama") +``` + +### ✅ 3. Datenbank-Schema + +**KEINE Änderungen nötig!** +- ✅ Nutzt existierende `archon_settings` Tabelle +- ✅ Neue Keys werden automatisch gespeichert: + - `OLLAMA_CHAT_AUTH_TOKEN` (Kategorie: rag_strategy) + - `OLLAMA_EMBEDDING_AUTH_TOKEN` (Kategorie: rag_strategy) + +## Deployment Status + +### ✅ 4. Frontend Build + +```bash +cd archon-ui-main +npm run build +``` + +**Status**: ✅ ABGESCHLOSSEN + +### ✅ 5. Docker Images neu bauen und deployen + +```bash +cd /Volumes/DATEN/Coding/INFRASTRUCTURE_PROJECT/archon-local_supabase/archon +docker compose down +docker compose build --no-cache +docker compose up -d +``` + +**Status**: ✅ ABGESCHLOSSEN + +**Deployment Zeitpunkt**: 2025-11-20 + +**Laufende Services**: +- ✅ `archon-server` (Port 8181) - healthy +- ✅ `archon-mcp` (Port 8051) - running +- ✅ `archon-ui` (Port 3737) - running + +### 🧪 6. Testing + +**Bereit zum Testen!** Das System ist deployed und läuft. + +**Test-Anleitung**: + +1. **UI öffnen**: http://localhost:3737 +2. **Settings öffnen** → RAG Settings Tab +3. **LLM Instance konfigurieren**: + - Klicke auf "Edit" bei der LLM Instance + - Aktiviere "Use Authentication" Checkbox + - Trage dein Ollama Auth-Token ein + - Speichern +4. **Embedding Instance konfigurieren**: + - Klicke auf "Edit" bei der Embedding Instance + - Aktiviere "Use Authentication" Checkbox + - Trage dein Ollama Auth-Token ein (kann unterschiedlich sein) + - Speichern +5. **RAG-Funktionalität testen**: + - Starte einen Crawl oder Search + - Verifiziere, dass die geschützte Ollama-Instanz verwendet wird +6. **Backend-Logs prüfen** (optional): + ```bash + docker compose logs -f archon-server | grep -i "ollama\|auth" + ``` + +**Erwartetes Verhalten**: +- ✅ Auth-Token wird als Bearer Token im Authorization Header gesendet +- ✅ Ollama-Instanz akzeptiert authentifizierte Requests +- ✅ Ohne Auth-Token: Placeholder "required-but-ignored" wird verwendet (abwärtskompatibel) + +## Update: 2025-11-20 - Health-Check & Summary Fixes + +### Problem +Nach dem initialen Deployment wurden zwei Probleme identifiziert: +1. **Health-Check zeigt "Offline"**: Der Health-Check-Endpoint verwendete kein Auth-Token für geschützte Instanzen +2. **Auth-Token nicht sichtbar in Summary**: Die Summary-Tabelle zeigte nicht an, ob ein Auth-Token konfiguriert ist + +### Implementierte Fixes + +#### ✅ Frontend: Auth-Token Status in Summary anzeigen +**Datei**: `archon-ui-main/src/components/settings/RAGSettings.tsx` (Zeile 1723-1750) + +Neue Zeile in der Summary-Tabelle zwischen "Instance URL" und "Status": +```typescript + + Authentication + + {activeSelection === 'chat' ? ( + llmInstanceConfig.authToken ? ( + + + + + Token configured + + ) : ( + No authentication + ) + ) : ( + // Gleiche Logik für Embedding Instance + )} + + +``` + +#### ✅ Backend: Health-Check mit Auth-Token Support + +**Datei 1**: `python/src/server/services/ollama/model_discovery_service.py` (Zeile 958-993) + +Erweiterte `check_instance_health()` Methode um optionalen `auth_token` Parameter: +```python +async def check_instance_health(self, instance_url: str, auth_token: str | None = None) -> InstanceHealthStatus: + # Prepare headers with optional auth token + headers = {} + if auth_token: + headers["Authorization"] = f"Bearer {auth_token}" + + async with httpx.AsyncClient(timeout=httpx.Timeout(10)) as client: + ping_url = f"{instance_url.rstrip('/')}/api/tags" + response = await client.get(ping_url, headers=headers) + # ... +``` + +**Datei 2**: `python/src/server/api_routes/ollama_api.py` (Zeile 142-199) + +Health-Check-Endpoint liest Auth-Tokens aus RAG Settings: +```python +@router.get("/instances/health") +async def health_check_endpoint( + instance_urls: list[str] = Query(...), + include_models: bool = Query(False) +) -> dict[str, Any]: + # Get auth tokens from RAG settings + rag_settings = await credential_service.get_credentials_by_category("rag_strategy") + + llm_base_url = rag_settings.get("LLM_BASE_URL", "").replace("/v1", "").rstrip("/") + embedding_base_url = rag_settings.get("OLLAMA_EMBEDDING_URL", "").replace("/v1", "").rstrip("/") + + chat_auth_token = rag_settings.get("OLLAMA_CHAT_AUTH_TOKEN", "") + embedding_auth_token = rag_settings.get("OLLAMA_EMBEDDING_AUTH_TOKEN", "") + + # Determine which auth token to use based on URL matching + for instance_url in instance_urls: + url = instance_url.rstrip('/') + auth_token = None + if url == llm_base_url and chat_auth_token: + auth_token = chat_auth_token + elif url == embedding_base_url and embedding_auth_token: + auth_token = embedding_auth_token + + health_status = await model_discovery_service.check_instance_health(url, auth_token=auth_token) + # ... +``` + +### ✅ Deployment (2025-11-20 16:00) + +- ✅ Frontend neu gebaut +- ✅ Docker Images neu gebaut (Frontend, Server, MCP) +- ✅ Container neu deployed +- ✅ Alle Services laufen: archon-server (healthy), archon-mcp (healthy), archon-ui (running) + +### Erwartetes Verhalten (nach Fix) + +1. **Summary zeigt Auth-Token Status**: + - ✅ "Token configured" mit Schloss-Icon wenn Token gesetzt + - ✅ "No authentication" wenn kein Token + +2. **Health-Check funktioniert mit Auth**: + - ✅ Backend sendet Bearer Token im Authorization Header + - ✅ Health-Check sollte jetzt "Online" zeigen für geschützte Instanzen + - ✅ Status-Anfrage erfolgt automatisch beim Öffnen der RAG Settings + +### ✅ Final Verification (2025-11-20 22:05) + +**Datenbank-Status**: Alle erforderlichen Settings sind gespeichert: +```sql + key | value | category +-----------------------------+-----------------------------------------+-------------- + LLM_BASE_URL | https://your-ollama-instance.example | rag_strategy + OLLAMA_CHAT_AUTH_TOKEN | ollama_xxx_placeholder_token | rag_strategy + OLLAMA_EMBEDDING_AUTH_TOKEN | ollama_xxx_placeholder_token | rag_strategy + OLLAMA_EMBEDDING_URL | https://your-ollama-instance.example | rag_strategy +``` + +**Health-Check-Test**: Erfolgreiche Authentifizierung mit geschützter Instanz: +```json +{ + "summary": { + "total_instances": 1, + "healthy_instances": 1, + "unhealthy_instances": 0, + "average_response_time_ms": 672.12 + }, + "instance_status": { + "https://your-ollama-instance.example": { + "is_healthy": true, + "response_time_ms": 672.12, + "models_available": 5, + "error_message": null + } + } +} +``` + +**Ergebnis**: ✅ **VOLLSTÄNDIG FUNKTIONSFÄHIG** +- Health-Check zeigt "Online" Status +- 5 Modelle erfolgreich erkannt +- Bearer Token-Authentifizierung funktioniert +- Response-Zeit: ~672ms (akzeptabel) + +## Geänderte Dateien (Gesamt) + +1. ✅ `archon-ui-main/src/components/settings/RAGSettings.tsx` (Initial + Summary-Fix) +2. ✅ `python/src/server/services/llm_provider_service.py` (Token-Auswahl) +3. ✅ `python/src/server/services/ollama/model_discovery_service.py` (Health-Check Auth) +4. ✅ `python/src/server/api_routes/ollama_api.py` (Health-Check Endpoint) + +## Technische Details + +### Frontend → Backend Datenfluss + +1. **User füllt Modal aus**: + - URL: `http://my-ollama:11434` + - Checkbox "Use Authentication": ✓ + - Auth Token: `my-secret-token` + +2. **Frontend speichert in archon_settings**: + ```json + { + "LLM_BASE_URL": "http://my-ollama:11434", + "OLLAMA_CHAT_AUTH_TOKEN": "my-secret-token" + } + ``` + +3. **Backend liest und verwendet**: + ```python + # In llm_provider_service.py + ollama_base_url = await _get_optimal_ollama_instance() # → "http://my-ollama:11434/v1" + ollama_auth_token = rag_settings.get("OLLAMA_CHAT_AUTH_TOKEN", "ollama") # → "my-secret-token" + + client = openai.AsyncOpenAI( + api_key=ollama_auth_token, # ← Wird als Bearer Token im HTTP Header verwendet + base_url=ollama_base_url + ) + ``` + +### Sicherheit + +- ✅ Token-Felder sind `type="password"` (versteckte Eingabe) +- ✅ Token wird nur gespeichert wenn Checkbox aktiviert ist +- ✅ Leerer Token = kein Auth-Header (abwärtskompatibel) +- ⚠️ Token wird im Klartext in `archon_settings` gespeichert (Future: Verschlüsselung) + +## Abwärtskompatibilität + +✅ **100% kompatibel mit bestehenden Installationen:** +- Ohne Auth-Token: Standard-Wert `"ollama"` wird verwendet +- Bestehende Instanzen funktionieren weiterhin +- Neue Felder sind optional + +## Lessons Learned + +1. ✅ Keine DB-Schema-Änderungen nötig bei generischen Key-Value-Tabellen +2. ✅ TypeScript `as any` für neue Settings-Keys akzeptabel während Entwicklung +3. ✅ Separate Token für Chat/Embedding ermöglicht flexible Deployment-Szenarien +4. ✅ useEffect Hooks müssen Token in Dependencies aufnehmen für korrektes Laden diff --git a/QUICKSTART.md b/QUICKSTART.md new file mode 100644 index 0000000000..b42dd806d3 --- /dev/null +++ b/QUICKSTART.md @@ -0,0 +1,47 @@ +# Archon + Supabase - Quick Start Guide + +> Schnellstart für die lokale Entwicklungsumgebung + +## ⚡ Schnellstart (für erfahrene Entwickler) + +```bash +# 1. Supabase starten +cd /Volumes/DATEN/Coding/archon/supabase +supabase start + +# 2. Archon starten +cd /Volumes/DATEN/Coding/archon +docker compose up -d + +# 3. Status prüfen +docker ps | grep -E "archon|supabase" +curl http://localhost:8181/health +``` + +## 🌐 Zugriff + +| Service | URL | Beschreibung | +|---------|-----|-------------| +| **Archon UI** | http://localhost:3737 | Hauptanwendung | +| **Supabase Studio** | http://localhost:54323 | Datenbank-UI | +| **API Server** | http://localhost:8181 | Backend API | +| **MCP Server** | http://localhost:8051/mcp | Model Context Protocol | + +## 🛑 Stoppen + +```bash +# Archon stoppen +docker compose down + +# Supabase stoppen +cd supabase && supabase stop +``` + +## 📚 Vollständige Dokumentation + +Siehe **[INFRASTRUCTURE.md](./INFRASTRUCTURE.md)** für: +- Detaillierte Installation +- Troubleshooting +- Datenbank-Management +- Sicherheitshinweise +- Backup/Restore diff --git a/archon-ui-main/.gitignore b/archon-ui-main/.gitignore index 202ae5886a..585c8c7a9a 100644 --- a/archon-ui-main/.gitignore +++ b/archon-ui-main/.gitignore @@ -16,6 +16,7 @@ dist-ssr coverage .nyc_output public/test-results +test-results/ test-results.json # Editor directories and files diff --git a/archon-ui-main/package-lock.json b/archon-ui-main/package-lock.json index 74f7568ea8..e03d108eb5 100644 --- a/archon-ui-main/package-lock.json +++ b/archon-ui-main/package-lock.json @@ -43,6 +43,7 @@ }, "devDependencies": { "@biomejs/biome": "2.2.2", + "@playwright/test": "^1.56.1", "@tailwindcss/postcss": "4.1.2", "@tailwindcss/vite": "4.1.2", "@testing-library/jest-dom": "^6.4.6", @@ -2390,6 +2391,22 @@ "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==", "license": "MIT" }, + "node_modules/@playwright/test": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.56.1.tgz", + "integrity": "sha512-vSMYtL/zOcFpvJCW71Q/OEGQb7KYBPAdKh35WNSkaZA75JlAO8ED8UN6GUNTm3drWomcbcqRPFqQbLae8yBTdg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.56.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@polka/url": { "version": "1.0.0-next.29", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", @@ -9753,6 +9770,53 @@ "dev": true, "license": "MIT" }, + "node_modules/playwright": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.56.1.tgz", + "integrity": "sha512-aFi5B0WovBHTEvpM3DzXTUaeN6eN0qWnTkKx4NQaH4Wvcmc153PdaY2UBdSYKaGYw+UyWXSVyxDUg5DoPEttjw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.56.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.56.1.tgz", + "integrity": "sha512-hutraynyn31F+Bifme+Ps9Vq59hKuUCz7H1kDOcBs+2oGguKkWTU50bBWrtz34OUWmIwpBTWDxaRPXrIXkgvmQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", diff --git a/archon-ui-main/package.json b/archon-ui-main/package.json index 9e1b4e642d..58fa6e671e 100644 --- a/archon-ui-main/package.json +++ b/archon-ui-main/package.json @@ -25,6 +25,11 @@ "test:coverage:stream": "vitest run --coverage --reporter=default --reporter=json --bail=false || true", "test:coverage:summary": "echo '\\n📊 ARCHON TEST & COVERAGE SUMMARY\\n═══════════════════════════════════════\\n' && node -e \"try { const data = JSON.parse(require('fs').readFileSync('coverage/test-results.json', 'utf8')); const passed = data.numPassedTests || 0; const failed = data.numFailedTests || 0; const total = data.numTotalTests || 0; const suites = data.numTotalTestSuites || 0; console.log('Test Suites: ' + (failed > 0 ? '\\x1b[31m' + failed + ' failed\\x1b[0m, ' : '') + '\\x1b[32m' + (suites - failed) + ' passed\\x1b[0m, ' + suites + ' total'); console.log('Tests: ' + (failed > 0 ? '\\x1b[31m' + failed + ' failed\\x1b[0m, ' : '') + '\\x1b[32m' + passed + ' passed\\x1b[0m, ' + total + ' total'); console.log('\\n✨ Results saved to coverage/test-results.json'); } catch(e) { console.log('⚠️ No test results found. Run tests first!'); }\" || true", "test:coverage:force": "vitest run --coverage --passWithNoTests || true", + "test:e2e": "playwright test", + "test:e2e:ui": "playwright test --ui", + "test:e2e:headed": "playwright test --headed", + "test:e2e:debug": "playwright test --debug", + "playwright:install": "playwright install chromium", "seed:projects": "node --loader ts-node/esm ../scripts/seed-project-data.ts" }, "dependencies": { @@ -63,6 +68,7 @@ }, "devDependencies": { "@biomejs/biome": "2.2.2", + "@playwright/test": "^1.56.1", "@tailwindcss/postcss": "4.1.2", "@tailwindcss/vite": "4.1.2", "@testing-library/jest-dom": "^6.4.6", diff --git a/archon-ui-main/playwright.config.ts b/archon-ui-main/playwright.config.ts new file mode 100644 index 0000000000..5596be25ee --- /dev/null +++ b/archon-ui-main/playwright.config.ts @@ -0,0 +1,46 @@ +import { defineConfig, devices } from "@playwright/test"; + +/** + * Playwright E2E Test Configuration + * @see https://playwright.dev/docs/test-configuration + */ +export default defineConfig({ + testDir: "./tests/e2e", + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 1 : undefined, + reporter: [ + ["html", { open: "never" }], + ["list"], + ], + use: { + baseURL: "http://localhost:3737", + trace: "on-first-retry", + screenshot: "only-on-failure", + }, + + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + // Uncomment for cross-browser testing + // { + // name: "firefox", + // use: { ...devices["Desktop Firefox"] }, + // }, + // { + // name: "webkit", + // use: { ...devices["Desktop Safari"] }, + // }, + ], + + // Run local dev server before tests (optional - comment out if running manually) + // webServer: { + // command: "npm run dev", + // url: "http://localhost:3737", + // reuseExistingServer: !process.env.CI, + // timeout: 120000, + // }, +}); diff --git a/archon-ui-main/src/components/DisconnectScreenOverlay.tsx b/archon-ui-main/src/components/DisconnectScreenOverlay.tsx index 11f6e6658e..9424fd3b92 100644 --- a/archon-ui-main/src/components/DisconnectScreenOverlay.tsx +++ b/archon-ui-main/src/components/DisconnectScreenOverlay.tsx @@ -1,5 +1,5 @@ import React, { useState } from 'react'; -import { X, Wifi, WifiOff } from 'lucide-react'; +import { X } from 'lucide-react'; import { DisconnectScreen } from './animations/DisconnectScreenAnimations'; import { NeonButton } from './ui/NeonButton'; diff --git a/archon-ui-main/src/components/agent-chat/ArchonChatPanel.tsx b/archon-ui-main/src/components/agent-chat/ArchonChatPanel.tsx index 4d72a6e1a6..f36c6e73ef 100644 --- a/archon-ui-main/src/components/agent-chat/ArchonChatPanel.tsx +++ b/archon-ui-main/src/components/agent-chat/ArchonChatPanel.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useState, useRef } from 'react'; -import { Send, User, WifiOff, RefreshCw, BookOpen, Search } from 'lucide-react'; +import { Send, User, WifiOff, RefreshCw } from 'lucide-react'; import { ArchonLoadingSpinner, EdgeLitEffect } from '../animations/Animations'; import { agentChatService, ChatMessage } from '../../services/agentChatService'; diff --git a/archon-ui-main/src/components/bug-report/BugReportModal.tsx b/archon-ui-main/src/components/bug-report/BugReportModal.tsx index 2bfcb00797..43b5496f76 100644 --- a/archon-ui-main/src/components/bug-report/BugReportModal.tsx +++ b/archon-ui-main/src/components/bug-report/BugReportModal.tsx @@ -1,6 +1,6 @@ import { useState } from "react"; import { motion, AnimatePresence } from "framer-motion"; -import { Bug, X, Send, Copy, ExternalLink, Loader } from "lucide-react"; +import { Bug, X, Send, Copy, Loader } from "lucide-react"; import { Button } from "../ui/Button"; import { Input } from "../ui/Input"; import { Card } from "../ui/Card"; diff --git a/archon-ui-main/src/components/bug-report/ErrorBoundaryWithBugReport.tsx b/archon-ui-main/src/components/bug-report/ErrorBoundaryWithBugReport.tsx index 6091d72657..9ae08cde50 100644 --- a/archon-ui-main/src/components/bug-report/ErrorBoundaryWithBugReport.tsx +++ b/archon-ui-main/src/components/bug-report/ErrorBoundaryWithBugReport.tsx @@ -1,4 +1,4 @@ -import React, { Component, ErrorInfo, ReactNode } from "react"; +import { Component, ErrorInfo, ReactNode } from "react"; import { AlertCircle, Bug, RefreshCw } from "lucide-react"; import { Button } from "../ui/Button"; import { Card } from "../ui/Card"; diff --git a/archon-ui-main/src/components/code/CodeViewerModal.tsx b/archon-ui-main/src/components/code/CodeViewerModal.tsx index bbf9d7ef76..659a974e89 100644 --- a/archon-ui-main/src/components/code/CodeViewerModal.tsx +++ b/archon-ui-main/src/components/code/CodeViewerModal.tsx @@ -6,7 +6,6 @@ import { Copy, Check, Code as CodeIcon, - FileText, TagIcon, Info, Search, diff --git a/archon-ui-main/src/components/settings/APIKeysSection.tsx b/archon-ui-main/src/components/settings/APIKeysSection.tsx index 0d92601448..91e6959e37 100644 --- a/archon-ui-main/src/components/settings/APIKeysSection.tsx +++ b/archon-ui-main/src/components/settings/APIKeysSection.tsx @@ -1,6 +1,5 @@ import { useState, useEffect } from 'react'; -import { Key, Plus, Trash2, Save, Lock, Unlock, Eye, EyeOff } from 'lucide-react'; -import { Input } from '../ui/Input'; +import { Plus, Trash2, Save, Lock, Unlock, Eye, EyeOff } from 'lucide-react'; import { Button } from '../ui/Button'; import { Card } from '../ui/Card'; import { credentialsService, Credential } from '../../services/credentialsService'; diff --git a/archon-ui-main/src/components/settings/OllamaConfigurationPanel.tsx b/archon-ui-main/src/components/settings/OllamaConfigurationPanel.tsx index 4da6f9a0de..a1bba2233e 100644 --- a/archon-ui-main/src/components/settings/OllamaConfigurationPanel.tsx +++ b/archon-ui-main/src/components/settings/OllamaConfigurationPanel.tsx @@ -35,6 +35,8 @@ const OllamaConfigurationPanel: React.FC = ({ const [newInstanceUrl, setNewInstanceUrl] = useState(''); const [newInstanceName, setNewInstanceName] = useState(''); const [newInstanceType, setNewInstanceType] = useState<'chat' | 'embedding'>('chat'); + const [newInstanceUseAuth, setNewInstanceUseAuth] = useState(false); + const [newInstanceAuthToken, setNewInstanceAuthToken] = useState(''); const [showAddInstance, setShowAddInstance] = useState(false); const [discoveringModels, setDiscoveringModels] = useState(false); const [modelDiscoveryResults, setModelDiscoveryResults] = useState(null); @@ -233,7 +235,8 @@ const OllamaConfigurationPanel: React.FC = ({ isEnabled: true, isPrimary: false, loadBalancingWeight: 100, - instanceType: separateHosts ? newInstanceType : 'both' + instanceType: separateHosts ? newInstanceType : 'both', + ...(newInstanceUseAuth && newInstanceAuthToken.trim() && { authToken: newInstanceAuthToken.trim() }) }; try { @@ -246,6 +249,8 @@ const OllamaConfigurationPanel: React.FC = ({ setNewInstanceUrl(''); setNewInstanceName(''); setNewInstanceType('chat'); + setNewInstanceUseAuth(false); + setNewInstanceAuthToken(''); setShowAddInstance(false); showToast(`Added new Ollama instance: ${newInstance.name}`, 'success'); @@ -703,8 +708,8 @@ const OllamaConfigurationPanel: React.FC = ({ size="sm" onClick={() => setNewInstanceType('chat')} className={cn( - newInstanceType === 'chat' - ? 'bg-blue-600 text-white' + newInstanceType === 'chat' + ? 'bg-blue-600 text-white' : 'text-blue-600 border-blue-600' )} > @@ -715,8 +720,8 @@ const OllamaConfigurationPanel: React.FC = ({ size="sm" onClick={() => setNewInstanceType('embedding')} className={cn( - newInstanceType === 'embedding' - ? 'bg-blue-600 text-white' + newInstanceType === 'embedding' + ? 'bg-blue-600 text-white' : 'text-blue-600 border-blue-600' )} > @@ -725,7 +730,29 @@ const OllamaConfigurationPanel: React.FC = ({ )} - + + {/* Authentication Settings */} +
+ + {newInstanceUseAuth && ( + setNewInstanceAuthToken(e.target.value)} + className="text-sm" + /> + )} +
+
+ +
+ + {activeSelection === 'chat' ? ( // Chat Model Configuration
- {llmInstanceConfig.name && llmInstanceConfig.url ? ( + {llmInstanceConfig.url ? ( <>
-
{llmInstanceConfig.name}
+
{llmInstanceConfig.name || 'LLM Instance'}
{llmInstanceConfig.url}
@@ -1600,10 +1754,10 @@ const manualTestConnection = async ( ) : ( // Embedding Model Configuration
- {embeddingInstanceConfig.name && embeddingInstanceConfig.url ? ( + {embeddingInstanceConfig.url ? ( <>
-
{embeddingInstanceConfig.name}
+
{embeddingInstanceConfig.name || 'Embedding Instance'}
{embeddingInstanceConfig.url}
@@ -1710,6 +1864,34 @@ const manualTestConnection = async ( } + + Authentication + + {activeSelection === 'chat' ? ( + llmInstanceConfig.authToken ? ( + + + + + Token configured + + ) : ( + No authentication + ) + ) : ( + embeddingInstanceConfig.authToken ? ( + + + + + Token configured + + ) : ( + No authentication + ) + )} + + Status @@ -1726,11 +1908,16 @@ const manualTestConnection = async ( Selected Model - - {activeSelection === 'chat' - ? (getDisplayedChatModel(ragSettings) || No model selected) - : (getDisplayedEmbeddingModel(ragSettings) || No model selected) - } + + {activeSelection === 'chat' ? ( + + {getDisplayedChatModel(ragSettings) || Not selected} + + ) : ( + + {getDisplayedEmbeddingModel(ragSettings) || Not selected} + + )} @@ -2165,7 +2352,9 @@ const manualTestConnection = async ( if (!embeddingInstanceConfig.url || !embeddingInstanceConfig.name) { setEmbeddingInstanceConfig({ name: llmInstanceConfig.name || 'Default Ollama', - url: newUrl + url: newUrl, + useAuth: false, + authToken: '' }); } }} @@ -2180,11 +2369,8 @@ const manualTestConnection = async ( checked={llmInstanceConfig.url === embeddingInstanceConfig.url && llmInstanceConfig.url !== ''} onChange={(e) => { if (e.target.checked) { - // Sync embedding instance with LLM instance - setEmbeddingInstanceConfig({ - name: llmInstanceConfig.name || 'Default Ollama', - url: llmInstanceConfig.url - }); + // Sync embedding instance with LLM instance (including auth settings) + setEmbeddingInstanceConfig(syncEmbeddingFromLLM(llmInstanceConfig)); } }} className="w-4 h-4 text-purple-600 bg-gray-100 border-gray-300 rounded focus:ring-purple-500 dark:focus:ring-purple-600 dark:ring-offset-gray-800 focus:ring-2 dark:bg-gray-700 dark:border-gray-600" @@ -2193,6 +2379,31 @@ const manualTestConnection = async ( Use same host for embedding instance
+ + {/* Authentication Settings */} +
+
+ setLLMInstanceConfig({...llmInstanceConfig, useAuth: e.target.checked})} + className="w-4 h-4 text-green-600 bg-gray-100 border-gray-300 rounded focus:ring-green-500 dark:focus:ring-green-600 dark:ring-offset-gray-800 focus:ring-2 dark:bg-gray-700 dark:border-gray-600" + /> + +
+ {llmInstanceConfig.useAuth && ( + setLLMInstanceConfig({...llmInstanceConfig, authToken: e.target.value})} + className="text-sm" + /> + )} +
@@ -2205,7 +2416,12 @@ const manualTestConnection = async (
@@ -2265,7 +2506,12 @@ const manualTestConnection = async (
', - code_content, - re.DOTALL, - ) - if cm_lines: - # Clean each line and join - cleaned_lines = [] - for line in cm_lines: - # Remove span tags but keep content - line = re.sub(r"]*>", "", line) - line = re.sub(r"", "", line) - # Remove other HTML tags - line = re.sub(r"<[^>]+>", "", line) - cleaned_lines.append(line) - code_content = "\n".join(cleaned_lines) - else: - # Fallback: just clean HTML - code_content = re.sub(r"]*>", "", code_content) - code_content = re.sub(r"", "", code_content) - code_content = re.sub(r"<[^>]+>", "\n", code_content) - - # For Monaco, extract text from nested divs - if source_type == "monaco": - # Extract actual code from Monaco's complex structure - code_content = re.sub(r"]*>", "\n", code_content) - code_content = re.sub(r"", "", code_content) - code_content = re.sub(r"]*>", "", code_content) - code_content = re.sub(r"", "", code_content) - - # Calculate dynamic minimum length - context_for_length = content[max(0, code_start_pos - 500) : code_start_pos + 500] - min_length = await self._calculate_min_length(language, context_for_length) - - # Skip if initial content is too short - if len(code_content) < min_length: - # Try to find complete block if we have a language - if language and code_start_pos > 0: - # Look for complete code block - complete_code, block_end_pos = await self._find_complete_code_block( - content, code_start_pos, min_length, language - ) - if len(complete_code) >= min_length: - code_content = complete_code - end_pos = block_end_pos - else: - continue - else: - continue - - # Extract position info for deduplication - start_pos = match.start() - end_pos = ( - match.end() - if len(code_content) <= len(match.group(0)) - else code_start_pos + len(code_content) - ) - - # Check if we've already extracted code from this position - position_key = (start_pos, end_pos) - overlapping = False - for existing_start, existing_end in extracted_positions: - # Check if this match overlaps with an existing extraction - if not (end_pos <= existing_start or start_pos >= existing_end): - overlapping = True - break - - if not overlapping: - extracted_positions.add(position_key) - - # Extract context - context_before = content[max(0, start_pos - 1000) : start_pos].strip() - context_after = content[end_pos : min(len(content), end_pos + 1000)].strip() - - # Clean the code content - cleaned_code = self._clean_code_content(code_content, language) - - # Validate code quality - if await self._validate_code_quality(cleaned_code, language): - # Log successful extraction - safe_logfire_info( - f"Extracted code block | source_type={source_type} | language={language} | min_length={min_length} | original_length={len(code_content)} | cleaned_length={len(cleaned_code)}" - ) - - code_blocks.append({ - "code": cleaned_code, - "language": language, - "context_before": context_before, - "context_after": context_after, - "full_context": f"{context_before}\n\n{cleaned_code}\n\n{context_after}", - "source_type": source_type, # Track which pattern matched - }) - else: - safe_logfire_info( - f"Code block failed validation | source_type={source_type} | language={language} | length={len(cleaned_code)}" - ) - - # Pattern 2: ... (standalone) - if not code_blocks: # Only if we didn't find pre/code blocks - code_pattern = r"]*>(.*?)" - matches = re.finditer(code_pattern, content, re.DOTALL | re.IGNORECASE) - - for match in matches: - code_content = match.group(1).strip() - # Clean the code content - cleaned_code = self._clean_code_content(code_content, "") - - # Check if it's multiline or substantial enough and validate quality - # Use a minimal length for standalone code tags - if len(cleaned_code) >= 100 and ("\n" in cleaned_code or len(cleaned_code) > 100): - if await self._validate_code_quality(cleaned_code, ""): - start_pos = match.start() - end_pos = match.end() - context_before = content[max(0, start_pos - 1000) : start_pos].strip() - context_after = content[end_pos : min(len(content), end_pos + 1000)].strip() - - code_blocks.append({ - "code": cleaned_code, - "language": "", - "context_before": context_before, - "context_after": context_after, - "full_context": f"{context_before}\n\n{cleaned_code}\n\n{context_after}", - }) - else: - safe_logfire_info( - f"Standalone code block failed validation | length={len(cleaned_code)}" - ) - - return code_blocks - - async def _extract_text_file_code_blocks( - self, content: str, url: str, min_length: int | None = None - ) -> list[dict[str, Any]]: - """ - Extract code blocks from plain text files (like .txt files). - Handles formats like llms.txt where code blocks may be indicated by: - - Triple backticks (```) - - Language indicators (e.g., "typescript", "python") - - Indentation patterns - - Code block separators - - Args: - content: The plain text content - url: The URL of the text file for context - min_length: Minimum length for code blocks - - Returns: - List of code blocks with metadata - """ - import re - - safe_logfire_info( - f"🔍 TEXT FILE EXTRACTION START | url={url} | content_length={len(content)}" - ) - safe_logfire_info(f"📄 First 1000 chars: {repr(content[:1000])}...") - safe_logfire_info( - f"📄 Sample showing backticks: {repr(content[5000:6000])}..." - if len(content) > 6000 - else "Content too short for mid-sample" - ) - - code_blocks = [] - - # Method 1: Look for triple backtick code blocks (Markdown style) - # Pattern allows for additional text after language (e.g., "typescript TypeScript") - backtick_pattern = r"```(\w*)[^\n]*\n(.*?)```" - matches = list(re.finditer(backtick_pattern, content, re.DOTALL | re.MULTILINE)) - safe_logfire_info(f"📊 Backtick pattern matches: {len(matches)}") - - for i, match in enumerate(matches): - language = match.group(1) or "" - code_content = match.group(2).strip() - - # Log match info without including the actual content that might break formatting - safe_logfire_info( - f"🔎 Match {i + 1}: language='{language}', raw_length={len(code_content)}" - ) - - # Get position info first - start_pos = match.start() - end_pos = match.end() - - # Calculate dynamic minimum length - context_around = content[max(0, start_pos - 500) : min(len(content), end_pos + 500)] - if min_length is None: - actual_min_length = await self._calculate_min_length(language, context_around) - else: - actual_min_length = min_length - - if len(code_content) >= actual_min_length: - # Get context - context_before = content[max(0, start_pos - 500) : start_pos].strip() - context_after = content[end_pos : min(len(content), end_pos + 500)].strip() - - # Clean and validate - cleaned_code = self._clean_code_content(code_content, language) - safe_logfire_info(f"🧹 After cleaning: length={len(cleaned_code)}") - - if await self._validate_code_quality(cleaned_code, language): - safe_logfire_info( - f"✅ VALID backtick code block | language={language} | length={len(cleaned_code)}" - ) - code_blocks.append({ - "code": cleaned_code, - "language": language, - "context_before": context_before, - "context_after": context_after, - "full_context": f"{context_before}\n\n{cleaned_code}\n\n{context_after}", - "source_type": "text_backticks", - }) - else: - safe_logfire_info( - f"❌ INVALID code block failed validation | language={language}" - ) - else: - safe_logfire_info( - f"❌ Code block too short: {len(code_content)} < {actual_min_length}" - ) - - # Method 2: Look for language-labeled code blocks (e.g., "TypeScript:" or "Python example:") - language_pattern = r"(?:^|\n)((?:typescript|javascript|python|java|c\+\+|rust|go|ruby|php|swift|kotlin|scala|r|matlab|julia|dart|elixir|erlang|haskell|clojure|lua|perl|shell|bash|sql|html|css|xml|json|yaml|toml|ini|dockerfile|makefile|cmake|gradle|maven|npm|yarn|pip|cargo|gem|pod|composer|nuget|apt|yum|brew|choco|snap|flatpak|appimage|msi|exe|dmg|pkg|deb|rpm|tar|zip|7z|rar|gz|bz2|xz|zst|lz4|lzo|lzma|lzip|lzop|compress|uncompress|gzip|gunzip|bzip2|bunzip2|xz|unxz|zstd|unzstd|lz4|unlz4|lzo|unlzo|lzma|unlzma|lzip|lunzip|lzop|unlzop)\s*(?:code|example|snippet)?)[:\s]*\n((?:(?:^[ \t]+.*\n?)+)|(?:.*\n)+?)(?=\n(?:[A-Z][a-z]+\s*:|^\s*$|\n#|\n\*|\n-|\n\d+\.))" - matches = re.finditer(language_pattern, content, re.IGNORECASE | re.MULTILINE) - - for match in matches: - language_info = match.group(1).lower() - # Extract just the language name - language = ( - re.match(r"(\w+)", language_info).group(1) - if re.match(r"(\w+)", language_info) - else "" - ) - code_content = match.group(2).strip() - - # Calculate dynamic minimum length for language-labeled blocks - if min_length is None: - actual_min_length_lang = await self._calculate_min_length( - language, code_content[:500] - ) - else: - actual_min_length_lang = min_length - - if len(code_content) >= actual_min_length_lang: - # Get context - start_pos = match.start() - end_pos = match.end() - context_before = content[max(0, start_pos - 500) : start_pos].strip() - context_after = content[end_pos : min(len(content), end_pos + 500)].strip() - - # Clean and validate - cleaned_code = self._clean_code_content(code_content, language) - if await self._validate_code_quality(cleaned_code, language): - safe_logfire_info( - f"Found language-labeled code block | language={language} | length={len(cleaned_code)}" - ) - code_blocks.append({ - "code": cleaned_code, - "language": language, - "context_before": context_before, - "context_after": context_after, - "full_context": f"{context_before}\n\n{cleaned_code}\n\n{context_after}", - "source_type": "text_language_label", - }) - - # Method 3: Look for consistently indented blocks (at least 4 spaces or 1 tab) - # This is more heuristic and should be used carefully - if len(code_blocks) == 0: # Only if we haven't found code blocks yet - # Split content into potential code sections - lines = content.split("\n") - current_block = [] - current_indent = None - block_start_idx = 0 - - for i, line in enumerate(lines): - # Check if line is indented - stripped = line.lstrip() - indent = len(line) - len(stripped) - - if indent >= 4 and stripped: # At least 4 spaces and not empty - if current_indent is None: - current_indent = indent - block_start_idx = i - current_block.append(line) - elif current_block: - block_text = "\n".join(current_block) - threshold = ( - min_length - if min_length is not None - else await self._get_min_code_length() - ) - if len(block_text) < threshold: - current_block = [] - current_indent = None - continue - - # End of indented block, check if it's code - code_content = block_text - - # Try to detect language from content - language = self._detect_language_from_content(code_content) - - # Get context - context_before_lines = lines[max(0, block_start_idx - 10) : block_start_idx] - context_after_lines = lines[i : min(len(lines), i + 10)] - context_before = "\n".join(context_before_lines).strip() - context_after = "\n".join(context_after_lines).strip() - - # Clean and validate - cleaned_code = self._clean_code_content(code_content, language) - if await self._validate_code_quality(cleaned_code, language): - safe_logfire_info( - f"Found indented code block | language={language} | length={len(cleaned_code)}" - ) - code_blocks.append({ - "code": cleaned_code, - "language": language, - "context_before": context_before, - "context_after": context_after, - "full_context": f"{context_before}\n\n{cleaned_code}\n\n{context_after}", - "source_type": "text_indented", - }) - - # Reset for next block - current_block = [] - current_indent = None - else: - # Reset if not indented - if current_block and not stripped: - # Allow empty lines within code blocks - current_block.append(line) - else: - current_block = [] - current_indent = None - - safe_logfire_info( - f"📊 TEXT FILE EXTRACTION COMPLETE | total_blocks={len(code_blocks)} | url={url}" - ) - for i, block in enumerate(code_blocks[:3]): # Log first 3 blocks - safe_logfire_info( - f"📦 Block {i + 1} summary: language='{block.get('language', '')}', source_type='{block.get('source_type', '')}', length={len(block.get('code', ''))}" - ) - return code_blocks - - async def _extract_pdf_code_blocks( - self, content: str, url: str - ) -> list[dict[str, Any]]: - """ - Extract code blocks from PDF-extracted text that lacks markdown formatting. - PDFs lose markdown delimiters, so we need to detect code patterns in plain text. - - This uses a much simpler approach - look for distinct code segments separated by prose. - """ - import re - - safe_logfire_info(f"🔍 PDF CODE EXTRACTION START | url={url} | content_length={len(content)}") - - code_blocks = [] - min_length = await self._get_min_code_length() - - # Split content into paragraphs/sections - # Use double newlines and page breaks as natural boundaries - sections = re.split(r'\n\n+|--- Page \d+ ---', content) - - safe_logfire_info(f"📄 Split PDF into {len(sections)} sections") - - for i, section in enumerate(sections): - section = section.strip() - if not section or len(section) < 50: # Skip very short sections - continue - - # Check if this section looks like code - if self._is_pdf_section_code_like(section): - safe_logfire_info(f"🔍 Analyzing section {i} as potential code (length: {len(section)})") - - # Try to detect language - language = self._detect_language_from_content(section) - - # Clean the content - cleaned_code = self._clean_code_content(section, language) - - # Check length after cleaning - if len(cleaned_code) >= min_length: - # Validate quality - if await self._validate_code_quality(cleaned_code, language): - # Get context from adjacent sections - context_before = sections[i-1].strip() if i > 0 else "" - context_after = sections[i+1].strip() if i < len(sections)-1 else "" - - safe_logfire_info(f"✅ PDF code section | language={language} | length={len(cleaned_code)}") - code_blocks.append({ - "code": cleaned_code, - "language": language, - "context_before": context_before, - "context_after": context_after, - "full_context": f"{context_before}\n\n{cleaned_code}\n\n{context_after}", - "source_type": "pdf_section", - }) - else: - safe_logfire_info(f"❌ PDF section failed validation | language={language}") - else: - safe_logfire_info(f"❌ PDF section too short after cleaning: {len(cleaned_code)} < {min_length}") - else: - safe_logfire_info(f"📝 Section {i} identified as prose/documentation") - - safe_logfire_info(f"🔍 PDF CODE EXTRACTION COMPLETE | total_blocks={len(code_blocks)} | url={url}") - return code_blocks - - def _is_pdf_section_code_like(self, section: str) -> bool: - """ - Determine if a PDF section contains code rather than prose. - """ - import re - - # Count code indicators vs prose indicators - code_score = 0 - prose_score = 0 - - # Code indicators (higher weight for stronger indicators) - code_patterns = [ - (r'\bfrom \w+(?:\.\w+)* import\b', 3), # Python imports (strong) - (r'\bdef \w+\s*\(', 3), # Function definitions (strong) - (r'\bclass \w+\s*[\(:]', 3), # Class definitions (strong) - (r'\w+\s*=\s*\w+\(', 2), # Function calls assigned (medium) - (r'\w+\s*=\s*\[.*\]', 2), # List assignments (medium) - (r'\w+\.\w+\(', 2), # Method calls (medium) - (r'^\s*#[^#]', 1), # Single-line comments (weak) - (r'\bpip install\b', 2), # Package management (medium) - (r'\bpytest\b', 2), # Testing commands (medium) - (r'\bgit clone\b', 2), # Git commands (medium) - (r':\s*\n\s+\w+:', 2), # YAML structure (medium) - (r'\blambda\s+\w+:', 2), # Lambda functions (medium) - ] - - # Prose indicators - prose_patterns = [ - (r'\b(the|this|that|these|those|are|is|was|were|will|would|should|could|have|has|had)\b', 1), - (r'[.!?]\s+[A-Z]', 2), # Sentence endings - (r'\b(however|therefore|furthermore|moreover|additionally|specifically)\b', 2), - (r'\bTable of Contents\b', 3), - (r'\bAPI Reference\b', 2), - ] - - # Count patterns - for pattern, weight in code_patterns: - matches = len(re.findall(pattern, section, re.IGNORECASE | re.MULTILINE)) - code_score += matches * weight - - for pattern, weight in prose_patterns: - matches = len(re.findall(pattern, section, re.IGNORECASE | re.MULTILINE)) - prose_score += matches * weight - - # Additional checks - lines = section.split('\n') - non_empty_lines = [line.strip() for line in lines if line.strip()] - - if not non_empty_lines: - return False - - # If section is mostly single words or very short lines, probably not code - short_lines = sum(1 for line in non_empty_lines if len(line.split()) < 3) - if len(non_empty_lines) > 0 and short_lines / len(non_empty_lines) > 0.7: - prose_score += 3 - - # If section has common code structure indicators - if any('(' in line and ')' in line for line in non_empty_lines[:5]): - code_score += 2 - - safe_logfire_info(f"📊 Section scoring: code_score={code_score}, prose_score={prose_score}") - - # Code-like if code score significantly higher than prose score - return code_score > prose_score and code_score > 2 - - def _detect_language_from_content(self, code: str) -> str: - """ - Try to detect programming language from code content. - This is a simple heuristic approach. - """ - import re - - # Language detection patterns - patterns = { - "python": [ - r"\bdef\s+\w+\s*\(", - r"\bclass\s+\w+", - r"\bimport\s+\w+", - r"\bfrom\s+\w+\s+import", - ], - "javascript": [ - r"\bfunction\s+\w+\s*\(", - r"\bconst\s+\w+\s*=", - r"\blet\s+\w+\s*=", - r"\bvar\s+\w+\s*=", - ], - "typescript": [ - r"\binterface\s+\w+", - r":\s*\w+\[\]", - r"\btype\s+\w+\s*=", - r"\bclass\s+\w+.*\{", - ], - "java": [ - r"\bpublic\s+class\s+\w+", - r"\bprivate\s+\w+\s+\w+", - r"\bpublic\s+static\s+void\s+main", - ], - "rust": [r"\bfn\s+\w+\s*\(", r"\blet\s+mut\s+\w+", r"\bimpl\s+\w+", r"\bstruct\s+\w+"], - "go": [r"\bfunc\s+\w+\s*\(", r"\bpackage\s+\w+", r"\btype\s+\w+\s+struct"], - } - - # Count matches for each language - scores = {} - for lang, lang_patterns in patterns.items(): - score = 0 - for pattern in lang_patterns: - if re.search(pattern, code, re.MULTILINE): - score += 1 - if score > 0: - scores[lang] = score - - # Return language with highest score - if scores: - return max(scores, key=scores.get) - - return "" - - async def _find_complete_code_block( - self, - content: str, - start_pos: int, - min_length: int = 250, - language: str = "", - max_length: int = None, - ) -> tuple[str, int]: - """ - Find a complete code block starting from a position, extending until we find a natural boundary. - - Args: - content: The full content to search in - start_pos: Starting position in the content - min_length: Minimum length for the code block - language: Detected language for language-specific patterns - - Returns: - Tuple of (complete_code_block, end_position) - """ - # Start with the minimum content - if start_pos + min_length > len(content): - return content[start_pos:], len(content) - - # Look for natural code boundaries - boundary_patterns = [ - r"\n}\s*$", # Closing brace at end of line - r"\n}\s*;?\s*$", # Closing brace with optional semicolon - r"\n\)\s*;?\s*$", # Closing parenthesis - r"\n\s*$\n\s*$", # Double newline (paragraph break) - r"\n(?=class\s)", # Before next class - r"\n(?=function\s)", # Before next function - r"\n(?=def\s)", # Before next Python function - r"\n(?=export\s)", # Before next export - r"\n(?=const\s)", # Before next const declaration - r"\n(?=//)", # Before comment block - r"\n(?=#)", # Before Python comment - r"\n(?=\*)", # Before JSDoc/comment - r"\n(?=```)", # Before next code block - ] - - # Add language-specific patterns if available - if language and language.lower() in self.LANGUAGE_PATTERNS: - lang_patterns = self.LANGUAGE_PATTERNS[language.lower()] - if "block_end" in lang_patterns: - boundary_patterns.insert(0, lang_patterns["block_end"]) - - # Extend until we find a boundary - extended_pos = start_pos + min_length - while extended_pos < len(content): - # Check next 500 characters for a boundary - lookahead_end = min(extended_pos + 500, len(content)) - lookahead = content[extended_pos:lookahead_end] - - for pattern in boundary_patterns: - match = re.search(pattern, lookahead, re.MULTILINE) - if match: - final_pos = extended_pos + match.end() - return content[start_pos:final_pos].rstrip(), final_pos - - # If no boundary found, extend by another chunk - extended_pos += 100 - - # Cap at maximum length - if max_length is None: - max_length = await self._get_max_code_length() - if extended_pos - start_pos > max_length: - break - - # Return what we have - return content[start_pos:extended_pos].rstrip(), extended_pos - - async def _calculate_min_length(self, language: str, context: str) -> int: - """ - Calculate appropriate minimum length based on language and context. - - Args: - language: The detected programming language - context: Surrounding context of the code - - Returns: - Calculated minimum length - """ - # Base lengths by language - # Check if contextual length adjustment is enabled - if not await self._is_contextual_length_enabled(): - # Return default minimum length - return await self._get_min_code_length() - - # Base lengths by language - base_lengths = { - "json": 100, # JSON can be short - "yaml": 100, # YAML too - "xml": 100, # XML structures - "html": 150, # HTML snippets - "css": 150, # CSS rules - "sql": 150, # SQL queries - "python": 200, # Python functions - "javascript": 250, # JavaScript typically longer - "typescript": 250, # TypeScript typically longer - "java": 300, # Java even more verbose - "c++": 300, # C++ similar to Java - "cpp": 300, # C++ alternative - "c": 250, # C slightly less verbose - "rust": 250, # Rust medium verbosity - "go": 200, # Go is concise - } - - # Get default minimum from settings - default_min = await self._get_min_code_length() - min_length = base_lengths.get(language.lower(), default_min) - - # Adjust based on context clues - context_lower = context.lower() - if any(word in context_lower for word in ["example", "snippet", "sample", "demo"]): - min_length = int(min_length * 0.7) # Examples can be shorter - elif any(word in context_lower for word in ["implementation", "complete", "full"]): - min_length = int(min_length * 1.5) # Full implementations should be longer - elif any(word in context_lower for word in ["minimal", "simple", "basic"]): - min_length = int(min_length * 0.8) # Simple examples can be shorter - - # Ensure reasonable bounds - return max(100, min(1000, min_length)) - - def _decode_html_entities(self, text: str) -> str: - """Decode common HTML entities and clean HTML tags from code.""" - import re - - # First, handle span tags that wrap individual tokens - # Check if spans are being used for syntax highlighting (no spaces between tags) - if "", "", text) - text = re.sub(r"]*>", "", text) - else: - # Normal span usage - might need spacing - # Only add space if there isn't already whitespace - text = re.sub(r"(?=[A-Za-z0-9])", " ", text) - text = re.sub(r"]*>", "", text) - - # Remove any other HTML tags but preserve their content - text = re.sub(r"]+>", "", text) - - # Decode HTML entities - replacements = { - "<": "<", - ">": ">", - "&": "&", - """: '"', - "'": "'", - " ": " ", - "'": "'", - "/": "/", - "<": "<", - ">": ">", - } - - for entity, char in replacements.items(): - text = text.replace(entity, char) - - # Replace escaped newlines with actual newlines - text = text.replace("\\n", "\n") - - # Clean up excessive whitespace while preserving intentional spacing - # Replace multiple spaces with single space, but preserve newlines - lines = text.split("\n") - cleaned_lines = [] - for line in lines: - # Replace multiple spaces with single space - line = re.sub(r" +", " ", line) - # Trim trailing spaces but preserve leading spaces (indentation) - line = line.rstrip() - cleaned_lines.append(line) - - text = "\n".join(cleaned_lines) - - return text - - def _clean_code_content(self, code: str, language: str = "") -> str: - """ - Clean and fix common issues in extracted code content. - - Args: - code: The code content to clean - language: The detected language (optional) - - Returns: - Cleaned code content - """ - import re - - # First apply HTML entity decoding and tag cleaning - code = self._decode_html_entities(code) - - # Fix common concatenation issues from span removal - # Common patterns where spaces are missing between keywords - spacing_fixes = [ - # Import statements - (r"(\b(?:from|import|as)\b)([A-Za-z])", r"\1 \2"), - # Function/class definitions - (r"(\b(?:def|class|async|await|return|raise|yield)\b)([A-Za-z])", r"\1 \2"), - # Control flow - (r"(\b(?:if|elif|else|for|while|try|except|finally|with)\b)([A-Za-z])", r"\1 \2"), - # Type hints and declarations - ( - r"(\b(?:int|str|float|bool|list|dict|tuple|set|None|True|False)\b)([A-Za-z])", - r"\1 \2", - ), - # Common Python keywords - (r"(\b(?:and|or|not|in|is|lambda)\b)([A-Za-z])", r"\1 \2"), - # Fix missing spaces around operators (but be careful with negative numbers) - (r"([A-Za-z_)])(\+|-|\*|/|=|<|>|%)", r"\1 \2"), - (r"(\+|-|\*|/|=|<|>|%)([A-Za-z_(])", r"\1 \2"), - ] - - for pattern, replacement in spacing_fixes: - code = re.sub(pattern, replacement, code) - - # Fix specific patterns for different languages - if language.lower() in ["python", "py"]: - # Fix Python-specific issues - code = re.sub(r"(\b(?:from|import)\b)(\w+)(\b(?:import)\b)", r"\1 \2 \3", code) - # Fix missing colons - code = re.sub( - r"(\b(?:def|class|if|elif|else|for|while|try|except|finally|with)\b[^:]+)$", - r"\1:", - code, - flags=re.MULTILINE, - ) - - # Remove backticks that might have been included - if code.startswith("```") and code.endswith("```"): - lines = code.split("\n") - if len(lines) > 2: - # Remove first and last line - code = "\n".join(lines[1:-1]) - elif code.startswith("`") and code.endswith("`"): - code = code[1:-1] - - # Final cleanup - # Remove any remaining excessive spaces while preserving indentation - lines = code.split("\n") - cleaned_lines = [] - for line in lines: - # Don't touch leading whitespace (indentation) - stripped = line.lstrip() - indent = line[: len(line) - len(stripped)] - # Clean the rest of the line - cleaned = re.sub(r" {2,}", " ", stripped) - cleaned_lines.append(indent + cleaned) - - return "\n".join(cleaned_lines).strip() - - async def _validate_code_quality(self, code: str, language: str = "") -> bool: - """ - Enhanced validation to ensure extracted content is actual code. - - Args: - code: The code content to validate - language: The detected language (optional) - - Returns: - True if code passes quality checks, False otherwise - """ - import re - - # Basic checks - if not code or len(code.strip()) < 20: - return False - - # Skip diagram languages if filtering is enabled - if await self._is_diagram_filtering_enabled(): - if language.lower() in ["mermaid", "plantuml", "graphviz", "dot", "diagram"]: - safe_logfire_info(f"Skipping diagram language: {language}") - return False - - # Check for common formatting issues that indicate poor extraction - bad_patterns = [ - # Concatenated keywords without spaces (but allow camelCase) - r"\b(from|import|def|class|if|for|while|return)(?=[a-z])", - # HTML entities that weren't decoded - r"&[lg]t;|&|"|&#\d+;", - # Excessive HTML tags - r"<[^>]{50,}>", # Very long HTML tags - # Multiple spans in a row (indicates poor extraction) - r"(]*>){5,}", - # Suspicious character sequences - r"[^\s]{200,}", # Very long unbroken strings (increased threshold) - ] - - for pattern in bad_patterns: - if re.search(pattern, code): - safe_logfire_info(f"Code failed quality check: pattern '{pattern}' found") - return False - - # Check for minimum code complexity using various indicators - code_indicators = { - "function_calls": r"\w+\s*\([^)]*\)", - "assignments": r"\w+\s*=\s*.+", - "control_flow": r"\b(if|for|while|switch|case|try|catch|except)\b", - "declarations": r"\b(var|let|const|def|class|function|interface|type|struct|enum)\b", - "imports": r"\b(import|from|require|include|using|use)\b", - "brackets": r"[\{\}\[\]]", - "operators": r"[\+\-\*\/\%\&\|\^<>=!]", - "method_chains": r"\.\w+", - "arrows": r"(=>|->)", - "keywords": r"\b(return|break|continue|yield|await|async)\b", - } - - indicator_count = 0 - indicator_details = [] - for name, pattern in code_indicators.items(): - if re.search(pattern, code): - indicator_count += 1 - indicator_details.append(name) - - # Require minimum code indicators - min_indicators = await self._get_min_code_indicators() - if indicator_count < min_indicators: - safe_logfire_info( - f"Code has insufficient indicators: {indicator_count} found ({', '.join(indicator_details)})" - ) - return False - - # Check code-to-comment ratio - lines = code.split("\n") - non_empty_lines = [line for line in lines if line.strip()] - - if not non_empty_lines: - return False - - # Count comment lines (various comment styles) - comment_patterns = [ - r"^\s*(//|#|/\*|\*|