Fix race condition in concurrent crawling with unique source IDs

archon-ui-main/src/App.tsx

@@ -11,7 +11,9 @@ import { SettingsProvider, useSettings } from './contexts/SettingsContext';
 import { ProjectPage } from './pages/ProjectPage';
 import { DisconnectScreenOverlay } from './components/DisconnectScreenOverlay';
 import { ErrorBoundaryWithBugReport } from './components/bug-report/ErrorBoundaryWithBugReport';
+import { MigrationBanner } from './components/ui/MigrationBanner';
 import { serverHealthService } from './services/serverHealthService';
+import { useMigrationStatus } from './hooks/useMigrationStatus';
 
 const AppRoutes = () => {
   const { projectsEnabled } = useSettings();
@@ -38,6 +40,8 @@ const AppContent = () => {
     enabled: true,
     delay: 10000
   });
+  const [migrationBannerDismissed, setMigrationBannerDismissed] = useState(false);
+  const migrationStatus = useMigrationStatus();
 
   useEffect(() => {
     // Load initial settings
@@ -77,6 +81,13 @@ const AppContent = () => {
       <Router>
         <ErrorBoundaryWithBugReport>
           <MainLayout>
+            {/* Migration Banner - shows when backend is up but DB schema needs work */}
+            {migrationStatus.migrationRequired && !migrationBannerDismissed && (
+              <MigrationBanner
+                message={migrationStatus.message || "Database migration required"}
+                onDismiss={() => setMigrationBannerDismissed(true)}
+              />
+            )}
             <AppRoutes />
           </MainLayout>
         </ErrorBoundaryWithBugReport>

archon-ui-main/src/components/BackendStartupError.tsx

@@ -40,8 +40,12 @@ export const BackendStartupError: React.FC = () => {
 
               <div className="bg-yellow-950/30 border border-yellow-700/30 rounded-lg p-3">
                 <p className="text-yellow-200 text-sm">
-                  <strong>Common issue:</strong> Using an ANON key instead of SERVICE key in your .env file
+                  <strong>Common issues:</strong>
                 </p>
+                <ul className="text-yellow-200 text-sm mt-1 space-y-1 list-disc list-inside">
+                  <li>Using an ANON key instead of SERVICE key in your .env file</li>
+                  <li>Database not set up - run <code className="bg-yellow-800/50 px-1 rounded">migration/complete_setup.sql</code> in Supabase SQL Editor</li>
+                </ul>
               </div>
 
               <div className="pt-4 border-t border-red-900/30">

archon-ui-main/src/components/ui/MigrationBanner.tsx

@@ -0,0 +1,60 @@
+import React from 'react';
+import { AlertTriangle, ExternalLink } from 'lucide-react';
+import { Card } from './Card';
+
+interface MigrationBannerProps {
+  message: string;
+  onDismiss?: () => void;
+}
+
+export const MigrationBanner: React.FC<MigrationBannerProps> = ({
+  message,
+  onDismiss
+}) => {
+  return (
+    <Card className="bg-red-50 border-red-200 dark:bg-red-900/20 dark:border-red-800 mb-6">
+      <div className="flex items-start gap-3 p-4">
+        <AlertTriangle className="w-6 h-6 text-red-500 flex-shrink-0 mt-0.5" />
+        <div className="flex-1">
+          <h3 className="text-lg font-semibold text-red-800 dark:text-red-300 mb-2">
+            Database Migration Required
+          </h3>
+          <p className="text-red-700 dark:text-red-400 mb-3">
+            {message}
+          </p>
+          <div className="bg-red-100 dark:bg-red-900/40 border border-red-200 dark:border-red-800 rounded-lg p-3 mb-3">
+            <p className="text-sm font-medium text-red-800 dark:text-red-300 mb-2">
+              Follow these steps:
+            </p>
+            <ol className="text-sm text-red-700 dark:text-red-400 space-y-1 list-decimal list-inside">
+              <li>Open your Supabase project dashboard</li>
+              <li>Navigate to the SQL Editor</li>
+              <li>Copy and run the migration script from: <code className="bg-red-200 dark:bg-red-800 px-1 rounded">migration/add_source_url_display_name.sql</code></li>
+              <li>Restart Docker containers: <code className="bg-red-200 dark:bg-red-800 px-1 rounded">docker compose down && docker compose up --build -d</code></li>
+              <li>If you used a profile, add it: <code className="bg-red-200 dark:bg-red-800 px-1 rounded">--profile full</code></li>
+            </ol>
+          </div>
+          <div className="flex items-center gap-3">
+            <a
+              href="https://supabase.com/dashboard"
+              target="_blank"
+              rel="noopener noreferrer"
+              className="inline-flex items-center gap-2 bg-red-600 hover:bg-red-700 text-white px-4 py-2 rounded-lg text-sm font-medium transition-colors"
+            >
+              <ExternalLink className="w-4 h-4" />
+              Open Supabase Dashboard
+            </a>
+            {onDismiss && (
+              <button
+                onClick={onDismiss}
+                className="text-red-600 dark:text-red-400 hover:text-red-800 dark:hover:text-red-200 text-sm font-medium"
+              >
+                Dismiss (temporarily)
+              </button>
+            )}
+          </div>
+        </div>
+      </div>
+    </Card>
+  );
+};

archon-ui-main/src/hooks/useMigrationStatus.ts

@@ -0,0 +1,51 @@
+import { useState, useEffect } from 'react';
+
+interface MigrationStatus {
+  migrationRequired: boolean;
+  message?: string;
+  loading: boolean;
+}
+
+export const useMigrationStatus = (): MigrationStatus => {
+  const [status, setStatus] = useState<MigrationStatus>({
+    migrationRequired: false,
+    loading: true,
+  });
+
+  useEffect(() => {
+    const checkMigrationStatus = async () => {
+      try {
+        const response = await fetch('/api/health');
+        const healthData = await response.json();
+
+        if (healthData.status === 'migration_required') {
+          setStatus({
+            migrationRequired: true,
+            message: healthData.message,
+            loading: false,
+          });
+        } else {
+          setStatus({
+            migrationRequired: false,
+            loading: false,
+          });
+        }
+      } catch (error) {
+        console.error('Failed to check migration status:', error);
+        setStatus({
+          migrationRequired: false,
+          loading: false,
+        });
+      }
+    };
+
+    checkMigrationStatus();
+
+    // Check periodically (every 30 seconds) to detect when migration is complete
+    const interval = setInterval(checkMigrationStatus, 30000);
+
+    return () => clearInterval(interval);
+  }, []);
+
+  return status;
+};

migration/add_source_url_display_name.sql

@@ -0,0 +1,36 @@
+-- =====================================================
+-- Add source_url and source_display_name columns
+-- =====================================================
+-- This migration adds two new columns to better identify sources:
+-- - source_url: The original URL that was crawled
+-- - source_display_name: Human-readable name for UI display
+--
+-- This solves the race condition issue where multiple crawls
+-- to the same domain would conflict by using domain as source_id
+-- =====================================================
+
+-- Add new columns to archon_sources table
+ALTER TABLE archon_sources 
+ADD COLUMN IF NOT EXISTS source_url TEXT,
+ADD COLUMN IF NOT EXISTS source_display_name TEXT;
+
+-- Add indexes for the new columns for better query performance
+CREATE INDEX IF NOT EXISTS idx_archon_sources_url ON archon_sources(source_url);
+CREATE INDEX IF NOT EXISTS idx_archon_sources_display_name ON archon_sources(source_display_name);
+
+-- Add comments to document the new columns
+COMMENT ON COLUMN archon_sources.source_url IS 'The original URL that was crawled to create this source';
+COMMENT ON COLUMN archon_sources.source_display_name IS 'Human-readable name for UI display (e.g., "GitHub - microsoft/typescript")';
+
+-- Backfill existing data
+-- For existing sources, copy source_id to both new fields as a fallback
+UPDATE archon_sources 
+SET 
+    source_url = COALESCE(source_url, source_id),
+    source_display_name = COALESCE(source_display_name, source_id)
+WHERE 
+    source_url IS NULL 
+    OR source_display_name IS NULL;
+
+-- Note: source_id will now contain a unique hash instead of domain
+-- This ensures no conflicts when multiple sources from same domain are crawled

migration/complete_setup.sql

@@ -170,6 +170,8 @@ COMMENT ON TABLE archon_settings IS 'Stores application configuration including
 -- Create the sources table
 CREATE TABLE IF NOT EXISTS archon_sources (
     source_id TEXT PRIMARY KEY,
+    source_url TEXT,
+    source_display_name TEXT,
     summary TEXT,
     total_word_count INTEGER DEFAULT 0,
     title TEXT,
@@ -180,10 +182,15 @@ CREATE TABLE IF NOT EXISTS archon_sources (
 
 -- Create indexes for better query performance
 CREATE INDEX IF NOT EXISTS idx_archon_sources_title ON archon_sources(title);
+CREATE INDEX IF NOT EXISTS idx_archon_sources_url ON archon_sources(source_url);
+CREATE INDEX IF NOT EXISTS idx_archon_sources_display_name ON archon_sources(source_display_name);
 CREATE INDEX IF NOT EXISTS idx_archon_sources_metadata ON archon_sources USING GIN(metadata);
 CREATE INDEX IF NOT EXISTS idx_archon_sources_knowledge_type ON archon_sources((metadata->>'knowledge_type'));
 
--- Add comments to document the new columns
+-- Add comments to document the columns
+COMMENT ON COLUMN archon_sources.source_id IS 'Unique hash identifier for the source (16-char SHA256 hash of URL)';
+COMMENT ON COLUMN archon_sources.source_url IS 'The original URL that was crawled to create this source';
+COMMENT ON COLUMN archon_sources.source_display_name IS 'Human-readable name for UI display (e.g., "GitHub - microsoft/typescript")';
 COMMENT ON COLUMN archon_sources.title IS 'Descriptive title for the source (e.g., "Pydantic AI API Reference")';
 COMMENT ON COLUMN archon_sources.metadata IS 'JSONB field storing knowledge_type, tags, and other metadata';
 

python/src/server/api_routes/knowledge_api.py

@@ -27,10 +27,6 @@
 
 # Import unified logging
 from ..config.logfire_config import get_logger, safe_logfire_error, safe_logfire_info
-from ..services.crawler_manager import get_crawler
-from ..services.search.rag_service import RAGService
-from ..services.storage import DocumentStorageService
-from ..utils import get_supabase_client
 from ..utils.document_processing import extract_text_from_document
 
 # Get logger for this module
@@ -513,11 +509,6 @@ async def upload_document(
 ):
     """Upload and process a document with progress tracking."""
     try:
-        # DETAILED LOGGING: Track knowledge_type parameter flow
-        safe_logfire_info(
-            f"📋 UPLOAD: Starting document upload | filename={file.filename} | content_type={file.content_type} | knowledge_type={knowledge_type}"
-        )
-
         safe_logfire_info(
             f"Starting document upload | filename={file.filename} | content_type={file.content_type} | knowledge_type={knowledge_type}"
         )
@@ -871,7 +862,22 @@ async def get_database_metrics():
 
 @router.get("/health")
 async def knowledge_health():
-    """Knowledge API health check."""
+    """Knowledge API health check with migration detection."""
+    # Check for database migration needs
+    from ..main import _check_database_schema
+
+    schema_status = await _check_database_schema()
+    if not schema_status["valid"]:
+        return {
+            "status": "migration_required",
+            "service": "knowledge-api", 
+            "timestamp": datetime.now().isoformat(),
+            "ready": False,
+            "migration_required": True,
+            "message": schema_status["message"],
+            "migration_instructions": "Open Supabase Dashboard → SQL Editor → Run: migration/add_source_url_display_name.sql"
+        }
+
     # Removed health check logging to reduce console noise
     result = {
         "status": "healthy",

python/src/server/main.py

@@ -246,12 +246,27 @@ async def health_check():
             "ready": False,
         }
 
+    # Check for required database schema
+    schema_status = await _check_database_schema()
+    if not schema_status["valid"]:
+        return {
+            "status": "migration_required",
+            "service": "archon-backend", 
+            "timestamp": datetime.now().isoformat(),
+            "ready": False,
+            "migration_required": True,
+            "message": schema_status["message"],
+            "migration_instructions": "Open Supabase Dashboard → SQL Editor → Run: migration/add_source_url_display_name.sql",
+            "schema_valid": False
+        }
+
     return {
         "status": "healthy",
         "service": "archon-backend",
         "timestamp": datetime.now().isoformat(),
         "ready": True,
         "credentials_loaded": True,
+        "schema_valid": True,
     }
 
 
@@ -262,6 +277,78 @@ async def api_health_check():
     return await health_check()
 
 
+# Cache schema check result to avoid repeated database queries
+_schema_check_cache = {"valid": None, "checked_at": 0}
+
+async def _check_database_schema():
+    """Check if required database schema exists - only for existing users who need migration."""
+    import time
+
+    # If we've already confirmed schema is valid, don't check again
+    if _schema_check_cache["valid"] is True:
+        return {"valid": True, "message": "Schema is up to date (cached)"}
+
+    # If we recently failed, don't spam the database (wait at least 30 seconds)
+    current_time = time.time()
+    if (_schema_check_cache["valid"] is False and 
+        current_time - _schema_check_cache["checked_at"] < 30):
+        return _schema_check_cache["result"]
+
+    try:
+        from .services.client_manager import get_supabase_client
+
+        client = get_supabase_client()
+
+        # Try to query the new columns directly - if they exist, schema is up to date
+        test_query = client.table('archon_sources').select('source_url, source_display_name').limit(1).execute()
+
+        # Cache successful result permanently
+        _schema_check_cache["valid"] = True
+        _schema_check_cache["checked_at"] = current_time
+
+        return {"valid": True, "message": "Schema is up to date"}
+
+    except Exception as e:
+        error_msg = str(e).lower()
+
+        # Log schema check error for debugging
+        api_logger.debug(f"Schema check error: {type(e).__name__}: {str(e)}")
+
+        # Check for specific error types based on PostgreSQL error codes and messages
+
+        # Check for missing columns first (more specific than table check)
+        missing_source_url = 'source_url' in error_msg and ('column' in error_msg or 'does not exist' in error_msg)
+        missing_source_display = 'source_display_name' in error_msg and ('column' in error_msg or 'does not exist' in error_msg)
+
+        # Also check for PostgreSQL error code 42703 (undefined column)
+        is_column_error = '42703' in error_msg or 'column' in error_msg
+
+        if (missing_source_url or missing_source_display) and is_column_error:
+            result = {
+                "valid": False, 
+                "message": "Database schema outdated - missing required columns from recent updates"
+            }
+            # Cache failed result with timestamp
+            _schema_check_cache["valid"] = False
+            _schema_check_cache["checked_at"] = current_time
+            _schema_check_cache["result"] = result
+            return result
+
+        # Check for table doesn't exist (less specific, only if column check didn't match)
+        # Look for relation/table errors specifically
+        if ('relation' in error_msg and 'does not exist' in error_msg) or ('table' in error_msg and 'does not exist' in error_msg):
+            # Table doesn't exist - not a migration issue, it's a setup issue
+            return {"valid": True, "message": "Table doesn't exist - handled by startup error"}
+
+        # Other errors don't necessarily mean migration needed
+        result = {"valid": True, "message": f"Schema check inconclusive: {str(e)}"}
+        # Don't cache inconclusive results - allow retry
+        return result
+
+
+# Export for Socket.IO
+
+
 # Create Socket.IO app wrapper
 # This wraps the FastAPI app with Socket.IO functionality
 socket_app = create_socketio_app(app)