Fix/(llms.txt) not crawling links inside of file

archon-ui-main/src/components/knowledge-base/CrawlingProgressCard.tsx

@@ -724,12 +724,12 @@ export const CrawlingProgressCard: React.FC<CrawlingProgressCardProps> = ({
                         </div>
 
                         {/* Single batch progress bar */}
-                        <div className="w-full bg-gray-200 dark:bg-zinc-700 rounded-full h-2">
+                        <div className="w-full bg-gray-200 dark:bg-zinc-700 rounded-full h-2 min-w-0">
                           <motion.div
                             className="h-2 rounded-full bg-blue-500 dark:bg-blue-400"
                             initial={{ width: 0 }}
                             animate={{ 
-                              width: `${Math.round(((progressData.completed_batches || 0) / (progressData.total_batches || 1)) * 100)}%` 
+                              width: `${Math.min(100, Math.max(0, Math.round(((progressData.completed_batches || 0) / (progressData.total_batches || 1)) * 100)))}%`
                             }}
                             transition={{ duration: 0.5, ease: 'easeOut' }}
                           />

python/src/server/services/crawling/crawling_service.py

@@ -362,16 +362,16 @@ async def doc_storage_callback(
             ):
                 if self.progress_id:
                     _ensure_socketio_imports()
-                    # Map percentage to document storage range (20-85%)
-                    mapped_percentage = 20 + int((percentage / 100) * (85 - 20))
+                    # Use ProgressMapper to consistently map document storage progress
+                    overall_progress = self.progress_mapper.map_progress("document_storage", percentage)
                     safe_logfire_info(
-                        f"Document storage progress mapping: {percentage}% -> {mapped_percentage}%"
+                        f"Document storage progress mapping: {percentage}% -> {overall_progress}%"
                     )
 
                     # Update progress state while preserving existing fields
                     self.progress_state.update({
                         "status": "document_storage",
-                        "percentage": mapped_percentage,
+                        "percentage": overall_progress,
                         "log": message,
                     })
 
@@ -492,6 +492,36 @@ async def code_progress_callback(data: dict):
                     f"Unregistered orchestration service on error | progress_id={self.progress_id}"
                 )
 
+    def _is_self_link(self, link: str, base_url: str) -> bool:
+        """
+        Check if a link is a self-referential link to the base URL.
+        Handles query parameters, fragments, and trailing slashes.
+
+        Args:
+            link: The link to check
+            base_url: The base URL to compare against
+
+        Returns:
+            True if the link is self-referential, False otherwise
+        """
+        try:
+            from urllib.parse import urlparse
+
+            # Parse both URLs to compare their core components
+            link_parsed = urlparse(link)
+            base_parsed = urlparse(base_url)
+
+            # Compare scheme, netloc, and path (ignoring query and fragment)
+            link_core = f"{link_parsed.scheme}://{link_parsed.netloc}{link_parsed.path.rstrip('/')}"
+            base_core = f"{base_parsed.scheme}://{base_parsed.netloc}{base_parsed.path.rstrip('/')}"
+
+            return link_core == base_core
+
+        except Exception as e:
+            logger.warning(f"Error checking if link is self-referential: {e}")
+            # Fallback to simple string comparison
+            return link.rstrip('/') == base_url.rstrip('/')
+
     async def _crawl_by_url_type(self, url: str, request: Dict[str, Any]) -> tuple:
         """
         Detect URL type and perform appropriate crawling.
@@ -504,14 +534,16 @@ async def _crawl_by_url_type(self, url: str, request: Dict[str, Any]) -> tuple:
         crawl_results = []
         crawl_type = None
 
-        if self.url_handler.is_txt(url):
+        if self.url_handler.is_txt(url) or self.url_handler.is_markdown(url):
             # Handle text files
             if self.progress_id:
                 self.progress_state.update({
                     "status": "crawling",
                     "percentage": 10,
-                    "log": "Detected text file, fetching content...",
+                    "log": "Detected text/markdown file, fetching content...",
                 })
+                # Keep heartbeat stage/progress in sync with direct emissions
+                self.progress_mapper.map_progress("crawling", 10)
                 await update_crawl_progress(self.progress_id, self.progress_state)
             crawl_results = await self.crawl_markdown_file(
                 url,
@@ -520,6 +552,52 @@ async def _crawl_by_url_type(self, url: str, request: Dict[str, Any]) -> tuple:
                 end_progress=20,
             )
             crawl_type = "text_file"
+
+            # Check if this is a link collection file and extract links
+            if crawl_results and len(crawl_results) > 0:
+                content = crawl_results[0].get('markdown', '')
+                if self.url_handler.is_link_collection_file(url, content):
+                    # Extract links from the content
+                    extracted_links = self.url_handler.extract_markdown_links(content, url)
+
+                    # Filter out self-referential links to avoid redundant crawling
+                    if extracted_links:
+                        original_count = len(extracted_links)
+                        extracted_links = [
+                            link for link in extracted_links
+                            if not self._is_self_link(link, url)
+                        ]
+                        self_filtered_count = original_count - len(extracted_links)
+                        if self_filtered_count > 0:
+                            logger.info(f"Filtered out {self_filtered_count} self-referential links from {original_count} extracted links")
+
+                    # Filter out binary files (PDFs, images, archives, etc.) to avoid wasteful crawling
+                    if extracted_links:
+                        original_count = len(extracted_links)
+                        extracted_links = [link for link in extracted_links if not self.url_handler.is_binary_file(link)]
+                        filtered_count = original_count - len(extracted_links)
+                        if filtered_count > 0:
+                            logger.info(f"Filtered out {filtered_count} binary files from {original_count} extracted links")
+
+                    if extracted_links:
+                        # Crawl the extracted links using batch crawling
+                        logger.info(f"Crawling {len(extracted_links)} extracted links from {url}")
+                        batch_results = await self.crawl_batch_with_progress(
+                            extracted_links,
+                            max_concurrent=request.get('max_concurrent'),  # None -> use DB settings
+                            progress_callback=await self._create_crawl_progress_callback("crawling"),
+                            start_progress=10,
+                            end_progress=20,
+                        )
+
+                        # Combine original text file results with batch results
+                        crawl_results.extend(batch_results)
+                        crawl_type = "link_collection_with_crawled_links"
+
+                        logger.info(f"Link collection crawling completed: {len(crawl_results)} total results (1 text file + {len(batch_results)} extracted links)")
+                    else:
+                        logger.info(f"No valid links found in link collection file: {url}")
+                        logger.info(f"Text file crawling completed: {len(crawl_results)} results")
 
         elif self.url_handler.is_sitemap(url):
             # Handle sitemaps

python/src/server/services/crawling/helpers/url_handler.py

@@ -6,7 +6,8 @@
 
 import hashlib
 import re
-from urllib.parse import urlparse
+from urllib.parse import urlparse, urljoin
+from typing import List, Optional
 
 from ....config.logfire_config import get_logger
 
@@ -32,6 +33,29 @@ def is_sitemap(url: str) -> bool:
         except Exception as e:
             logger.warning(f"Error checking if URL is sitemap: {e}")
             return False
+<<<<<<< HEAD
+
+    @staticmethod  
+    def is_markdown(url: str) -> bool:
+        """
+        Check if a URL points to a markdown file (.md, .mdx, .markdown).
+
+        Args:
+            url: URL to check
+
+        Returns:
+            True if URL is a markdown file, False otherwise
+        """
+        try:
+            parsed = urlparse(url)
+            # Normalize to lowercase and ignore query/fragment
+            path = parsed.path.lower()
+            return path.endswith(('.md', '.mdx', '.markdown'))
+        except Exception as e:
+            logger.warning(f"Error checking if URL is markdown file: {e}", exc_info=True)
+            return False
+=======
+>>>>>>> origin/main
 
     @staticmethod
     def is_txt(url: str) -> bool:
@@ -45,9 +69,15 @@ def is_txt(url: str) -> bool:
             True if URL is a text file, False otherwise
         """
         try:
+<<<<<<< HEAD
+            parsed = urlparse(url)
+            # Normalize to lowercase and ignore query/fragment
+            return parsed.path.lower().endswith('.txt')
+=======
             return url.endswith(".txt")
+>>>>>>> origin/main
         except Exception as e:
-            logger.warning(f"Error checking if URL is text file: {e}")
+            logger.warning(f"Error checking if URL is text file: {e}", exc_info=True)
             return False
 
     @staticmethod
@@ -166,7 +196,13 @@ def transform_github_url(url: str) -> str:
         match = re.match(github_file_pattern, url)
         if match:
             owner, repo, branch, path = match.groups()
+<<<<<<< HEAD
+            # Strip query parameters and fragments that break raw URLs
+            path = path.split('?', 1)[0].split('#', 1)[0]
+            raw_url = f'https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{path}'
+=======
             raw_url = f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{path}"
+>>>>>>> origin/main
             logger.info(f"Transformed GitHub file URL to raw: {url} -> {raw_url}")
             return raw_url
 
@@ -191,6 +227,164 @@ def generate_unique_source_id(url: str) -> str:
         for distinct canonical URLs, solving race condition issues when multiple crawls
         target the same domain.
 
+<<<<<<< HEAD
+        return url
+
+    @staticmethod
+    def extract_markdown_links(content: str, base_url: Optional[str] = None) -> List[str]:
+        """
+        Extract markdown-style links from text content.
+
+        Args:
+            content: Text content to extract links from
+            base_url: Base URL to resolve relative links against
+
+        Returns:
+            List of absolute URLs found in the content
+        """
+        try:
+            if not content:
+                return []
+
+            # Ultimate URL pattern with comprehensive format support:
+            #  1) [text](url) - markdown links
+            #  2) <https://...> - autolinks  
+            #  3) https://... - bare URLs with protocol
+            #  4) //example.com - protocol-relative URLs
+            #  5) www.example.com - scheme-less www URLs
+            combined_pattern = re.compile(
+                r'\[(?P<text>[^\]]*)\]\((?P<md>[^)]+)\)'      # named: md
+                r'|<\s*(?P<auto>https?://[^>\s]+)\s*>'        # named: auto
+                r'|(?P<bare>https?://[^\s<>()\[\]"]+)'        # named: bare
+                r'|(?P<proto>//[^\s<>()\[\]"]+)'              # named: protocol-relative
+                r'|(?P<www>www\.[^\s<>()\[\]"]+)'             # named: www.* without scheme
+            )
+
+            def _clean_url(u: str) -> str:
+                # Trim whitespace and comprehensive trailing punctuation
+                # Also remove invisible Unicode characters that can break URLs
+                import unicodedata
+                cleaned = u.strip().rstrip('.,;:)]>')
+                # Remove invisible/control characters but keep valid URL characters
+                cleaned = ''.join(c for c in cleaned if unicodedata.category(c) not in ('Cf', 'Cc'))
+                return cleaned
+
+            urls = []
+            for match in re.finditer(combined_pattern, content):
+                url = (
+                    match.group('md')
+                    or match.group('auto')
+                    or match.group('bare')
+                    or match.group('proto')
+                    or match.group('www')
+                )
+                if not url:
+                    continue
+                url = _clean_url(url)
+
+                # Skip empty URLs, anchors, and mailto links
+                if not url or url.startswith('#') or url.startswith('mailto:'):
+                    continue
+
+                # Normalize all URL formats to https://
+                if url.startswith('//'):
+                    url = f'https:{url}'
+                elif url.startswith('www.'):
+                    url = f'https://{url}'
+
+                # Convert relative URLs to absolute if base_url provided
+                if base_url and not url.startswith(('http://', 'https://')):
+                    try:
+                        url = urljoin(base_url, url)
+                    except Exception as e:
+                        logger.warning(f"Failed to resolve relative URL {url} with base {base_url}: {e}")
+                        continue
+
+                # Only include HTTP/HTTPS URLs
+                if url.startswith(('http://', 'https://')):
+                    urls.append(url)
+
+            # Remove duplicates while preserving order
+            seen = set()
+            unique_urls = []
+            for url in urls:
+                if url not in seen:
+                    seen.add(url)
+                    unique_urls.append(url)
+
+            logger.info(f"Extracted {len(unique_urls)} unique links from content")
+            return unique_urls
+
+        except Exception as e:
+            logger.error(f"Error extracting markdown links: {e}", exc_info=True)
+            return []
+
+    @staticmethod
+    def is_link_collection_file(url: str, content: Optional[str] = None) -> bool:
+        """
+        Check if a URL/file appears to be a link collection file like llms.txt.
+
+        Args:
+            url: URL to check
+            content: Optional content to analyze for link density
+
+        Returns:
+            True if file appears to be a link collection, False otherwise
+        """
+        try:
+            # Extract filename from URL
+            parsed = urlparse(url)
+            filename = parsed.path.split('/')[-1].lower()
+
+            # Check for specific link collection filenames
+            # Note: "full-*" or "*-full" patterns are NOT link collections - they contain complete content, not just links
+            link_collection_patterns = [
+                # .txt variants - files that typically contain lists of links
+                'llms.txt', 'links.txt', 'resources.txt', 'references.txt',
+                # .md/.mdx/.markdown variants
+                'llms.md', 'links.md', 'resources.md', 'references.md',
+                'llms.mdx', 'links.mdx', 'resources.mdx', 'references.mdx',
+                'llms.markdown', 'links.markdown', 'resources.markdown', 'references.markdown',
+            ]
+
+            # Direct filename match
+            if filename in link_collection_patterns:
+                logger.info(f"Detected link collection file by filename: {filename}")
+                return True
+
+            # Pattern-based detection for variations, but exclude "full" variants
+            # Only match files that are likely link collections, not complete content files
+            if filename.endswith(('.txt', '.md', '.mdx', '.markdown')):
+                # Exclude files with "full" in the name - these typically contain complete content, not just links
+                if 'full' not in filename:
+                    # Match files that start with common link collection prefixes
+                    base_patterns = ['llms', 'links', 'resources', 'references']
+                    if any(filename.startswith(pattern + '.') or filename.startswith(pattern + '-') for pattern in base_patterns):
+                        logger.info(f"Detected potential link collection file: {filename}")
+                        return True
+
+            # Content-based detection if content is provided
+            if content:
+                # Reuse extractor to avoid regex divergence and maintain consistency
+                extracted_links = URLHandler.extract_markdown_links(content)
+                total_links = len(extracted_links)
+
+                # Calculate link density (links per 100 characters)
+                content_length = len(content.strip())
+                if content_length > 0:
+                    link_density = (total_links * 100) / content_length
+
+                    # If more than 2% of content is links, likely a link collection
+                    if link_density > 2.0 and total_links > 3:
+                        logger.info(f"Detected link collection by content analysis: {total_links} links, density {link_density:.2f}%")
+                        return True
+
+            return False
+
+        except Exception as e:
+            logger.warning(f"Error checking if file is link collection: {e}", exc_info=True)
+            return False
+=======
         Uses 16-char SHA256 prefix (64 bits) which provides
         ~18 quintillion unique values. Collision probability
         is negligible for realistic usage (<1M sources).
@@ -400,3 +594,4 @@ def extract_display_name(url: str) -> str:
             logger.warning(f"Error extracting display name for {url}: {e}, using URL")
             # Fallback: return truncated URL
             return url[:50] + "..." if len(url) > 50 else url
+>>>>>>> origin/main

python/src/server/services/embeddings/contextual_embedding_service.py

@@ -219,4 +219,4 @@ async def generate_contextual_embeddings_batch(
     except Exception as e:
         search_logger.error(f"Error in contextual embedding batch: {e}")
         # Return non-contextual for all chunks
-        return [(chunk, False) for chunk in chunks]
+        return [(chunk, False) for chunk in chunks]