feat: Implement priority-based automatic discovery of llms.txt and sitemap.xml files

archon-ui-main/src/features/progress/components/CrawlingProgress.tsx

@@ -12,6 +12,7 @@ import { Button } from "../../ui/primitives";
 import { cn } from "../../ui/primitives/styles";
 import { useCrawlProgressPolling } from "../hooks";
 import type { ActiveOperation } from "../types/progress";
+import { isValidHttpUrl } from "../utils/urlValidation";
 
 interface CrawlingProgressProps {
   onSwitchToBrowse: () => void;
@@ -129,6 +130,7 @@ export const CrawlingProgress: React.FC<CrawlingProgressProps> = ({ onSwitchToBr
             "in_progress",
             "starting",
             "initializing",
+            "discovery",
             "analyzing",
             "storing",
             "source_creation",
@@ -245,6 +247,63 @@ export const CrawlingProgress: React.FC<CrawlingProgressProps> = ({ onSwitchToBr
                     )}
                   </div>
 
+                  {/* Discovery Information */}
+                  {operation.discovered_file && (
+                    <div className="pt-2 border-t border-white/10">
+                      <div className="flex items-center gap-2 mb-2">
+                        <span className="text-xs font-semibold text-cyan-400">Discovery Result</span>
+                        {operation.discovered_file_type && (
+                          <span className="px-2 py-0.5 text-xs rounded bg-cyan-500/10 border border-cyan-500/20 text-cyan-300">
+                            {operation.discovered_file_type}
+                          </span>
+                        )}
+                      </div>
+                      {isValidHttpUrl(operation.discovered_file) ? (
+                        <a
+                          href={operation.discovered_file}
+                          target="_blank"
+                          rel="noopener noreferrer"
+                          className="text-sm text-gray-400 hover:text-cyan-400 transition-colors truncate block"
+                        >
+                          {operation.discovered_file}
+                        </a>
+                      ) : (
+                        <span className="text-sm text-gray-400 truncate block">
+                          {operation.discovered_file}
+                        </span>
+                      )}
+                    </div>
+                  )}
+
+                  {/* Linked Files */}
+                  {operation.linked_files && operation.linked_files.length > 0 && (
+                    <div className="pt-2 border-t border-white/10">
+                      <div className="text-xs font-semibold text-cyan-400 mb-2">
+                        Following {operation.linked_files.length} Linked File
+                        {operation.linked_files.length > 1 ? "s" : ""}
+                      </div>
+                      <div className="space-y-1 max-h-32 overflow-y-auto">
+                        {operation.linked_files.map((file: string, idx: number) => (
+                          isValidHttpUrl(file) ? (
+                            <a
+                              key={idx}
+                              href={file}
+                              target="_blank"
+                              rel="noopener noreferrer"
+                              className="text-xs text-gray-400 hover:text-cyan-400 transition-colors truncate block"
+                            >
+                              • {file}
+                            </a>
+                          ) : (
+                            <span key={idx} className="text-xs text-gray-400 truncate block">
+                              • {file}
+                            </span>
+                          )
+                        ))}
+                      </div>
+                    </div>
+                  )}
+
                   {/* Current Action or Operation Type Info */}
                   {(operation.current_url || operation.operation_type) && (
                     <div className="pt-2 border-t border-white/10">

archon-ui-main/src/features/progress/types/progress.ts

@@ -6,6 +6,7 @@
 export type ProgressStatus =
   | "starting"
   | "initializing"
+  | "discovery"
   | "analyzing"
   | "crawling"
   | "processing"
@@ -24,7 +25,16 @@ export type ProgressStatus =
   | "cancelled"
   | "stopping";
 
-export type CrawlType = "normal" | "sitemap" | "llms-txt" | "text_file" | "refresh";
+export type CrawlType =
+  | "normal"
+  | "sitemap"
+  | "llms-txt"
+  | "text_file"
+  | "refresh"
+  | "llms_txt_with_linked_files"
+  | "llms_txt_linked_files"
+  | "discovery_single_file"
+  | "discovery_sitemap";
 export type UploadType = "document";
 
 export interface BaseProgressData {
@@ -48,6 +58,10 @@ export interface CrawlProgressData extends BaseProgressData {
   codeBlocksFound?: number;
   totalSummaries?: number;
   completedSummaries?: number;
+  // Discovery-related fields
+  discoveredFile?: string;
+  discoveredFileType?: string;
+  linkedFiles?: string[];
   originalCrawlParams?: {
     url: string;
     knowledge_type?: string;
@@ -100,6 +114,10 @@ export interface ActiveOperation {
     code_examples_found?: number;
     current_operation?: string;
   };
+  // Discovery information
+  discovered_file?: string;
+  discovered_file_type?: string;
+  linked_files?: string[];
 }
 
 export interface ActiveOperationsResponse {
@@ -127,6 +145,13 @@ export interface ProgressResponse {
   codeBlocksFound?: number;
   totalSummaries?: number;
   completedSummaries?: number;
+  // Discovery-related fields
+  discoveredFile?: string;
+  discovered_file?: string; // Snake case from backend
+  discoveredFileType?: string;
+  discovered_file_type?: string; // Snake case from backend
+  linkedFiles?: string[];
+  linked_files?: string[]; // Snake case from backend
   fileName?: string;
   fileSize?: number;
   chunksProcessed?: number;

archon-ui-main/src/features/progress/utils/urlValidation.ts

@@ -0,0 +1,44 @@
+/**
+ * Client-side URL validation utility for discovered files.
+ * Ensures only safe HTTP/HTTPS URLs are rendered as clickable links.
+ */
+
+const SAFE_PROTOCOLS = ["http:", "https:"];
+
+/**
+ * Validates that a URL is safe to render as a clickable link.
+ * Only allows http: and https: protocols.
+ *
+ * @param url - URL string to validate
+ * @returns true if URL is safe (http/https), false otherwise
+ */
+export function isValidHttpUrl(url: string | undefined | null): boolean {
+	if (!url || typeof url !== "string") {
+		return false;
+	}
+
+	// Trim whitespace
+	const trimmed = url.trim();
+	if (!trimmed) {
+		return false;
+	}
+
+	try {
+		const parsed = new URL(trimmed);
+
+		// Only allow http and https protocols
+		if (!SAFE_PROTOCOLS.includes(parsed.protocol)) {
+			return false;
+		}
+
+		// Basic hostname validation (must have at least one dot or be localhost)
+		if (!parsed.hostname.includes(".") && parsed.hostname !== "localhost") {
+			return false;
+		}
+
+		return true;
+	} catch {
+		// URL parsing failed - not a valid URL
+		return false;
+	}
+}

python/pyproject.toml

@@ -59,6 +59,7 @@ server = [
     "pydantic>=2.0.0",
     "python-dotenv>=1.0.0",
     "docker>=6.1.0",
+    "tldextract>=5.0.0",
     # Logging
     "logfire>=0.30.0",
     # Testing (needed for UI-triggered tests)
@@ -116,6 +117,7 @@ all = [
     "cryptography>=41.0.0",
     "slowapi>=0.1.9",
     "docker>=6.1.0",
+    "tldextract>=5.0.0",
     "logfire>=0.30.0",
     # MCP specific (mcp version)
     "mcp==1.12.2",