feat: Implement priority-based automatic discovery of llms.txt and sitemap.xml files

archon-ui-main/src/features/progress/components/CrawlingProgress.tsx

@@ -129,6 +129,7 @@ export const CrawlingProgress: React.FC<CrawlingProgressProps> = ({ onSwitchToBr
             "in_progress",
             "starting",
             "initializing",
+            "discovery",
             "analyzing",
             "storing",
             "source_creation",
@@ -245,6 +246,51 @@ export const CrawlingProgress: React.FC<CrawlingProgressProps> = ({ onSwitchToBr
                     )}
                   </div>
 
+                  {/* Discovery Information */}
+                  {(operation as any).discovered_file && (
+                    <div className="pt-2 border-t border-white/10">
+                      <div className="flex items-center gap-2 mb-2">
+                        <span className="text-xs font-semibold text-cyan-400">Discovery Result</span>
+                        {(operation as any).discovered_file_type && (
+                          <span className="px-2 py-0.5 text-xs rounded bg-cyan-500/10 border border-cyan-500/20 text-cyan-300">
+                            {(operation as any).discovered_file_type}
+                          </span>
+                        )}
+                      </div>
+                      <a
+                        href={(operation as any).discovered_file}
+                        target="_blank"
+                        rel="noopener noreferrer"
+                        className="text-sm text-gray-400 hover:text-cyan-400 transition-colors truncate block"
+                      >
+                        {(operation as any).discovered_file}
+                      </a>
+                    </div>
+                  )}
+
+                  {/* Linked Files */}
+                  {(operation as any).linked_files && (operation as any).linked_files.length > 0 && (
+                    <div className="pt-2 border-t border-white/10">
+                      <div className="text-xs font-semibold text-cyan-400 mb-2">
+                        Following {(operation as any).linked_files.length} Linked File
+                        {(operation as any).linked_files.length > 1 ? "s" : ""}
+                      </div>
+                      <div className="space-y-1 max-h-32 overflow-y-auto">
+                        {(operation as any).linked_files.map((file: string, idx: number) => (
+                          <a
+                            key={idx}
+                            href={file}
+                            target="_blank"
+                            rel="noopener noreferrer"
+                            className="text-xs text-gray-400 hover:text-cyan-400 transition-colors truncate block"
+                          >
+                            • {file}
+                          </a>
+                        ))}
+                      </div>
+                    </div>
+                  )}
+
                   {/* Current Action or Operation Type Info */}
                   {(operation.current_url || operation.operation_type) && (
                     <div className="pt-2 border-t border-white/10">

archon-ui-main/src/features/progress/types/progress.ts

@@ -6,6 +6,7 @@
 export type ProgressStatus =
   | "starting"
   | "initializing"
+  | "discovery"
   | "analyzing"
   | "crawling"
   | "processing"
@@ -24,7 +25,16 @@ export type ProgressStatus =
   | "cancelled"
   | "stopping";
 
-export type CrawlType = "normal" | "sitemap" | "llms-txt" | "text_file" | "refresh";
+export type CrawlType =
+  | "normal"
+  | "sitemap"
+  | "llms-txt"
+  | "text_file"
+  | "refresh"
+  | "llms_txt_with_linked_files"
+  | "llms_txt_linked_files"
+  | "discovery_single_file"
+  | "discovery_sitemap";
 export type UploadType = "document";
 
 export interface BaseProgressData {
@@ -48,6 +58,10 @@ export interface CrawlProgressData extends BaseProgressData {
   codeBlocksFound?: number;
   totalSummaries?: number;
   completedSummaries?: number;
+  // Discovery-related fields
+  discoveredFile?: string;
+  discoveredFileType?: string;
+  linkedFiles?: string[];
   originalCrawlParams?: {
     url: string;
     knowledge_type?: string;
@@ -127,6 +141,13 @@ export interface ProgressResponse {
   codeBlocksFound?: number;
   totalSummaries?: number;
   completedSummaries?: number;
+  // Discovery-related fields
+  discoveredFile?: string;
+  discovered_file?: string; // Snake case from backend
+  discoveredFileType?: string;
+  discovered_file_type?: string; // Snake case from backend
+  linkedFiles?: string[];
+  linked_files?: string[]; // Snake case from backend
   fileName?: string;
   fileSize?: number;
   chunksProcessed?: number;