RAG Enhancements (Page Level Retrieval)

CLAUDE.md

@@ -268,6 +268,8 @@ When connected to Claude/Cursor/Windsurf, the following tools are available:
 - `archon:rag_search_knowledge_base` - Search knowledge base for relevant content
 - `archon:rag_search_code_examples` - Find code snippets in the knowledge base
 - `archon:rag_get_available_sources` - List available knowledge sources
+- `archon:rag_list_pages_for_source` - List all pages for a given source (browse documentation structure)
+- `archon:rag_read_full_page` - Retrieve full page content by page_id or URL
 
 ### Project Management
 

migration/0.1.0/010_add_page_metadata_table.sql

@@ -0,0 +1,74 @@
+-- =====================================================
+-- Add archon_page_metadata table for page-based RAG retrieval
+-- =====================================================
+-- This migration adds support for storing complete documentation pages
+-- alongside chunks for improved agent context retrieval.
+--
+-- Features:
+-- - Full page content storage with metadata
+-- - Support for llms-full.txt section-based pages
+-- - Foreign key relationship from chunks to pages
+-- =====================================================
+
+-- Create archon_page_metadata table
+CREATE TABLE IF NOT EXISTS archon_page_metadata (
+    -- Primary identification
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    source_id TEXT NOT NULL,
+    url TEXT NOT NULL,
+
+    -- Content
+    full_content TEXT NOT NULL,
+
+    -- Section metadata (for llms-full.txt H1 sections)
+    section_title TEXT,
+    section_order INT DEFAULT 0,
+
+    -- Statistics
+    word_count INT NOT NULL,
+    char_count INT NOT NULL,
+    chunk_count INT NOT NULL DEFAULT 0,
+
+    -- Timestamps
+    created_at TIMESTAMPTZ DEFAULT NOW(),
+    updated_at TIMESTAMPTZ DEFAULT NOW(),
+
+    -- Flexible metadata storage
+    metadata JSONB DEFAULT '{}'::jsonb,
+
+    -- Constraints
+    CONSTRAINT archon_page_metadata_url_unique UNIQUE(url),
+    CONSTRAINT archon_page_metadata_source_fk FOREIGN KEY (source_id)
+        REFERENCES archon_sources(source_id) ON DELETE CASCADE
+);
+
+-- Add page_id foreign key to archon_crawled_pages
+-- This links chunks back to their parent page
+-- NULLABLE because existing chunks won't have a page_id yet
+ALTER TABLE archon_crawled_pages
+ADD COLUMN IF NOT EXISTS page_id UUID REFERENCES archon_page_metadata(id) ON DELETE SET NULL;
+
+-- Create indexes for query performance
+CREATE INDEX IF NOT EXISTS idx_archon_page_metadata_source_id ON archon_page_metadata(source_id);
+CREATE INDEX IF NOT EXISTS idx_archon_page_metadata_url ON archon_page_metadata(url);
+CREATE INDEX IF NOT EXISTS idx_archon_page_metadata_section ON archon_page_metadata(source_id, section_title, section_order);
+CREATE INDEX IF NOT EXISTS idx_archon_page_metadata_created_at ON archon_page_metadata(created_at);
+CREATE INDEX IF NOT EXISTS idx_archon_page_metadata_metadata ON archon_page_metadata USING GIN(metadata);
+CREATE INDEX IF NOT EXISTS idx_archon_crawled_pages_page_id ON archon_crawled_pages(page_id);
+
+-- Add comments to document the table structure
+COMMENT ON TABLE archon_page_metadata IS 'Stores complete documentation pages for agent retrieval';
+COMMENT ON COLUMN archon_page_metadata.source_id IS 'References the source this page belongs to';
+COMMENT ON COLUMN archon_page_metadata.url IS 'Unique URL of the page (synthetic for llms-full.txt sections with #anchor)';
+COMMENT ON COLUMN archon_page_metadata.full_content IS 'Complete markdown/text content of the page';
+COMMENT ON COLUMN archon_page_metadata.section_title IS 'H1 section title for llms-full.txt pages';
+COMMENT ON COLUMN archon_page_metadata.section_order IS 'Order of section in llms-full.txt file (0-based)';
+COMMENT ON COLUMN archon_page_metadata.word_count IS 'Number of words in full_content';
+COMMENT ON COLUMN archon_page_metadata.char_count IS 'Number of characters in full_content';
+COMMENT ON COLUMN archon_page_metadata.chunk_count IS 'Number of chunks created from this page';
+COMMENT ON COLUMN archon_page_metadata.metadata IS 'Flexible JSON metadata (page_type, knowledge_type, tags, etc)';
+COMMENT ON COLUMN archon_crawled_pages.page_id IS 'Foreign key linking chunk to parent page';
+
+-- =====================================================
+-- MIGRATION COMPLETE
+-- =====================================================

python/src/mcp_server/features/rag/rag_tools.py

@@ -77,7 +77,11 @@ async def rag_get_available_sources(ctx: Context) -> str:
 
     @mcp.tool()
     async def rag_search_knowledge_base(
-        ctx: Context, query: str, source_id: str | None = None, match_count: int = 5
+        ctx: Context,
+        query: str,
+        source_id: str | None = None,
+        match_count: int = 5,
+        return_mode: str = "pages"
     ) -> str:
         """
         Search knowledge base for relevant content using RAG.
@@ -90,20 +94,31 @@ async def rag_search_knowledge_base(
                       This is the 'id' field from available sources, NOT a URL or domain name.
                       Example: "src_1234abcd" not "docs.anthropic.com"
             match_count: Max results (default: 5)
+            return_mode: "pages" (default, full pages with metadata) or "chunks" (raw text chunks)
 
         Returns:
             JSON string with structure:
             - success: bool - Operation success status
-            - results: list[dict] - Array of matching documents with content and metadata
+            - results: list[dict] - Array of pages/chunks with content and metadata
+                      Pages include: page_id, url, title, preview, word_count, chunk_matches
+                      Chunks include: content, metadata, similarity
+            - return_mode: str - Mode used ("pages" or "chunks")
             - reranked: bool - Whether results were reranked
             - error: str|null - Error description if success=false
+
+        Note: Use "pages" mode for better context (recommended), or "chunks" for raw granular results.
+        After getting pages, use rag_read_full_page() to retrieve complete page content.
         """
         try:
             api_url = get_api_url()
             timeout = httpx.Timeout(30.0, connect=5.0)
 
             async with httpx.AsyncClient(timeout=timeout) as client:
-                request_data = {"query": query, "match_count": match_count}
+                request_data = {
+                    "query": query,
+                    "match_count": match_count,
+                    "return_mode": return_mode
+                }
                 if source_id:
                     request_data["source"] = source_id
 
@@ -115,6 +130,7 @@ async def rag_search_knowledge_base(
                         {
                             "success": True,
                             "results": result.get("results", []),
+                            "return_mode": result.get("return_mode", return_mode),
                             "reranked": result.get("reranked", False),
                             "error": None,
                         },
@@ -198,5 +214,148 @@ async def rag_search_code_examples(
             logger.error(f"Error searching code examples: {e}")
             return json.dumps({"success": False, "results": [], "error": str(e)}, indent=2)
 
+    @mcp.tool()
+    async def rag_list_pages_for_source(
+        ctx: Context, source_id: str, section: str | None = None
+    ) -> str:
+        """
+        List all pages for a given knowledge source.
+
+        Use this after rag_get_available_sources() to see all pages in a source.
+        Useful for browsing documentation structure or finding specific pages.
+
+        Args:
+            source_id: Source ID from rag_get_available_sources() (e.g., "src_1234abcd")
+            section: Optional filter for llms-full.txt section title (e.g., "# Core Concepts")
+
+        Returns:
+            JSON string with structure:
+            - success: bool - Operation success status
+            - pages: list[dict] - Array of page objects with id, url, section_title, word_count
+            - total: int - Total number of pages
+            - source_id: str - The source ID that was queried
+            - error: str|null - Error description if success=false
+
+        Example workflow:
+            1. Call rag_get_available_sources() to get source_id
+            2. Call rag_list_pages_for_source(source_id) to see all pages
+            3. Call rag_read_full_page(page_id) to read specific pages
+        """
+        try:
+            api_url = get_api_url()
+            timeout = httpx.Timeout(30.0, connect=5.0)
+
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                params = {"source_id": source_id}
+                if section:
+                    params["section"] = section
+
+                response = await client.get(
+                    urljoin(api_url, "/api/pages"),
+                    params=params
+                )
+
+                if response.status_code == 200:
+                    result = response.json()
+                    return json.dumps(
+                        {
+                            "success": True,
+                            "pages": result.get("pages", []),
+                            "total": result.get("total", 0),
+                            "source_id": result.get("source_id", source_id),
+                            "error": None,
+                        },
+                        indent=2,
+                    )
+                else:
+                    error_detail = response.text
+                    return json.dumps(
+                        {
+                            "success": False,
+                            "pages": [],
+                            "total": 0,
+                            "source_id": source_id,
+                            "error": f"HTTP {response.status_code}: {error_detail}",
+                        },
+                        indent=2,
+                    )
+
+        except Exception as e:
+            logger.error(f"Error listing pages for source {source_id}: {e}")
+            return json.dumps(
+                {
+                    "success": False,
+                    "pages": [],
+                    "total": 0,
+                    "source_id": source_id,
+                    "error": str(e)
+                },
+                indent=2
+            )
+
+    @mcp.tool()
+    async def rag_read_full_page(
+        ctx: Context, page_id: str | None = None, url: str | None = None
+    ) -> str:
+        """
+        Retrieve full page content from knowledge base.
+        Use this to get complete page content after RAG search.
+
+        Args:
+            page_id: Page UUID from search results (e.g., "550e8400-e29b-41d4-a716-446655440000")
+            url: Page URL (e.g., "https://docs.example.com/getting-started")
+
+        Note: Provide EITHER page_id OR url, not both.
+
+        Returns:
+            JSON string with structure:
+            - success: bool
+            - page: dict with full_content, title, url, metadata
+            - error: str|null
+        """
+        try:
+            if not page_id and not url:
+                return json.dumps(
+                    {"success": False, "error": "Must provide either page_id or url"},
+                    indent=2
+                )
+
+            api_url = get_api_url()
+            timeout = httpx.Timeout(30.0, connect=5.0)
+
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                if page_id:
+                    response = await client.get(urljoin(api_url, f"/api/pages/{page_id}"))
+                else:
+                    response = await client.get(
+                        urljoin(api_url, "/api/pages/by-url"),
+                        params={"url": url}
+                    )
+
+                if response.status_code == 200:
+                    page_data = response.json()
+                    return json.dumps(
+                        {
+                            "success": True,
+                            "page": page_data,
+                            "error": None,
+                        },
+                        indent=2,
+                    )
+                else:
+                    error_detail = response.text
+                    return json.dumps(
+                        {
+                            "success": False,
+                            "page": None,
+                            "error": f"HTTP {response.status_code}: {error_detail}",
+                        },
+                        indent=2,
+                    )
+
+        except Exception as e:
+            logger.error(f"Error reading page: {e}")
+            return json.dumps({"success": False, "page": None, "error": str(e)}, indent=2)
+
     # Log successful registration
     logger.info("✓ RAG tools registered (HTTP-based version)")