feat: add provider-neutral external knowledge sources

AGENTS.md

@@ -37,6 +37,8 @@ Wegent is an open-source AI-native operating system for defining, organizing, an
 - English version: [`docs/en/developer-guide/local-device-command-rpc.md`](docs/en/developer-guide/local-device-command-rpc.md)
 - External knowledge MCP (trusted external knowledge integrations): [`docs/zh/developer-guide/external-knowledge-mcp.md`](docs/zh/developer-guide/external-knowledge-mcp.md)
 - English version: [`docs/en/developer-guide/external-knowledge-mcp.md`](docs/en/developer-guide/external-knowledge-mcp.md)
+- External knowledge sources (provider-neutral runtime retrieval sources): [`docs/zh/developer-guide/external-knowledge-sources.md`](docs/zh/developer-guide/external-knowledge-sources.md)
+- English version: [`docs/en/developer-guide/external-knowledge-sources.md`](docs/en/developer-guide/external-knowledge-sources.md)
 - User runtime config (encrypted user CLI auth storage and device sync): [`docs/zh/developer-guide/user-runtime-config.md`](docs/zh/developer-guide/user-runtime-config.md)
 - English version: [`docs/en/developer-guide/user-runtime-config.md`](docs/en/developer-guide/user-runtime-config.md)
 - Sandbox workspace archive (preserve Sandbox files across idle cleanup and recreation): [`docs/zh/developer-guide/sandbox-workspace-archive.md`](docs/zh/developer-guide/sandbox-workspace-archive.md)

backend/app/api/api.py

@@ -76,6 +76,7 @@
     callback_router,
     chat_storage_router,
     conversion_callback_router,
+    knowledge_router,
     object_storage_router,
     rag_content_router,
     services_router,
@@ -272,6 +273,9 @@
 if not settings.STANDALONE_MODE:
     api_router.include_router(rag_router, prefix="/internal", tags=["internal-rag"])
 
+api_router.include_router(
+    knowledge_router, prefix="/internal", tags=["internal-knowledge"]
+)
 api_router.include_router(skills_router, prefix="/internal", tags=["internal-skills"])
 api_router.include_router(tables_router, prefix="/internal", tags=["internal-tables"])
 api_router.include_router(

backend/app/api/endpoints/adapter/task_knowledge_bases.py

@@ -9,14 +9,21 @@
 import logging
 from typing import List, Optional
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, HTTPException, status
 from pydantic import BaseModel
 from sqlalchemy.orm import Session
 
 from app.api.dependencies import get_db
 from app.core import security
 from app.models.user import User
+from app.schemas.external_knowledge import ExternalKnowledgeRef
+from app.services.chat.external_knowledge_refs import (
+    extract_task_external_knowledge_refs,
+    remove_task_external_knowledge_ref,
+)
 from app.services.knowledge import TaskKnowledgeBaseService
+from app.services.rag.sources import ExternalRefValidationError
+from app.stores.tasks import task_access_store
 
 router = APIRouter()
 logger = logging.getLogger(__name__)
@@ -64,6 +71,43 @@ class UnbindKnowledgeBaseResponse(BaseModel):
     kb_namespace: str
 
 
+class BoundExternalKnowledgeRefListResponse(BaseModel):
+    """Response for task-level external knowledge refs."""
+
+    items: List[ExternalKnowledgeRef]
+    total: int
+
+
+class RemoveExternalKnowledgeRefRequest(BaseModel):
+    """Request to remove one task-level external knowledge ref."""
+
+    ref: ExternalKnowledgeRef
+
+
+class RemoveExternalKnowledgeRefResponse(BaseModel):
+    """Response for removing one external knowledge ref."""
+
+    message: str
+    items: List[ExternalKnowledgeRef]
+    total: int
+
+
+def _get_accessible_task_or_404(db: Session, task_id: int, user_id: int):
+    """Return an active task if the user owns or can access it."""
+    if not task_access_store.is_member(db, task_id=task_id, user_id=user_id):
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Task not found",
+        )
+    task = task_access_store.get_task(db, task_id=task_id)
+    if not task:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Task not found",
+        )
+    return task
+
+
 # ============ API Endpoints ============
 
 
@@ -163,3 +207,60 @@ def unbind_knowledge_base(
         kb_name=kb_name,
         kb_namespace=kb_namespace,
     )
+
+
+@router.get(
+    "/{task_id}/external-knowledge-refs",
+    response_model=BoundExternalKnowledgeRefListResponse,
+)
+def get_bound_external_knowledge_refs(
+    task_id: int,
+    current_user: User = Depends(security.get_current_user),
+    db: Session = Depends(get_db),
+):
+    """
+    Get external knowledge refs bound to a task.
+    User must have access to the task to view.
+    """
+    task = _get_accessible_task_or_404(db, task_id, current_user.id)
+    refs = extract_task_external_knowledge_refs(task)
+    return BoundExternalKnowledgeRefListResponse(items=refs, total=len(refs))
+
+
+@router.post(
+    "/{task_id}/external-knowledge-refs/remove",
+    response_model=RemoveExternalKnowledgeRefResponse,
+)
+def remove_bound_external_knowledge_ref(
+    task_id: int,
+    request: RemoveExternalKnowledgeRefRequest,
+    current_user: User = Depends(security.get_current_user),
+    db: Session = Depends(get_db),
+):
+    """
+    Remove one external knowledge ref from a task-level binding.
+    User must have access to the task to modify bindings.
+    """
+    task = _get_accessible_task_or_404(db, task_id, current_user.id)
+    try:
+        refs = remove_task_external_knowledge_ref(
+            db,
+            task,
+            request.ref.model_dump(exclude_none=True),
+        )
+        db.commit()
+    except ExternalRefValidationError as exc:
+        db.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(exc),
+        ) from exc
+    except Exception:
+        db.rollback()
+        raise
+
+    return RemoveExternalKnowledgeRefResponse(
+        message="External knowledge ref removed successfully",
+        items=refs,
+        total=len(refs),
+    )

backend/app/api/endpoints/internal/__init__.py

@@ -11,6 +11,7 @@
 from .callback import router as callback_router
 from .chat_storage import router as chat_storage_router
 from .conversion_callback import router as conversion_callback_router
+from .knowledge import router as knowledge_router
 from .object_storage import router as object_storage_router
 from .rag_content import router as rag_content_router
 from .services import router as services_router
@@ -30,6 +31,7 @@
     "callback_router",
     "chat_storage_router",
     "conversion_callback_router",
+    "knowledge_router",
     "object_storage_router",
     "rag_content_router",
     "services_router",

backend/app/api/endpoints/internal/knowledge.py

@@ -0,0 +1,135 @@
+# SPDX-FileCopyrightText: 2025 Weibo, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Internal knowledge capability endpoints for service-to-service calls."""
+
+import logging
+from typing import Literal, Optional
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel, Field
+
+from app.schemas.external_knowledge import ExternalKnowledgeRef
+from app.services.auth.internal_service_token import verify_internal_service_token
+from app.services.rag.sources import (
+    RetrievalContext,
+    retrieval_source_registry,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(
+    prefix="/knowledge",
+    tags=["internal-knowledge"],
+    dependencies=[Depends(verify_internal_service_token)],
+)
+
+
+class KnowledgeListDocumentsRequest(BaseModel):
+    """Request for listing documents from mounted knowledge sources."""
+
+    user_id: Optional[int] = Field(default=None, description="Current user ID")
+    user_name: Optional[str] = Field(default=None, description="Current user name")
+    external_knowledge_refs: list[ExternalKnowledgeRef] = Field(
+        ..., min_length=1, description="External knowledge refs to list"
+    )
+    limit: int = Field(default=50, ge=1, le=100)
+    offset: int = Field(default=0, ge=0)
+
+
+class KnowledgeDocumentItem(BaseModel):
+    """Normalized knowledge document list item."""
+
+    provider: str
+    source_id: str
+    source_name: Optional[str] = None
+    document_id: str
+    title: str
+    node_id: Optional[str] = None
+    parent_id: Optional[str] = None
+    mime_type: Optional[str] = None
+    file_extension: Optional[str] = None
+    source_uri: Optional[str] = None
+
+
+class KnowledgeListDocumentsResponse(BaseModel):
+    """Response for knowledge document listing."""
+
+    documents: list[KnowledgeDocumentItem]
+    total_returned: int
+    pagination_scope: Literal["per_provider"] = "per_provider"
+    warnings: list[str] = Field(default_factory=list)
+
+
+@router.post("/list-documents", response_model=KnowledgeListDocumentsResponse)
+async def list_documents(
+    request: KnowledgeListDocumentsRequest,
+):
+    """List documents from providers that disclose document listing support."""
+    if not request.user_id:
+        raise HTTPException(
+            status_code=400,
+            detail="user_id is required for external knowledge document listing",
+        )
+
+    refs_by_provider: dict[str, list[ExternalKnowledgeRef]] = {}
+    for ref in request.external_knowledge_refs:
+        refs_by_provider.setdefault(ref.provider, []).append(ref)
+
+    ctx = RetrievalContext(user_id=request.user_id, user_name=request.user_name)
+    documents: list[KnowledgeDocumentItem] = []
+    warnings: list[str] = []
+
+    for provider_name, provider_refs in refs_by_provider.items():
+        provider = retrieval_source_registry.get(provider_name)
+        if provider is None:
+            warnings.append(f"Knowledge provider is not registered: {provider_name}")
+            continue
+
+        list_provider_documents = getattr(provider, "list_documents", None)
+        if not callable(list_provider_documents):
+            warnings.append(
+                f"Knowledge provider does not support document listing: {provider_name}"
+            )
+            continue
+
+        try:
+            result = await list_provider_documents(
+                provider_refs,
+                ctx,
+                limit=request.limit,
+                offset=request.offset,
+            )
+        except Exception:
+            logger.warning(
+                "[internal_knowledge] Document listing provider failed: %s",
+                provider_name,
+                exc_info=True,
+            )
+            warnings.append(f"Knowledge provider listing failed: {provider_name}")
+            continue
+
+        warnings.extend(result.warnings)
+        documents.extend(
+            KnowledgeDocumentItem(
+                provider=document.provider,
+                source_id=document.source_id,
+                source_name=document.source_name,
+                document_id=document.document_id,
+                title=document.title,
+                node_id=document.node_id,
+                parent_id=document.parent_id,
+                mime_type=document.mime_type,
+                file_extension=document.file_extension,
+                source_uri=document.source_uri,
+            )
+            for document in result.documents
+        )
+
+    return KnowledgeListDocumentsResponse(
+        documents=documents,
+        total_returned=len(documents),
+        pagination_scope="per_provider",
+        warnings=warnings,
+    )