restructured entire gptr agent

backend/report_type/basic_report/basic_report.py

@@ -1,7 +1,7 @@
 from fastapi import WebSocket
+from typing import Any
 
-from gpt_researcher.master.agent import GPTResearcher
-from gpt_researcher.utils.enum import Tone
+from gpt_researcher import GPTResearcher
 
 
 class BasicReport:
@@ -11,7 +11,7 @@ def __init__(
         report_type: str,
         report_source: str,
         source_urls,
-        tone: Tone,
+        tone: Any,
         config_path: str,
         websocket: WebSocket,
         headers=None

backend/report_type/detailed_report/detailed_report.py

@@ -1,17 +1,8 @@
 import asyncio
-from typing import List, Dict, Set, Optional
+from typing import List, Dict, Set, Optional, Any
 from fastapi import WebSocket
 
-from gpt_researcher.master.actions import (
-    add_references,
-    extract_headers,
-    extract_sections,
-    table_of_contents,
-)
-from gpt_researcher.master.agent import GPTResearcher
-from gpt_researcher.utils.enum import Tone
-from gpt_researcher.utils.validators import Subtopics
-from gpt_researcher.master.actions.markdown_processing import extract_headers
+from gpt_researcher import GPTResearcher
 
 
 class DetailedReport:
@@ -22,7 +13,7 @@ def __init__(
         report_source: str,
         source_urls: List[str] = [],
         config_path: str = None,
-        tone: Tone = Tone.Formal,
+        tone: Any = "",
         websocket: WebSocket = None,
         subtopics: List[Dict] = [],
         headers: Optional[Dict] = None
@@ -37,7 +28,7 @@ def __init__(
         self.subtopics = subtopics
         self.headers = headers or {}
 
-        self.main_task_assistant = GPTResearcher(
+        self.gpt_researcher = GPTResearcher(
             query=self.query,
             report_type="research_report",
             report_source=self.report_source,
@@ -56,22 +47,22 @@ def __init__(
     async def run(self) -> str:
         await self._initial_research()
         subtopics = await self._get_all_subtopics()
-        report_introduction = await self.main_task_assistant.write_introduction()
+        report_introduction = await self.gpt_researcher.write_introduction()
         _, report_body = await self._generate_subtopic_reports(subtopics)
-        self.main_task_assistant.visited_urls.update(self.global_urls)
+        self.gpt_researcher.visited_urls.update(self.global_urls)
         report = await self._construct_detailed_report(report_introduction, report_body)
         return report
 
     async def _initial_research(self) -> None:
-        await self.main_task_assistant.conduct_research()
-        self.global_context = self.main_task_assistant.context
-        self.global_urls = self.main_task_assistant.visited_urls
+        await self.gpt_researcher.conduct_research()
+        self.global_context = self.gpt_researcher.context
+        self.global_urls = self.gpt_researcher.visited_urls
 
     async def _get_all_subtopics(self) -> List[Dict]:
-        subtopics_data: Subtopics = await self.main_task_assistant.get_subtopics()
+        subtopics_data = await self.gpt_researcher.get_subtopics()
 
         all_subtopics = []
-        if isinstance(subtopics_data, Subtopics):
+        if subtopics_data and subtopics_data.subtopics:
             for subtopic in subtopics_data.subtopics:
                 all_subtopics.append({"task": subtopic.task})
         else:
@@ -102,8 +93,8 @@ async def _get_subtopic_report(self, subtopic: Dict) -> Dict[str, str]:
             parent_query=self.query,
             subtopics=self.subtopics,
             visited_urls=self.global_urls,
-            agent=self.main_task_assistant.agent,
-            role=self.main_task_assistant.role,
+            agent=self.gpt_researcher.agent,
+            role=self.gpt_researcher.role,
             tone=self.tone,
         )
 
@@ -115,7 +106,7 @@ async def _get_subtopic_report(self, subtopic: Dict) -> Dict[str, str]:
         if not isinstance(draft_section_titles, str):
             draft_section_titles = str(draft_section_titles)
 
-        parse_draft_section_titles = extract_headers(draft_section_titles)
+        parse_draft_section_titles = self.gpt_researcher.extract_headers(draft_section_titles)
         parse_draft_section_titles_text = [header.get(
             "text", "") for header in parse_draft_section_titles]
 
@@ -125,21 +116,21 @@ async def _get_subtopic_report(self, subtopic: Dict) -> Dict[str, str]:
 
         subtopic_report = await subtopic_assistant.write_report(self.existing_headers, relevant_contents)
 
-        self.global_written_sections.extend(extract_sections(subtopic_report))
+        self.global_written_sections.extend(self.gpt_researcher.extract_sections(subtopic_report))
         self.global_context = list(set(subtopic_assistant.context))
         self.global_urls.update(subtopic_assistant.visited_urls)
 
         self.existing_headers.append({
             "subtopic task": current_subtopic_task,
-            "headers": extract_headers(subtopic_report),
+            "headers": self.gpt_researcher.extract_headers(subtopic_report),
         })
 
         return {"topic": subtopic, "report": subtopic_report}
 
     async def _construct_detailed_report(self, introduction: str, report_body: str) -> str:
-        toc = table_of_contents(report_body)
-        conclusion = await self.main_task_assistant.write_report_conclusion(report_body)
-        conclusion_with_references = add_references(
-            conclusion, self.main_task_assistant.visited_urls)
+        toc = self.gpt_researcher.table_of_contents(report_body)
+        conclusion = await self.gpt_researcher.write_report_conclusion(report_body)
+        conclusion_with_references = self.gpt_researcher.add_references(
+            conclusion, self.gpt_researcher.visited_urls)
         report = f"{introduction}\n\n{toc}\n\n{report_body}\n\n{conclusion_with_references}"
         return report

backend/server/server.py

@@ -3,22 +3,16 @@
 from typing import Dict, List
 
 from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect, File, UploadFile, Header
-from fastapi.responses import JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from pydantic import BaseModel
 
-from backend.server.server_utils import generate_report_files
 from backend.server.websocket_manager import WebSocketManager
-from multi_agents.main import run_research_task
-from gpt_researcher.document.document import DocumentLoader
-from gpt_researcher.master.actions import stream_output
 from backend.server.server_utils import (
-    sanitize_filename, handle_start_command, handle_human_feedback,
-    generate_report_files, send_file_paths, get_config_dict,
+    get_config_dict,
     update_environment_variables, handle_file_upload, handle_file_deletion,
-    execute_multi_agents, handle_websocket_communication, extract_command_data
+    execute_multi_agents, handle_websocket_communication
 )
 
 # Models

backend/server/websocket_manager.py

@@ -7,7 +7,7 @@
 from backend.report_type import BasicReport, DetailedReport
 from gpt_researcher.utils.enum import ReportType, Tone
 from multi_agents.main import run_research_task
-from gpt_researcher.master.actions import stream_output  # Import stream_output
+from gpt_researcher.actions import stream_output  # Import stream_output
 
 
 class WebSocketManager:

docs/blog/2024-05-19-gptr-langgraph/index.md

@@ -2,7 +2,7 @@
 slug: gptr-langgraph
 title: How to Build the Ultimate Research Multi-Agent Assistant
 authors: [assafe]
-tags: [multi-agents, gpt-researcher, langchain, langgraph]
+tags: [multi-skills, gpt-researcher, langchain, langgraph]
 ---
 ![Header](./blog-langgraph.jpeg)
 # Introducing the GPT Researcher Multi-Agent Assistant
@@ -114,7 +114,7 @@ As you can see above, we’ve created an instance of the Research agent. Now let
 
 ```python
 def init_research_team(self):
-    # Initialize agents
+    # Initialize skills
     editor_agent = EditorAgent(self.task)
     research_agent = ResearchAgent()
     writer_agent = WriterAgent()

frontend/styles.css

@@ -177,7 +177,7 @@ footer p {
 a {
     color: #ffffff;
     font-weight: bold;
-    text-decoration: none;
+    text-decoration: underline;
 }
 
 a:hover {

gpt_researcher/__init__.py

@@ -1,4 +1,3 @@
-from .master import GPTResearcher
-from .config import Config
+from .agent import GPTResearcher
 
-__all__ = ['GPTResearcher', 'Config']
+__all__ = ['GPTResearcher']

gpt_researcher/master/actions/query_processing.py → gpt_researcher/actions/query_processing.py

@@ -1,7 +1,7 @@
 import json
 import re
 import json_repair
-from ...utils.llm import create_chat_completion
+from gpt_researcher.utils.llm import create_chat_completion
 from ..prompts import auto_agent_instructions, generate_search_queries_prompt
 from typing import Any
 

gpt_researcher/master/actions/report_generation.py → gpt_researcher/actions/report_generation.py

@@ -1,15 +1,15 @@
 import asyncio
 from typing import List, Dict, Any
-from ...config.config import Config
-from ...utils.llm import create_chat_completion
-from ...utils.logger import get_formatted_logger
+from gpt_researcher.config.config import Config
+from gpt_researcher.utils.llm import create_chat_completion
+from gpt_researcher.utils.logger import get_formatted_logger
 from ..prompts import (
     generate_report_introduction,
     generate_draft_titles_prompt,
     generate_report_conclusion,
     get_prompt_by_report_type,
 )
-from ...utils.enum import Tone
+from gpt_researcher.utils.enum import Tone
 
 logger = get_formatted_logger()
 

gpt_researcher/master/actions/retriever.py → gpt_researcher/actions/retriever.py

@@ -1,5 +1,5 @@
 from typing import List, Type
-from ...config.config import Config
+from gpt_researcher.config.config import Config
 
 def get_retriever(retriever):
     """

gpt_researcher/master/actions/utils.py → gpt_researcher/actions/utils.py

@@ -1,6 +1,6 @@
 from typing import Dict, Any, Callable
 from fastapi import WebSocket
-from ...utils.logger import get_formatted_logger
+from gpt_researcher.utils.logger import get_formatted_logger
 
 logger = get_formatted_logger()
 

gpt_researcher/master/actions/web_scraping.py → gpt_researcher/actions/web_scraping.py

@@ -1,8 +1,8 @@
 import asyncio
 from typing import List, Dict, Any
-from ...scraper.scraper import Scraper
-from ...config.config import Config
-from ...utils.logger import get_formatted_logger
+from gpt_researcher.scraper.scraper import Scraper
+from gpt_researcher.config.config import Config
+from gpt_researcher.utils.logger import get_formatted_logger
 
 logger = get_formatted_logger()
 

gpt_researcher/master/agent/master.py → gpt_researcher/agent.py

@@ -1,17 +1,25 @@
 from typing import Optional, List, Dict, Any, Set
 
-from ...config import Config
-from ...memory import Memory
-from ...utils.enum import ReportSource, ReportType, Tone
-from ...llm_provider import GenericLLMProvider
-from ..actions import get_retrievers, choose_agent
-from ...vector_store import VectorStoreWrapper
-
-# Research agents
-from .researcher import ResearchConductor
-from .scraper import ReportScraper
-from .writer import ReportGenerator
-from .context_manager import ContextManager
+from .config import Config
+from .memory import Memory
+from .utils.enum import ReportSource, ReportType, Tone
+from .llm_provider import GenericLLMProvider
+from .vector_store import VectorStoreWrapper
+
+# Research skills
+from .skills.researcher import ResearchConductor
+from .skills.scraper import ReportScraper
+from .skills.writer import ReportGenerator
+from .skills.context_manager import ContextManager
+
+from .actions import (
+    add_references,
+    extract_headers,
+    extract_sections,
+    table_of_contents,
+    get_retrievers,
+    choose_agent
+)
 
 
 class GPTResearcher:
@@ -118,6 +126,18 @@ async def get_similar_written_contents_by_draft_section_titles(
         )
 
     # Utility methods
+    def add_references(self, report_markdown: str, visited_urls: set) -> str:
+        return add_references(report_markdown, visited_urls)
+
+    def extract_headers(self, markdown_text: str) -> List[Dict]:
+        return extract_headers(markdown_text)
+
+    def extract_sections(self, markdown_text: str) -> List[Dict]:
+        return extract_sections(markdown_text)
+
+    def table_of_contents(self, markdown_text: str) -> str:
+        return table_of_contents(markdown_text)
+
     def get_source_urls(self) -> list:
         return list(self.visited_urls)
 

gpt_researcher/master/prompts.py → gpt_researcher/prompts.py

@@ -1,7 +1,7 @@
 import warnings
 from datetime import date, datetime, timezone
 
-from ..utils.enum import ReportSource, ReportType, Tone
+from gpt_researcher.utils.enum import ReportSource, ReportType, Tone
 
 
 def generate_search_queries_prompt(

gpt_researcher/master/agent/__init__.py → gpt_researcher/skills/__init__.py

@@ -1,11 +1,9 @@
 from .context_manager import ContextManager
-from .master import GPTResearcher
 from .researcher import ResearchConductor
 from .scraper import ReportScraper
 from .writer import ReportGenerator
 
 __all__ = [
-    'GPTResearcher',
     'ResearchConductor',
     'ReportScraper',
     'ReportGenerator',

gpt_researcher/master/agent/context_manager.py → gpt_researcher/skills/context_manager.py

@@ -1,9 +1,9 @@
 import asyncio
 from typing import List, Dict, Optional, Set
 
-from ...context.compression import ContextCompressor, WrittenContentCompressor, VectorstoreCompressor
-from ...document import DocumentLoader, LangChainDocumentLoader
-from ...utils.enum import ReportSource
+from gpt_researcher.context.compression import ContextCompressor, WrittenContentCompressor, VectorstoreCompressor
+from gpt_researcher.document import DocumentLoader, LangChainDocumentLoader
+from gpt_researcher.utils.enum import ReportSource
 from ..actions.utils import stream_output
 
 

gpt_researcher/master/agent/researcher.py → gpt_researcher/skills/researcher.py

@@ -5,8 +5,8 @@
 from ..actions.utils import stream_output
 from ..actions import scrape_urls
 from ..actions.query_processing import get_sub_queries
-from ...document import DocumentLoader, LangChainDocumentLoader
-from ...utils.enum import ReportSource, ReportType, Tone
+from gpt_researcher.document import DocumentLoader, LangChainDocumentLoader
+from gpt_researcher.utils.enum import ReportSource, ReportType, Tone
 
 
 class ResearchConductor:

gpt_researcher/master/agent/writer.py → gpt_researcher/skills/writer.py

@@ -1,6 +1,6 @@
 from typing import Dict, Optional
 
-from ...utils.llm import construct_subtopics
+from gpt_researcher.utils.llm import construct_subtopics
 from ..actions import (
     stream_output,
     generate_report,

gpt_researcher/utils/llm.py

@@ -9,7 +9,7 @@
 from langchain.output_parsers import PydanticOutputParser
 from langchain.prompts import PromptTemplate
 
-from gpt_researcher.master.prompts import generate_subtopics_prompt
+from ..prompts import generate_subtopics_prompt
 from .costs import estimate_llm_cost
 from .validators import Subtopics
 

multi_agents/agents/editor.py

@@ -115,7 +115,7 @@ def _format_planning_instructions(self, initial_research: str, include_human_fee
                    sections: ['section header 1', 'section header 2', 'section header 3' ...]}}'."""
 
     def _initialize_agents(self) -> Dict[str, any]:
-        """Initialize the research, reviewer, and reviser agents."""
+        """Initialize the research, reviewer, and reviser skills."""
         return {
             "research": ResearchAgent(self.websocket, self.stream_output, self.headers),
             "reviewer": ReviewerAgent(self.websocket, self.stream_output, self.headers),