diff --git a/paper_search_mcp/academic_platforms/__init__.py b/paper_search_mcp/academic_platforms/__init__.py index e69de29..bdd2522 100644 --- a/paper_search_mcp/academic_platforms/__init__.py +++ b/paper_search_mcp/academic_platforms/__init__.py @@ -0,0 +1,30 @@ +# paper_search_mcp/academic_platforms/__init__.py + +""" +This package provides modules for searching various academic platforms. +Each module should contain a searcher class that implements a common interface +(e.g., inheriting from a base PaperSource class and having a 'search' method). +""" + +from .arxiv import ArxivSearcher +from .biorxiv import BiorxivSearcher +from .google_scholar import GoogleScholarSearcher +# hub.py is not a searcher, so it's not imported here for direct use as a platform +from .iacr import IACRSearcher +from .medrxiv import MedrxivSearcher +from .pubmed import PubMedSearcher +from .scopus import ScopusSearcher +from .semantic import SemanticSearcher +from .shodhganga import ShodhgangaSearcher + +__all__ = [ + "ArxivSearcher", + "BiorxivSearcher", + "GoogleScholarSearcher", + "IACRSearcher", + "MedrxivSearcher", + "PubMedSearcher", + "ScopusSearcher", + "SemanticSearcher", + "ShodhgangaSearcher", +] diff --git a/paper_search_mcp/academic_platforms/hub.py b/paper_search_mcp/academic_platforms/hub.py index e69de29..a341223 100644 --- a/paper_search_mcp/academic_platforms/hub.py +++ b/paper_search_mcp/academic_platforms/hub.py @@ -0,0 +1,74 @@ +# paper_search_mcp/academic_platforms/hub.py + +""" +Central hub for accessing different academic platform searchers. +This allows for dynamic instantiation of searchers based on a key. +""" + +from .arxiv import ArxivSearcher +from .biorxiv import BiorxivSearcher +from .google_scholar import GoogleScholarSearcher +from .iacr import IACRSearcher +from .medrxiv import MedrxivSearcher +from .pubmed import PubMedSearcher +from .scopus import ScopusSearcher +from .semantic import SemanticSearcher +from .shodhganga import ShodhgangaSearcher + +# A dictionary mapping platform names (keys) to their searcher classes. +# This allows for easy lookup and instantiation of searchers. +AVAILABLE_SEARCHERS = { + "arxiv": ArxivSearcher, + "biorxiv": BiorxivSearcher, + "google_scholar": GoogleScholarSearcher, + "iacr": IACRSearcher, + "medrxiv": MedrxivSearcher, + "pubmed": PubMedSearcher, + "scopus": ScopusSearcher, + "semantic_scholar": SemanticSearcher, # Assuming 'semantic_scholar' as key for SemanticSearcher + "shodhganga": ShodhgangaSearcher, +} + +def get_searcher(platform_name: str): + """ + Returns an instance of the searcher for the given platform name. + + Args: + platform_name (str): The key for the desired platform + (e.g., "arxiv", "pubmed", "shodhganga"). + + Returns: + An instance of the searcher class if found, otherwise None. + + Raises: + ValueError: If the platform_name is not recognized. + """ + platform_name = platform_name.lower() + searcher_class = AVAILABLE_SEARCHERS.get(platform_name) + if searcher_class: + return searcher_class() # Instantiate the class + else: + raise ValueError(f"Unknown platform: {platform_name}. Available platforms are: {list(AVAILABLE_SEARCHERS.keys())}") + +if __name__ == '__main__': + # Example usage: + print(f"Available searcher platforms: {list(AVAILABLE_SEARCHERS.keys())}") + + try: + arxiv_searcher = get_searcher("arxiv") + print(f"Successfully got searcher for 'arxiv': {type(arxiv_searcher)}") + + shodhganga_searcher = get_searcher("shodhganga") + print(f"Successfully got searcher for 'shodhganga': {type(shodhganga_searcher)}") + + # Test a non-existent platform + # get_searcher("nonexistent_platform") + + except ValueError as e: + print(f"Error: {e}") + except ImportError as e: + print(f"ImportError: {e}. This might indicate an issue with the class names in __init__.py or the files themselves.") + print("Please ensure all Searcher classes (e.g., ArxivSearcher, PubMedSearcher) are correctly defined and imported.") + +# TODO: Consider adding a more robust plugin system if the number of platforms grows significantly. +# TODO: Potentially load API keys or configurations here if needed by searchers in the future. diff --git a/paper_search_mcp/academic_platforms/shodhganga.py b/paper_search_mcp/academic_platforms/shodhganga.py new file mode 100644 index 0000000..94d2e90 --- /dev/null +++ b/paper_search_mcp/academic_platforms/shodhganga.py @@ -0,0 +1,273 @@ +from typing import List, Optional +from datetime import datetime +import requests +from bs4 import BeautifulSoup +import time +import random +import re # Added for regex in date parsing +from ..paper import Paper # Assuming Paper class is in parent directory +import logging + +logger = logging.getLogger(__name__) + +# Define a base class for paper sources, similar to what's seen in other modules. +# If a central PaperSource exists, this searcher should inherit from it. +class PaperSource: + """Abstract base class for paper sources""" + def search(self, query: str, **kwargs) -> List[Paper]: + raise NotImplementedError + + def download_pdf(self, paper_id: str, save_path: str) -> str: + raise NotImplementedError + + def read_paper(self, paper_id: str, save_path: str) -> str: + raise NotImplementedError + +class ShodhgangaSearcher(PaperSource): + """ + Searcher for theses and dissertations on Shodhganga (https://shodhganga.inflibnet.ac.in/). + + NOTE: This implementation is based on assumed HTML structures and search parameter patterns + due to limitations in directly accessing the website during development. + It will require validation and potential adjustments with actual website responses. + """ + + BASE_URL = "https://shodhganga.inflibnet.ac.in" + SEARCH_PATH = "/simple-search" # Assuming this is the correct path for simple search + + # Common browser user agents to rotate + BROWSERS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:98.0) Gecko/20100101 Firefox/98.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0" + ] + + def __init__(self): + """Initialize the session with a random user agent.""" + self.session = requests.Session() + self.session.headers.update({ + 'User-Agent': random.choice(self.BROWSERS), + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5' + }) + + def _parse_single_item(self, item_soup: BeautifulSoup, base_url: str) -> Optional[Paper]: + """ + Parses a single search result item from Shodhganga. + ASSUMPTION: This method is based on a hypothetical HTML structure. + Actual class names and tags will need to be verified. + """ + try: + # --- ASSUMED HTML STRUCTURE --- + title_tag = item_soup.select_one('h4.discovery-result-title a') # Example:

Title

+ if not title_tag: + logger.warning("Could not find title tag in item.") + return None + + title = title_tag.get_text(strip=True) + item_url = title_tag.get('href') + if not item_url: + logger.warning(f"Could not find URL for title: {title}") + return None + + # Ensure URL is absolute + if item_url.startswith('/'): + item_url = base_url + item_url + + # Authors - Example:
Author One, Author Two
+ # Or authors might be in or

+ # We will look for a div with "author" in its class name or specific metadata fields + author_tags = item_soup.select('div.authors span[title="author"], meta[name="DC.creator"]') + authors = [] + if author_tags: + authors = [tag.get_text(strip=True) if tag.name == 'span' else tag.get('content', '') for tag in author_tags] + authors = [a for a in authors if a] # Filter out empty strings + + if not authors: # Fallback if specific tags not found + author_div = item_soup.select_one('div[class*="author"]') # Generic author div + if author_div: + authors = [a.strip() for a in author_div.get_text(strip=True).split(';')] + + + # Abstract/Description - Example:

This is the abstract...
+ # Or
+ abstract_tag = item_soup.select_one('div.abstract-full, div.item-abstract') + abstract = abstract_tag.get_text(strip=True) if abstract_tag else "No abstract available." + + # Publication Date (Year) - Example:
2023
or + # Or + date_tag = item_soup.select_one('div.dateinfo, span.date, meta[name="DC.date.issued"]') + year = None + if date_tag: + date_text = date_tag.get_text(strip=True) if date_tag.name != 'meta' else date_tag.get('content', '') + # Try to extract a 4-digit year + match = re.search(r'\b(\d{4})\b', date_text) + if match: + year = int(match.group(1)) + + published_date = datetime(year, 1, 1) if year else None + + # Paper ID - can be derived from the URL or a specific metadata field + paper_id = f"shodhganga_{item_url.split('/')[-1]}" if item_url else f"shodhganga_{hash(title)}" + + return Paper( + paper_id=paper_id, + title=title, + authors=authors if authors else ["Unknown Author"], + abstract=abstract, + url=item_url, + pdf_url="", # Shodhganga links to landing pages, not direct PDFs + published_date=published_date, + updated_date=None, # Shodhganga may not provide this + source='shodhganga', + categories=[], # May need to parse if available + keywords=[], # May need to parse if available + doi="" # Shodhganga items are theses, may not always have DOIs + ) + except Exception as e: + logger.error(f"Error parsing Shodhganga item: {e}", exc_info=True) + return None + + def search(self, query: str, max_results: int = 10) -> List[Paper]: + """ + Search Shodhganga for theses and dissertations. + + ASSUMPTION: This method relies on assumed URL parameters and HTML structure + for Shodhganga's search results. These need to be verified. + """ + papers: List[Paper] = [] + # ASSUMPTION: Search parameters. Common ones are 'query' or 'rpp' (results per page). + # Shodhganga's simple search form uses 'query', 'filter_field_1', 'filter_type_1', 'filter_value_1' + # For a simple keyword search, 'query' might be enough, or it might be 'filter_value_1' with 'filter_field_1=all' + + # Let's try a structure based on typical DSpace simple search + # Example: /simple-search?query=myquery&sort_by=score&order=desc&rpp=10&etal=0&start=0 + search_url = self.BASE_URL + self.SEARCH_PATH + + # Pagination: DSpace typically uses 'start' for the offset. + # 'rpp' for results per page. + results_to_fetch_this_page = min(max_results, 20) # Shodhganga might cap results per page (e.g. at 20) + current_start_index = 0 + + while len(papers) < max_results: + params = { + 'query': query, + 'rpp': results_to_fetch_this_page, + 'sort_by': 'score', # Or 'dc.date.issued' for newest + 'order': 'desc', + 'start': current_start_index + } + + logger.info(f"Searching Shodhganga: {search_url} with params: {params}") + + try: + # Add a small delay to be polite to the server + time.sleep(random.uniform(1.0, 3.0)) + response = self.session.get(search_url, params=params) + response.raise_for_status() # Raise an exception for HTTP errors + + soup = BeautifulSoup(response.content, 'html.parser') + + # --- ASSUMED HTML STRUCTURE for results list --- + # Example:
...
...
+ # Or
  • ...
+ # Looking for elements that seem to contain individual search results. + # Common DSpace class for a list of items: 'ds-artifact-list' or 'discovery-result-results' + # Common DSpace class for one item: 'ds-artifact-item' or 'artifact-description' + result_items = soup.select('div.ds-artifact-item, div.artifact-description, li.ds-artifact-item') + + if not result_items: + logger.info("No more results found on Shodhganga or page structure not recognized.") + break + + found_on_page = 0 + for item_soup in result_items: + if len(papers) >= max_results: + break + paper = self._parse_single_item(item_soup, self.BASE_URL) + if paper: + papers.append(paper) + found_on_page +=1 + + logger.info(f"Found {found_on_page} items on this page. Total papers collected: {len(papers)}.") + + if found_on_page == 0: # No items parsed on this page, stop. + logger.info("No parsable items found on this page, stopping pagination.") + break + + # Pagination: Look for a 'next' link + # ASSUMPTION: Next page link has class 'next-page' or text 'Next'. + # DSpace pagination usually updates the 'start' parameter. + # If we successfully got `results_to_fetch_this_page` items, we assume there might be more. + # A more robust way is to check for an explicit "next" link. + # For DSpace, if current_start_index + results_to_fetch_this_page < total_hits, there's a next page. + # Total hits might be displayed as: 1-10 of 123 + + # Simple pagination: increment start index + current_start_index += results_to_fetch_this_page + + # Check if there's a clear 'next' button to decide if we should continue + next_page_tag = soup.select_one('a.next-page, a:contains("Next"), a[title="next"]') + if not next_page_tag and found_on_page < results_to_fetch_this_page : + logger.info("No 'next page' link found or fewer results than requested, assuming end of results.") + break + + + except requests.exceptions.RequestException as e: + logger.error(f"HTTP request to Shodhganga failed: {e}") + break # Stop searching if there's a request error + except Exception as e: + logger.error(f"An error occurred during Shodhganga search: {e}", exc_info=True) + break # Stop on other errors + + return papers[:max_results] + + def download_pdf(self, paper_id: str, save_path: str) -> str: + """ + Shodhganga typically links to thesis pages which may contain PDFs. + Direct PDF download via a simple ID is not assumed to be supported. + """ + raise NotImplementedError( + "Shodhganga does not provide direct PDF downloads via this interface. " + "Please use the paper URL from the search results to navigate to the thesis page and find download options." + ) + + def read_paper(self, paper_id: str, save_path: str = "./downloads") -> str: + """ + Reading papers directly from Shodhganga is not supported. + Metadata and links are provided; full text access is via the website. + """ + return ( + "Shodhganga papers cannot be read directly through this tool. " + "Please use the paper's URL to access the full text on the Shodhganga website." + ) + +if __name__ == '__main__': + # This section can be used for basic testing once the search method is implemented. + # For now, it will just demonstrate class instantiation. + searcher = ShodhgangaSearcher() + print("ShodhgangaSearcher initialized.") + + # Example of how search might be called (will currently return empty list and warning): + # try: + # papers = searcher.search("artificial intelligence", max_results=5) + # if not papers: + # print("Search returned no results (as expected for now).") + # for paper in papers: + # print(paper.title) + # except Exception as e: + # print(f"Error during search: {e}") + + # Test not implemented methods + try: + searcher.download_pdf("some_id", "./") + except NotImplementedError as e: + print(f"Caught expected error for download_pdf: {e}") + + try: + message = searcher.read_paper("some_id") + print(f"Response from read_paper: {message}") + except Exception as e: # Should not happen if it returns a message + print(f"Error during read_paper: {e}") diff --git a/tests/test_shodhganga.py b/tests/test_shodhganga.py new file mode 100644 index 0000000..0ba4d7c --- /dev/null +++ b/tests/test_shodhganga.py @@ -0,0 +1,235 @@ +import unittest +from unittest.mock import patch, MagicMock +from datetime import datetime + +# Ensure the local path is prioritized for imports, especially for 'paper' +import sys +import os +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from paper_search_mcp.paper import Paper +from paper_search_mcp.academic_platforms.shodhganga import ShodhgangaSearcher + +# --- Sample HTML Snippets based on DSpace structure assumptions --- + +# This is a HYPOTHETICAL HTML structure. It needs to be validated against the actual Shodhganga website. +SAMPLE_SHODHGANGA_RESULT_ITEM_HTML = """ +
+
+
+

+ A Study on Fictional Technology +

+
+ Researcher, Anand R.; + Guide, Priya S. +
+
University of Example, Department of Studies
+
Issued Date: 2023-01-15
+
+ This thesis explores the impact of fictional technology on modern society. + It covers various aspects and provides detailed analysis. +
+
+ /jspui/handle/10603/12345 +
+
+
+
+""" + +SAMPLE_SHODHGANGA_SEARCH_PAGE_HTML_TEMPLATE = """ + +Search Results + +
+ {items_html} +
+ {pagination_html} + + +""" + +SAMPLE_SHODHGANGA_PAGINATION_NEXT = 'Next' +NO_RESULTS_HTML = """ +
+""" + + +class TestShodhgangaSearcher(unittest.TestCase): + + def setUp(self): + self.searcher = ShodhgangaSearcher() + # Keep a reference to the original session for tests that might need it (though not typical for unit tests) + self.original_session = self.searcher.session + + def tearDown(self): + # Restore original session if it was modified + self.searcher.session = self.original_session + + @patch('requests.Session.get') + def test_search_url_construction_and_basic_call(self, mock_get): + """Test if the search method constructs the URL and params correctly.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.content = NO_RESULTS_HTML # No results needed for this test + mock_get.return_value = mock_response + + self.searcher.search("test query", max_results=5) + + expected_url = "https://shodhganga.inflibnet.ac.in/simple-search" + expected_params = { + 'query': "test query", + 'rpp': 5, + 'sort_by': 'score', + 'order': 'desc', + 'start': 0 + } + mock_get.assert_called_once() + args, kwargs = mock_get.call_args + self.assertEqual(args[0], expected_url) + self.assertDictEqual(kwargs['params'], expected_params) + + def test_parse_single_item_success(self): + """Test parsing a single valid HTML item.""" + from bs4 import BeautifulSoup + item_soup = BeautifulSoup(SAMPLE_SHODHGANGA_RESULT_ITEM_HTML, 'html.parser') + + paper = self.searcher._parse_single_item(item_soup.select_one('div.ds-artifact-item'), self.searcher.BASE_URL) + + self.assertIsNotNone(paper) + self.assertEqual(paper.title, "A Study on Fictional Technology") + self.assertListEqual(paper.authors, ["Researcher, Anand R.", "Guide, Priya S."]) + self.assertEqual(paper.url, "https://shodhganga.inflibnet.ac.in/jspui/handle/10603/12345") + self.assertEqual(paper.abstract[:50], "This thesis explores the impact of fictional tech") + self.assertEqual(paper.published_date, datetime(2023, 1, 1)) + self.assertEqual(paper.source, "shodhganga") + self.assertTrue(paper.paper_id.startswith("shodhganga_")) + + def test_parse_single_item_missing_fields(self): + """Test parsing an item with some missing fields.""" + from bs4 import BeautifulSoup + html_missing_author_abstract = """ +
+

Minimal Item

+
2021
+
+ """ + item_soup = BeautifulSoup(html_missing_author_abstract, 'html.parser') + paper = self.searcher._parse_single_item(item_soup.select_one('div.ds-artifact-item'), self.searcher.BASE_URL) + + self.assertIsNotNone(paper) + self.assertEqual(paper.title, "Minimal Item") + self.assertListEqual(paper.authors, ["Unknown Author"]) # Default value + self.assertEqual(paper.abstract, "No abstract available.") # Default value + self.assertEqual(paper.published_date, datetime(2021, 1, 1)) + self.assertEqual(paper.url, "https://shodhganga.inflibnet.ac.in/handle/123/broken") + + + @patch('requests.Session.get') + def test_search_parses_results(self, mock_get): + """Test that the search method uses _parse_single_item and returns Paper objects.""" + mock_response = MagicMock() + mock_response.status_code = 200 + # Simulate a page with one item and no next page link + mock_response.content = SAMPLE_SHODHGANGA_SEARCH_PAGE_HTML_TEMPLATE.format( + items_html=SAMPLE_SHODHGANGA_RESULT_ITEM_HTML, + pagination_html="" + ) + mock_get.return_value = mock_response + + papers = self.searcher.search("fictional technology", max_results=1) + + self.assertEqual(len(papers), 1) + self.assertIsInstance(papers[0], Paper) + self.assertEqual(papers[0].title, "A Study on Fictional Technology") + + @patch('requests.Session.get') + def test_search_handles_no_results(self, mock_get): + """Test search with a response that contains no results.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.content = NO_RESULTS_HTML + mock_get.return_value = mock_response + + papers = self.searcher.search("nonexistent query", max_results=10) + self.assertEqual(len(papers), 0) + + @patch('requests.Session.get') + def test_search_pagination_logic(self, mock_get): + """Test that search attempts to fetch multiple pages if max_results implies it.""" + # First response: one item + next page link + response1_html = SAMPLE_SHODHGANGA_SEARCH_PAGE_HTML_TEMPLATE.format( + items_html=SAMPLE_SHODHGANGA_RESULT_ITEM_HTML.replace("12345", "page1item"), + pagination_html=SAMPLE_SHODHGANGA_PAGINATION_NEXT + ) + mock_response1 = MagicMock(status_code=200, content=response1_html.encode('utf-8')) + + # Second response: another item, no next page link + item2_html = SAMPLE_SHODHGANGA_RESULT_ITEM_HTML.replace( + "A Study on Fictional Technology", "Another Study" + ).replace("12345", "page2item") + response2_html = SAMPLE_SHODHGANGA_SEARCH_PAGE_HTML_TEMPLATE.format( + items_html=item2_html, + pagination_html="" + ) + mock_response2 = MagicMock(status_code=200, content=response2_html.encode('utf-8')) + + # Third response (should not be strictly needed if max_results is met, but good for safety) + mock_response_empty = MagicMock(status_code=200, content=NO_RESULTS_HTML.encode('utf-8')) + + mock_get.side_effect = [mock_response1, mock_response2, mock_response_empty] + + papers = self.searcher.search("test query", max_results=2) # Request 2 results + + self.assertEqual(len(papers), 2) + self.assertEqual(mock_get.call_count, 2) # Should make two calls due to pagination + + # Check params for the second call (start index should have incremented) + args1, kwargs1 = mock_get.call_args_list[0] + args2, kwargs2 = mock_get.call_args_list[1] + + self.assertEqual(kwargs1['params']['start'], 0) + # rpp is min(max_results_remaining, configured_rpp_cap_in_shodhganga.py) + # max_results=2, so initial rpp = min(2, 20) = 2 + self.assertEqual(kwargs1['params']['rpp'], 2) + + self.assertEqual(kwargs2['params']['start'], 2) # start = 0 + 2 (rpp from first call) + self.assertEqual(kwargs2['params']['rpp'], 2) # rpp = min(2-1, 20) = 1, but code uses initial rpp throughout loop. + # This might be an area for refinement in shodhganga.py if needed, + # but current test reflects current code. + # The loop condition `len(papers) < max_results` will stop it. + + def test_download_pdf_not_implemented(self): + """Test that download_pdf raises NotImplementedError.""" + with self.assertRaisesRegex(NotImplementedError, "Shodhganga does not provide direct PDF downloads"): + self.searcher.download_pdf("some_id", "./downloads") + + def test_read_paper_returns_message(self): + """Test that read_paper returns the correct informational message.""" + message = self.searcher.read_paper("some_id") + self.assertIn("Shodhganga papers cannot be read directly", message) + + @patch('requests.Session.get') + def test_search_http_error(self, mock_get): + """Test search handling of HTTP errors.""" + mock_response = MagicMock() + mock_response.status_code = 500 + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("Server Error") + mock_get.return_value = mock_response + + papers = self.searcher.search("test query", max_results=10) + self.assertEqual(len(papers), 0) # Should return empty list on error + + def test_parse_single_item_no_title_tag(self): + """Test _parse_single_item when the main title tag is missing.""" + from bs4 import BeautifulSoup + html_no_title = """
No title here
""" + item_soup = BeautifulSoup(html_no_title, 'html.parser') + paper = self.searcher._parse_single_item(item_soup.select_one('div.ds-artifact-item'), self.searcher.BASE_URL) + self.assertIsNone(paper) + +if __name__ == '__main__': + print("NOTE: These tests for ShodhgangaSearcher rely on ASSUMED HTML structures.") + print("Actual functionality needs validation against the live Shodhganga website.\n") + unittest.main()