replace scraper logic with github .md files parsing

Author: rudransh-shrivastava

backend/apps/owasp/management/commands/owasp_scrape_chapters.py

@@ -4,10 +4,11 @@
 import time
 
 from django.core.management.base import BaseCommand
+from github.GithubException import UnknownObjectException
 
+from apps.github.auth import get_github_client
 from apps.github.utils import normalize_url
 from apps.owasp.models.chapter import Chapter
-from apps.owasp.scraper import OwaspScraper
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -26,6 +27,8 @@ def add_arguments(self, parser) -> None:
 
     def handle(self, *args, **options) -> None:
         """Handle the command execution."""
+        gh = get_github_client()
+
         active_chapters = Chapter.active_chapters.order_by("-created_at")
         active_chapters_count = active_chapters.count()
         offset = options["offset"]
@@ -34,8 +37,9 @@ def handle(self, *args, **options) -> None:
             prefix = f"{idx + offset + 1} of {active_chapters_count}"
             print(f"{prefix:<10} {chapter.owasp_url}")
 
-            scraper = OwaspScraper(chapter.owasp_url)
-            if scraper.page_tree is None:
+            try:
+                gh.get_repo(f"owasp/{chapter.key}")
+            except UnknownObjectException:
                 chapter.deactivate()
                 continue
 
@@ -62,7 +66,7 @@ def handle(self, *args, **options) -> None:
             invalid_urls = set()
             related_urls = set()
             for scraped_url in scraped_urls:
-                verified_url = scraper.verify_url(scraped_url)
+                verified_url = chapter.verify_url(scraped_url)
                 if not verified_url:
                     invalid_urls.add(scraped_url)
                     continue

backend/apps/owasp/management/commands/owasp_scrape_committees.py

@@ -4,10 +4,11 @@
 import time
 
 from django.core.management.base import BaseCommand
+from github.GithubException import UnknownObjectException
 
+from apps.github.auth import get_github_client
 from apps.github.utils import normalize_url
 from apps.owasp.models.committee import Committee
-from apps.owasp.scraper import OwaspScraper
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -26,6 +27,8 @@ def add_arguments(self, parser) -> None:
 
     def handle(self, *args, **options) -> None:
         """Handle the command execution."""
+        gh = get_github_client()
+
         active_committees = Committee.active_committees.order_by("-created_at")
         active_committees_count = active_committees.count()
         offset = options["offset"]
@@ -34,8 +37,9 @@ def handle(self, *args, **options) -> None:
             prefix = f"{idx + offset + 1} of {active_committees_count}"
             print(f"{prefix:<10} {committee.owasp_url}")
 
-            scraper = OwaspScraper(committee.owasp_url)
-            if scraper.page_tree is None:
+            try:
+                gh.get_repo(f"owasp/{committee.key}")
+            except UnknownObjectException:
                 committee.deactivate()
                 continue
 
@@ -62,7 +66,7 @@ def handle(self, *args, **options) -> None:
             invalid_urls = set()
             related_urls = set()
             for scraped_url in scraped_urls:
-                verified_url = scraper.verify_url(scraped_url)
+                verified_url = committee.verify_url(scraped_url)
                 if not verified_url:
                     invalid_urls.add(scraped_url)
                     continue

backend/apps/owasp/management/commands/owasp_scrape_projects.py

@@ -10,7 +10,6 @@
 from apps.github.constants import GITHUB_USER_RE
 from apps.github.utils import normalize_url
 from apps.owasp.models.project import Project
-from apps.owasp.scraper import OwaspScraper
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -46,8 +45,9 @@ def handle(self, *args, **options) -> None:
             prefix = f"{idx + offset + 1} of {active_projects_count}"
             print(f"{prefix:<10} {project.owasp_url}")
 
-            scraper = OwaspScraper(project.owasp_url)
-            if scraper.page_tree is None:
+            try:
+                gh.get_repo(f"owasp/{project.key}")
+            except UnknownObjectException:
                 project.deactivate()
                 continue
 
@@ -69,7 +69,7 @@ def handle(self, *args, **options) -> None:
             invalid_urls: set[str] = set()
             related_urls: set[str] = set()
             for scraped_url in scraped_urls:
-                verified_url = scraper.verify_url(scraped_url)
+                verified_url = project.verify_url(scraped_url)
                 if not verified_url:
                     invalid_urls.add(scraped_url)
                     continue

backend/apps/owasp/models/common.py

@@ -5,8 +5,10 @@
 import itertools
 import logging
 import re
+from http import HTTPStatus
 from urllib.parse import urlparse
 
+import requests
 import yaml
 from django.contrib.contenttypes.models import ContentType
 from django.db import models
@@ -25,6 +27,8 @@
 
 logger = logging.getLogger(__name__)
 
+TIMEOUT = 5, 10
+
 
 class RepositoryBasedEntityModel(models.Model):
     """Repository based entity model."""
@@ -345,3 +349,35 @@ def sync_leaders(self, leaders_emails):
 
         if leaders:
             BulkSaveModel.bulk_save(EntityMember, leaders)
+
+    def verify_url(self, url):
+        """Verify URL."""
+        location = urlparse(url).netloc.lower()
+        if not location:
+            return None
+
+        if location.endswith(("linkedin.com", "slack.com", "youtube.com")):
+            return url
+
+        try:
+            # Check for redirects.
+            response = requests.get(url, allow_redirects=False, timeout=TIMEOUT)
+        except requests.exceptions.RequestException:
+            logger.exception("Request failed", extra={"url": url})
+            return None
+
+        if response.status_code == HTTPStatus.OK:
+            return url
+
+        if response.status_code in {
+            HTTPStatus.MOVED_PERMANENTLY,  # 301
+            HTTPStatus.FOUND,  # 302
+            HTTPStatus.SEE_OTHER,  # 303
+            HTTPStatus.TEMPORARY_REDIRECT,  # 307
+            HTTPStatus.PERMANENT_REDIRECT,  # 308
+        }:
+            return self.verify_url(response.headers["Location"])
+
+        logger.warning("Couldn't verify URL %s", url)
+
+        return None

backend/apps/owasp/scraper.py

@@ -1,11 +1,11 @@
+import os
 from unittest import mock
 
 import pytest
 
 from apps.owasp.management.commands.owasp_scrape_chapters import (
     Chapter,
     Command,
-    OwaspScraper,
     normalize_url,
 )
 
@@ -34,17 +34,16 @@ def mock_chapter(self):
             (1, 8),
         ],
     )
-    @mock.patch.dict("os.environ", {"SCRAPER_API_KEY": "test-token"})
+    @mock.patch.dict(os.environ, {"GITHUB_TOKEN": "test-token"})
     @mock.patch.object(Chapter, "bulk_save", autospec=True)
-    def test_handle(self, mock_bulk_save, command, mock_chapter, offset, chapters):
-        mock_scraper = mock.Mock(spec=OwaspScraper)
+    @mock.patch("apps.owasp.management.commands.owasp_scrape_chapters.get_github_client")
+    def test_handle(self, mock_github, mock_bulk_save, command, mock_chapter, offset, chapters):
         mock_chapter.get_urls.return_value = [
             "https://example.com/repo1",
             "https://example.com/repo2",
             "https://invalid.com/repo3",
         ]
-        mock_scraper.verify_url.side_effect = lambda url: None if "invalid" in url else url
-        mock_scraper.page_tree = True
+        mock_chapter.verify_url.side_effect = lambda url: None if "invalid" in url else url
 
         mock_chapter.get_related_url.side_effect = lambda url, **_: url
 
@@ -60,14 +59,14 @@ def test_handle(self, mock_bulk_save, command, mock_chapter, offset, chapters):
         )
         mock_active_chapters.order_by.return_value = mock_active_chapters
 
+        mock_github_instance = mock.Mock()
+        mock_github.return_value = mock_github_instance
+        mock_github_instance.get_repo.return_value = mock.Mock()
+
         with (
             mock.patch.object(Chapter, "active_chapters", mock_active_chapters),
             mock.patch("builtins.print") as mock_print,
             mock.patch("time.sleep", return_value=None),
-            mock.patch(
-                "apps.owasp.management.commands.owasp_scrape_chapters.OwaspScraper",
-                return_value=mock_scraper,
-            ),
             mock.patch(
                 "apps.owasp.management.commands.owasp_scrape_chapters.normalize_url",
                 side_effect=normalize_url,