Migrate Scraper to GitHub .md Files

backend/apps/owasp/management/commands/owasp_scrape_chapters.py

@@ -4,10 +4,11 @@
 import time
 
 from django.core.management.base import BaseCommand
+from github.GithubException import GithubException, UnknownObjectException
 
+from apps.github.auth import get_github_client
 from apps.github.utils import normalize_url
 from apps.owasp.models.chapter import Chapter
-from apps.owasp.scraper import OwaspScraper
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -26,6 +27,8 @@ def add_arguments(self, parser) -> None:
 
     def handle(self, *args, **options) -> None:
         """Handle the command execution."""
+        gh = get_github_client()
+
         active_chapters = Chapter.active_chapters.order_by("-created_at")
         active_chapters_count = active_chapters.count()
         offset = options["offset"]
@@ -34,10 +37,15 @@ def handle(self, *args, **options) -> None:
             prefix = f"{idx + offset + 1} of {active_chapters_count}"
             print(f"{prefix:<10} {chapter.owasp_url}")
 
-            scraper = OwaspScraper(chapter.owasp_url)
-            if scraper.page_tree is None:
+            try:
+                gh.get_repo(f"owasp/{chapter.key}")
+            except UnknownObjectException:
                 chapter.deactivate()
                 continue
+            except GithubException as e:
+                logger.warning("GitHub API error for %s: %s", chapter.key, e)
+                time.sleep(1)
+                continue
 
             chapter.leaders_raw = chapter.get_leaders()
             if leaders_emails := chapter.get_leaders_emails():
@@ -62,7 +70,7 @@ def handle(self, *args, **options) -> None:
             invalid_urls = set()
             related_urls = set()
             for scraped_url in scraped_urls:
-                verified_url = scraper.verify_url(scraped_url)
+                verified_url = chapter.verify_url(scraped_url)
                 if not verified_url:
                     invalid_urls.add(scraped_url)
                     continue

backend/apps/owasp/management/commands/owasp_scrape_committees.py

@@ -4,10 +4,11 @@
 import time
 
 from django.core.management.base import BaseCommand
+from github.GithubException import GithubException, UnknownObjectException
 
+from apps.github.auth import get_github_client
 from apps.github.utils import normalize_url
 from apps.owasp.models.committee import Committee
-from apps.owasp.scraper import OwaspScraper
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -26,6 +27,8 @@ def add_arguments(self, parser) -> None:
 
     def handle(self, *args, **options) -> None:
         """Handle the command execution."""
+        gh = get_github_client()
+
         active_committees = Committee.active_committees.order_by("-created_at")
         active_committees_count = active_committees.count()
         offset = options["offset"]
@@ -34,10 +37,15 @@ def handle(self, *args, **options) -> None:
             prefix = f"{idx + offset + 1} of {active_committees_count}"
             print(f"{prefix:<10} {committee.owasp_url}")
 
-            scraper = OwaspScraper(committee.owasp_url)
-            if scraper.page_tree is None:
+            try:
+                gh.get_repo(f"owasp/{committee.key}")
+            except UnknownObjectException:
                 committee.deactivate()
                 continue
+            except GithubException as e:
+                logger.warning("GitHub API error for %s: %s", committee.key, e)
+                time.sleep(1)
+                continue
 
             committee.leaders_raw = committee.get_leaders()
             if leaders_emails := committee.get_leaders_emails():
@@ -62,7 +70,7 @@ def handle(self, *args, **options) -> None:
             invalid_urls = set()
             related_urls = set()
             for scraped_url in scraped_urls:
-                verified_url = scraper.verify_url(scraped_url)
+                verified_url = committee.verify_url(scraped_url)
                 if not verified_url:
                     invalid_urls.add(scraped_url)
                     continue

backend/apps/owasp/management/commands/owasp_scrape_projects.py

@@ -4,13 +4,12 @@
 import time
 
 from django.core.management.base import BaseCommand
-from github.GithubException import UnknownObjectException
+from github.GithubException import GithubException, UnknownObjectException
 
 from apps.github.auth import get_github_client
 from apps.github.constants import GITHUB_USER_RE
 from apps.github.utils import normalize_url
 from apps.owasp.models.project import Project
-from apps.owasp.scraper import OwaspScraper
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -46,10 +45,15 @@ def handle(self, *args, **options) -> None:
             prefix = f"{idx + offset + 1} of {active_projects_count}"
             print(f"{prefix:<10} {project.owasp_url}")
 
-            scraper = OwaspScraper(project.owasp_url)
-            if scraper.page_tree is None:
+            try:
+                gh.get_repo(f"owasp/{project.key}")
+            except UnknownObjectException:
                 project.deactivate()
                 continue
+            except GithubException as e:
+                logger.warning("GitHub API error for %s: %s", project.key, e)
+                time.sleep(1)
+                continue
 
             project.audience = project.get_audience()
             project.leaders_raw = project.get_leaders()
@@ -69,7 +73,7 @@ def handle(self, *args, **options) -> None:
             invalid_urls: set[str] = set()
             related_urls: set[str] = set()
             for scraped_url in scraped_urls:
-                verified_url = scraper.verify_url(scraped_url)
+                verified_url = project.verify_url(scraped_url)
                 if not verified_url:
                     invalid_urls.add(scraped_url)
                     continue

backend/apps/owasp/models/common.py

@@ -5,8 +5,10 @@
 import itertools
 import logging
 import re
+from http import HTTPStatus
 from urllib.parse import urlparse
 
+import requests
 import yaml
 from django.contrib.contenttypes.models import ContentType
 from django.db import models
@@ -25,6 +27,8 @@
 
 logger = logging.getLogger(__name__)
 
+TIMEOUT = 5, 10
+
 
 class RepositoryBasedEntityModel(models.Model):
     """Repository based entity model."""
@@ -345,3 +349,35 @@ def sync_leaders(self, leaders_emails):
 
         if leaders:
             BulkSaveModel.bulk_save(EntityMember, leaders)
+
+    def verify_url(self, url):
+        """Verify URL."""
+        location = urlparse(url).netloc.lower()
+        if not location:
+            return None
+
+        if location.endswith(("linkedin.com", "slack.com", "youtube.com")):
+            return url
+
+        try:
+            # Check for redirects.
+            response = requests.get(url, allow_redirects=False, timeout=TIMEOUT)
+        except requests.exceptions.RequestException:
+            logger.exception("Request failed", extra={"url": url})
+            return None
+
+        if response.status_code == HTTPStatus.OK:
+            return url
+
+        if response.status_code in {
+            HTTPStatus.MOVED_PERMANENTLY,  # 301
+            HTTPStatus.FOUND,  # 302
+            HTTPStatus.SEE_OTHER,  # 303
+            HTTPStatus.TEMPORARY_REDIRECT,  # 307
+            HTTPStatus.PERMANENT_REDIRECT,  # 308
+        }:
+            return self.verify_url(response.headers["Location"])
+
+        logger.warning("Couldn't verify URL %s", url)
+
+        return None