Refactor issue parser - lifecycle methods, use pydantic models, support reviewer lists

pyproject.toml

@@ -40,7 +40,8 @@ dev = [
     "flake8",
     "pre-commit",
     "pytest",
-    "pytest-cov"
+    "pytest-cov",
+    "pytest-mock"
 ]
 
 [project.urls]

src/pyosmeta/cli/process_reviews.py

@@ -18,8 +18,7 @@
 
 import pickle
 
-from pydantic import ValidationError
-from pyosmeta import ProcessIssues, ReviewModel
+from pyosmeta import ProcessIssues
 from pyosmeta.github_api import GitHubAPI
 
 
@@ -35,27 +34,21 @@ def main():
     # Get all issues for approved packages - load as dict
     # TODO: this doesn't have to be in process issues at all. it could fully
     # Call the github module
-    issues = process_review.return_response()
-    accepted_reviews = process_review.parse_issue_header(issues, 45)
+    issues = process_review.get_issues()
+    accepted_reviews, errors = process_review.parse_issues(issues)
+    for url, error in errors.items():
+        print(f"Error in review at url: {url}")
+        print(error)
+        print("-" * 20)
 
     # Update gh metrics via api for all packages
     repo_endpoints = process_review.get_repo_endpoints(accepted_reviews)
     all_reviews = process_review.get_gh_metrics(
         repo_endpoints, accepted_reviews
     )
 
-    # Populate model objects with review data + metrics
-    final_reviews = {}
-    for key, review in all_reviews.items():
-        # First add gh meta to each dict
-        print("Parsing & validating", key)
-        try:
-            final_reviews[key] = ReviewModel(**review)
-        except ValidationError as ve:
-            print(key, ":", ve)
-
     with open("all_reviews.pickle", "wb") as f:
-        pickle.dump(final_reviews, f)
+        pickle.dump(all_reviews, f)
 
 
 if __name__ == "__main__":

src/pyosmeta/cli/update_review_teams.py

@@ -28,7 +28,7 @@
 from pyosmeta.contributors import ProcessContributors
 from pyosmeta.file_io import clean_export_yml, load_pickle
 from pyosmeta.github_api import GitHubAPI
-from pyosmeta.models import PersonModel
+from pyosmeta.models import PersonModel, ReviewUser
 from pyosmeta.utils_clean import get_clean_user
 
 
@@ -46,12 +46,13 @@ def main():
     for pkg_name, issue_meta in packages.items():
         print("Processing review team for:", pkg_name)
         for issue_role in contrib_types.keys():
-            if issue_role == "all_current_maintainers":
+            if issue_role in ("all_current_maintainers", "reviewers"):
                 # Loop through each maintainer in the list
                 for i, a_maintainer in enumerate(
                     issue_meta.all_current_maintainers
                 ):
-                    gh_user = get_clean_user(a_maintainer["github_username"])
+                    a_maintainer: ReviewUser
+                    gh_user = get_clean_user(a_maintainer.github_username)
 
                     if gh_user not in contribs.keys():
                         print("Found a new contributor!", gh_user)
@@ -79,16 +80,16 @@ def main():
                     )
 
                     # If name is missing in issue, populate from contribs
-                    if a_maintainer["name"] == "":
+                    if a_maintainer.name == "":
                         name = getattr(contribs[gh_user], "name")
-                        packages[pkg_name].all_current_maintainers[i][
-                            "name"
-                        ] = name
+                        packages[pkg_name].all_current_maintainers[
+                            i
+                        ].name = name
 
             else:
                 # Else we are processing editors, reviewers...
                 gh_user = get_clean_user(
-                    getattr(packages[pkg_name], issue_role)["github_username"]
+                    getattr(packages[pkg_name], issue_role).github_username
                 )
 
                 if gh_user not in contribs.keys():
@@ -116,11 +117,9 @@ def main():
                 )
 
                 # If users's name is missing in issue, populate from contribs
-                if getattr(issue_meta, issue_role)["name"] == "":
+                if getattr(issue_meta, issue_role).name == "":
                     attribute_value = getattr(packages[pkg_name], issue_role)
-                    attribute_value["name"] = getattr(
-                        contribs[gh_user], "name"
-                    )
+                    attribute_value.name = getattr(contribs[gh_user], "name")
 
     # Export to yaml
     contribs_ls = [model.model_dump() for model in contribs.values()]

src/pyosmeta/contributors.py

@@ -39,8 +39,7 @@ def __init__(self, github_api: GitHubAPI, json_files: List) -> None:
         ]
 
         self.contrib_types = {
-            "reviewer_1": ["packages_reviewed", ["reviewer", "peer-review"]],
-            "reviewer_2": ["packages_reviewed", ["reviewer", "peer-review"]],
+            "reviewers": ["packages_reviewed", ["reviewer", "peer-review"]],
             "editor": ["packages_editor", ["editor", "peer-review"]],
             "submitting_author": [
                 "packages_submitted",

src/pyosmeta/models/__init__.py

@@ -0,0 +1,15 @@
+from pyosmeta.models.base import (
+    GhMeta,
+    PersonModel,
+    ReviewModel,
+    ReviewUser,
+    UrlValidatorMixin,
+)
+
+__all__ = [
+    "UrlValidatorMixin",
+    "PersonModel",
+    "GhMeta",
+    "ReviewModel",
+    "ReviewUser",
+]

src/pyosmeta/models.py → src/pyosmeta/models/base.py

@@ -4,7 +4,8 @@
 """
 
 import re
-from typing import Optional, Set, Union
+from datetime import datetime
+from typing import Any, Optional, Set, Union
 
 import requests
 from pydantic import (
@@ -202,6 +203,21 @@ def clean_date(cls, a_date: Optional[str]) -> str:
         return clean_date(a_date)
 
 
+class ReviewUser(BaseModel):
+    """Minimal model of a github user, used in several places in review parsing"""
+
+    name: str
+    github_username: str
+
+    @field_validator("github_username", mode="after")
+    def deurl_github_username(cls, github_username: str) -> str:
+        return github_username.replace("https://github.com/", "")
+
+    @field_validator("name", mode="after")
+    def demarkdown_name(cls, name: str) -> str:
+        return re.sub(r"\[|\]", "", name)
+
+
 class ReviewModel(BaseModel):
     # Make sure model populates both aliases and original attr name
     model_config = ConfigDict(
@@ -214,23 +230,22 @@ class ReviewModel(BaseModel):
     package_description: str = Field(
         "", validation_alias=AliasChoices("one-line_description_of_package")
     )
-    submitting_author: dict[str, str | None] = {}
-    all_current_maintainers: list[dict[str, str | None]] = {}
-    repository_link: str | None = None
+    submitting_author: ReviewUser | None = None
+    all_current_maintainers: list[ReviewUser] = Field(default_factory=list)
+    repository_link: str
     version_submitted: Optional[str] = None
     categories: Optional[list[str]] = None
-    editor: dict[str, str | None] = {}
-    reviewer_1: dict[str, str | None] = {}
-    reviewer_2: dict[str, str | None] = {}
+    editor: ReviewUser = {}
+    reviewers: list[ReviewUser] = Field(default_factory=list)
     archive: str | None = None
     version_accepted: str | None = None
     date_accepted: str | None = Field(
         default=None,
         validation_alias=AliasChoices("Date accepted", "date_accepted"),
     )
-    created_at: str = None
-    updated_at: str = None
-    closed_at: Optional[str] = None
+    created_at: datetime = None
+    updated_at: datetime = None
+    closed_at: Optional[datetime] = None
     issue_link: str = None
     joss: Optional[str] = None
     partners: Optional[list[str]] = None
@@ -255,22 +270,6 @@ def clean_date_review(cls, a_date: Optional[str]) -> str:
             else:
                 return f"{new_date[2]}-{new_date[0]}-{new_date[1]}"
 
-    @field_validator(
-        "created_at",
-        "updated_at",
-        "closed_at",
-        mode="before",
-    )
-    @classmethod
-    def clean_date(cls, a_date: Optional[str]) -> str:
-        """Cleans up a datetime from github and returns a date string
-
-        Runs the general clean_date function in this module as a validator.
-
-        """
-
-        return clean_date(a_date)
-
     @field_validator(
         "package_name",
         mode="before",
@@ -310,33 +309,12 @@ def clean_markdown_url(cls, repo: str) -> str:
         else:
             return repo
 
-    @field_validator(
-        "editor",
-        "reviewer_1",
-        "reviewer_2",
-        mode="before",
-    )
-    @classmethod
-    def clean_gh_url(cls, user: dict[str, str]) -> dict[str, str]:
-        """Remove markdown link remnants from gh usernames and name.
-
-        Sometimes editors and reviewers add names using github links.
-        Remove the link data.
-        """
-
-        user["github_username"] = user["github_username"].replace(
-            "https://github.com/", ""
-        )
-        user["name"] = re.sub(r"\[|\]", "", user["name"])
-
-        return user
-
     @field_validator(
         "categories",
         mode="before",
     )
     @classmethod
-    def clean_categories(cls, categories: list[str]) -> list[str]:
+    def clean_categories(cls, categories: list[str]) -> list[str] | None:
         """Make sure each category in the list is a valid value.
 
         Valid pyos software categories are:
@@ -358,6 +336,8 @@ def clean_categories(cls, categories: list[str]) -> list[str]:
         list[str]
             List of cleaned categories.
         """
+        if categories is None:
+            return None
 
         valid_categories = {
             "data-processing": "data-processing-munging",
@@ -375,3 +355,12 @@ def clean_categories(cls, categories: list[str]) -> list[str]:
                 # No match found, keep the original category
                 cleaned_cats.append(category)
         return cleaned_cats
+
+    @field_validator("all_current_maintainers", mode="before")
+    @classmethod
+    def listify(cls, item: Any):
+        """Make a field that's expected to be plural so before any validation"""
+        if not isinstance(item, list):
+            return [item]
+        else:
+            return item