Filter tracks based on latest push date

Author: SanmerDev

Diff for: sync/cli/Main.py

@@ -6,6 +6,8 @@
 from datetime import datetime
 from pathlib import Path
 
+from dateutil.parser import parse
+
 from .Parameters import Parameters
 from .TypeDict import ConfigDict, TrackDict
 from ..core import (
@@ -219,7 +221,8 @@ def github(cls) -> int:
         tracks = GithubTracks(
             modules_folder=modules_folder,
             config=config,
-            api_token=cls._args.api_token
+            api_token=cls._args.api_token,
+            after_date=parse(cls._args.after_date).date()
         )
 
         if cls._args.update:
@@ -250,6 +253,9 @@ def github(cls) -> int:
                 use_ssh=cls._args.ssh
             )
 
+        if cls._args.clear:
+            tracks.clear_tracks()
+
         return cls.CODE_SUCCESS
 
     @classmethod

Diff for: sync/cli/Parameters.py

@@ -246,6 +246,15 @@ def configure_parser_github(cls, sub_parsers):
             default=Index.latest_version,
             help="Version of the index file ({0}), default: {1}.".format(ModulesJson.filename(), "%(default)s")
         )
+        p.add_argument(
+            "-d",
+            "--date",
+            dest="after_date",
+            metavar="DATE",
+            type=str,
+            default="2016-09-08",
+            help="Filter latest push date (before it), default: {0}.".format("%(default)s")
+        )
         p.add_argument(
             "-S",
             "--ssh",
@@ -263,6 +272,11 @@ def configure_parser_github(cls, sub_parsers):
             action="store_true",
             help="Update modules to the latest version."
         )
+        p.add_argument(
+            "--clear",
+            action="store_true",
+            help="Remove tracks, exclude those in the current session."
+        )
 
         cls.add_parser_git(p)
         env = cls.add_parser_env(p, add_quiet=True)
@@ -415,7 +429,7 @@ def add_parser_git(cls, p, add_set_size=True):
                 metavar="MAX_SIZE",
                 type=float,
                 default=50.0,
-                help="Limit size of file, default: {0} MB.".format("%(default)s")
+                help="Filter size of file (less than it), default: {0} MB.".format("%(default)s")
             )
 
         return git

Diff for: sync/core/Sync.py

@@ -97,11 +97,12 @@ def _check_tracks(obj, cls):
 
         return isinstance(obj, cls)
 
-    def create_github_tracks(self, api_token):
+    def create_github_tracks(self, api_token, after_date=None):
         self._tracks = GithubTracks(
             modules_folder=self._modules_folder,
             config=self._config,
-            api_token=api_token
+            api_token=api_token,
+            after_date=after_date
         )
         return self._tracks
 
@@ -112,13 +113,6 @@ def create_local_tracks(self):
         )
         return self._tracks
 
-    def create_tracks(self, **kwargs):
-        api_token = kwargs.get("api_token")
-        if api_token is not None:
-            return self.create_github_tracks(api_token)
-        else:
-            return self.create_local_tracks()
-
     def update(self, module_ids=None, force=False, **kwargs):
         user_name = kwargs.get("user_name")
         if user_name is not None:

Diff for: sync/core/Sync.pyi

@@ -1,3 +1,4 @@
+from datetime import date
 from pathlib import Path
 from typing import Optional, List
 
@@ -25,8 +26,7 @@ class Sync:
     def _update_jsons(self, track: TrackJson, force: bool) -> Optional[OnlineModule]: ...
     @staticmethod
     def _check_tracks(obj: BaseTracks, cls: type) -> bool: ...
-    def create_github_tracks(self, api_token: str) -> BaseTracks: ...
+    def create_github_tracks(self, api_token: str, after_date: Optional[date] = ...) -> BaseTracks: ...
     def create_local_tracks(self) -> BaseTracks: ...
-    def create_tracks(self, **kwargs) -> BaseTracks: ...
     def update(self, module_ids: Optional[List[str]] = ..., force: bool = ..., **kwargs): ...
     def push_by_git(self, branch: str): ...

Diff for: sync/track/GithubTracks.py

@@ -1,24 +1,30 @@
 import concurrent.futures
+import shutil
 from concurrent.futures import ThreadPoolExecutor
+from datetime import date
 
-import requests
 from github import Github, Auth, UnknownObjectException
 from github.Repository import Repository
 
 from .BaseTracks import BaseTracks
 from .LocalTracks import LocalTracks
 from ..error import MagiskModuleError, Result
 from ..model import TrackJson
-from ..utils import Log, StrUtils
+from ..utils import Log, GitHubGraphQLAPI
 
 
 class GithubTracks(BaseTracks):
-    def __init__(self, modules_folder, config, *, api_token):
+    def __init__(self, modules_folder, config, *, api_token, after_date=None):
         self._log = Log("GithubTracks", enable_log=config.enable_log, log_dir=config.log_dir)
         self._modules_folder = modules_folder
 
+        if after_date is None:
+            after_date = date(2016, 9, 8)
+
         self._api_token = api_token
+        self._after_date = after_date
         self._github = Github(auth=Auth.Token(api_token))
+        self._graphql_api = GitHubGraphQLAPI(api_token=api_token)
         self._tracks = list()
 
     @Result.catching()
@@ -41,18 +47,28 @@ def _get_from_repo_common(self, repo: Repository, use_ssh):
         else:
             issues = ""
 
-        donate_urls = self.get_sponsor_url(self._api_token, repo)
+        donate_urls = self._graphql_api.get_sponsor_url(
+            owner=repo.owner.login,
+            name=repo.name
+        )
         if len(donate_urls) == 0:
             donate = ""
         else:
             donate = donate_urls[0]
 
+        homepage = self._graphql_api.get_homepage_url(
+            owner=repo.owner.login,
+            name=repo.name
+        )
+        if homepage is None:
+            homepage = repo.html_url
+
         return TrackJson(
             id=repo.name,
             update_to=update_to,
             license=self.get_license(repo),
             changelog=changelog,
-            homepage=self.get_homepage_url(self._api_token, repo),
+            homepage=homepage,
             source=repo.clone_url,
             support=issues,
             donate=donate
@@ -61,6 +77,18 @@ def _get_from_repo_common(self, repo: Repository, use_ssh):
     def _get_from_repo(self, repo, cover, use_ssh):
         self._log.d(f"_get_from_repo: repo_name = {repo.name}")
 
+        pushed_at = self._graphql_api.get_pushed_at(
+            owner=repo.owner.login,
+            name=repo.name
+        )
+        if pushed_at is None:
+            return None
+
+        if pushed_at.date() < self._after_date:
+            msg = f"pushed at {pushed_at.date()}, too old"
+            self._log.w(f"_get_from_repo: [{repo.name}] -> {msg}")
+            return None
+
         result = self._get_from_repo_common(repo, use_ssh)
         if result.is_failure:
             msg = Log.get_msg(result.error)
@@ -111,6 +139,13 @@ def get_tracks(self, user_name, repo_names=None, *, cover=False, use_ssh=True):
         self._log.i(f"get_tracks: size = {self.size}")
         return self._tracks
 
+    def clear_tracks(self):
+        names = [track.id for track in self._tracks]
+        for module_folder in self._modules_folder.glob("*/"):
+            if module_folder.name not in names:
+                self._log.i(f"clear_tracks: [{module_folder.name}] -> removed")
+                shutil.rmtree(module_folder, ignore_errors=True)
+
     @property
     def size(self):
         return self._tracks.__len__()
@@ -148,59 +183,3 @@ def is_module_repo(cls, repo):
             return True
         except UnknownObjectException:
             return False
-
-    @classmethod
-    def _graphql_query(cls, api_token, query):
-        query = {"query": query}
-
-        response = requests.post(
-            url="https://api.github.com/graphql",
-            headers={
-                "Authorization": f"bearer {api_token}",
-                "Content-Type": "application/json",
-            },
-            json=query
-        )
-
-        if response.ok:
-            return response.json()
-        else:
-            return None
-
-    @classmethod
-    def get_sponsor_url(cls, api_token, repo):
-        params = "owner: \"{}\", name: \"{}\"".format(repo.owner.login, repo.name)
-        query = "query { repository(%s) { fundingLinks { platform url } } }" % params
-        result = cls._graphql_query(api_token, query)
-        if result is None:
-            return list()
-
-        links = list()
-        data = result["data"]
-        repository = data["repository"]
-        funding_links = repository["fundingLinks"]
-
-        for item in funding_links:
-            if item["platform"] == "GITHUB":
-                name = item["url"].split("/")[-1]
-                links.append(f"https://github.com/sponsors/{name}")
-            else:
-                links.append(item["url"])
-
-        return links
-
-    @classmethod
-    def get_homepage_url(cls, api_token, repo):
-        params = "owner: \"{}\", name: \"{}\"".format(repo.owner.login, repo.name)
-        query = "query { repository(%s) { homepageUrl } }" % params
-        result = cls._graphql_query(api_token, query)
-        if result is None:
-            return repo.html_url
-
-        data = result["data"]
-        repository = data["repository"]
-        homepage_url = repository["homepageUrl"]
-        if StrUtils.is_not_none(homepage_url):
-            return homepage_url
-        else:
-            return repo.html_url

Diff for: sync/track/GithubTracks.pyi

@@ -1,3 +1,4 @@
+from datetime import date
 from pathlib import Path
 from typing import Optional, List
 
@@ -7,17 +8,26 @@ from github.Repository import Repository
 from .BaseTracks import BaseTracks
 from ..error import Result
 from ..model import TrackJson, ConfigJson
-from ..utils import Log
+from ..utils import Log, GitHubGraphQLAPI
 
 
 class GithubTracks(BaseTracks):
     _log: Log
     _modules_folder: Path
     _api_token: str
+    _after_date: date
     _github: Github
+    _graphql_api: GitHubGraphQLAPI
     _tracks: List[TrackJson]
 
-    def __init__(self, modules_folder: Path, config: ConfigJson, *, api_token: str): ...
+    def __init__(
+        self,
+        modules_folder: Path,
+        config: ConfigJson,
+        *,
+        api_token: str,
+        after_date: Optional[date] = ...
+    ): ...
     @Result.catching()
     def _get_from_repo_common(self, repo: Repository, use_ssh: bool) -> Result: ...
     def _get_from_repo(self, repo: Repository, cover: bool, use_ssh: bool) -> Optional[TrackJson]: ...
@@ -37,6 +47,7 @@ class GithubTracks(BaseTracks):
         cover: bool = ...,
         use_ssh: bool = ...
     ) -> List[TrackJson]: ...
+    def clear_tracks(self): ...
     @property
     def size(self) -> int: ...
     @property
@@ -47,9 +58,3 @@ class GithubTracks(BaseTracks):
     def get_changelog(cls, repo: Repository) -> str: ...
     @classmethod
     def is_module_repo(cls, repo: Repository) -> bool: ...
-    @classmethod
-    def _graphql_query(cls, api_token: str, query: str) -> Optional[dict]: ...
-    @classmethod
-    def get_sponsor_url(cls, api_token: str, repo: Repository) -> List[str]: ...
-    @classmethod
-    def get_homepage_url(cls, api_token: str, repo: Repository) -> str: ...