Implement multithreading for improved performance

Author: SanmerDev

Diff for: sync/core/Sync.py

@@ -1,4 +1,6 @@
+import concurrent.futures
 import subprocess
+from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 
 from .Config import Config
@@ -133,12 +135,17 @@ def update(self, module_ids=None, force=False, **kwargs):
         else:
             tracks = self._tracks.get_tracks(module_ids)
 
-        for track in tracks:
-            online_module = self._update_jsons(track=track, force=force)
-            if online_module is None:
-                continue
+        with ThreadPoolExecutor() as executor:
+            futures = []
+            for track in tracks:
+                futures.append(
+                    executor.submit(self._update_jsons, track=track, force=force)
+                )
 
-            self._log.i(f"update: [{track.id}] -> update to {online_module.version_display}")
+            for future in concurrent.futures.as_completed(futures):
+                online_module = future.result()
+                if online_module is not None:
+                    self._log.i(f"update: [{online_module.id}] -> update to {online_module.version_display}")
 
     def push_by_git(self, branch):
         json_file = self._json_folder.joinpath(ModulesJson.filename())

Diff for: sync/track/GithubTracks.py

@@ -1,3 +1,6 @@
+import concurrent.futures
+from concurrent.futures import ThreadPoolExecutor
+
 import requests
 from github import Github, Auth, UnknownObjectException
 from github.Repository import Repository
@@ -80,24 +83,30 @@ def get_tracks(self, user_name, repo_names=None, *, cover=False, use_ssh=True):
         self._log.i(f"get_tracks: user_name = {user_name}")
 
         user = self._github.get_user(user_name)
-        if repo_names is None:
-            for repo in user.get_repos():
-                track_json = self._get_from_repo(repo, cover, use_ssh)
-                if track_json is not None:
-                    self._tracks.append(track_json)
-        else:
+        repos = []
+
+        if repo_names is not None:
             for repo_name in repo_names:
                 try:
                     repo = user.get_repo(repo_name)
+                    repos.append(repo)
                 except UnknownObjectException as err:
-                    repo = None
                     msg = Log.get_msg(err)
                     self._log.e(f"get_tracks: [{repo_name}] -> {msg}")
+        else:
+            repos = user.get_repos()
+
+        with ThreadPoolExecutor() as executor:
+            futures = []
+            for repo in repos:
+                futures.append(
+                    executor.submit(self._get_from_repo, repo=repo, cover=cover, use_ssh=use_ssh)
+                )
 
-                if repo is not None:
-                    track_json = self._get_from_repo(repo, cover, use_ssh)
-                    if track_json is not None:
-                        self._tracks.append(track_json)
+            for future in concurrent.futures.as_completed(futures):
+                track_json = future.result()
+                if track_json is not None:
+                    self._tracks.append(track_json)
 
         self._log.i(f"get_tracks: size = {self.size}")
         return self._tracks