Added multiple QoL improvements + fixed slow download speeds (#112)

* - Added "Bypass SSL" option to ignore the weird Sony SSL when grabbing update meta file (without this sometimes Windows would not load the meta file) - Added "Add all updates to queue automatically" option to more quickly search for a title id and add all the updates to queue without opening the additional window - Exported links will contain `#name` and `#folder` to easily import them in jDownloader. The format for the folder is `GAME [TITLEID]` and the update files will be renamed to `GAME [TITLEID] VERSION.pkg` (make sure to import the jDownloader packagizer rule - Improved title id detection (eg. "BLES01807" was working but "BLES-01807" was not. The dashes are now ignored) * - Fixed slow download speeds - Folder and file names are now using the same format as the exported URL params * - updated README.md * - There is no need to restart the app in order for the "Bypass SSL" setting to be read correctly --------- Co-authored-by: Radu Ursache <[email protected]>
shinrax2 · Jan 28, 2024 · 524905d · 524905d
1 parent 7850190
commit 524905d
Show file tree

Hide file tree

Showing 6 changed files with 115 additions and 64 deletions.
diff --git a/PS3GUD.py b/PS3GUD.py
@@ -21,7 +21,7 @@
 import json
 import time
 import platform
-import ssl
+from urllib3.exceptions import InsecureRequestWarning
 
 #local files
 import utils
@@ -31,6 +31,8 @@
 import requests.adapters
 import keyring
 
+requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+
 class PS3GUD():
     def __init__(self, window=None):
         if window != None:
@@ -44,10 +46,6 @@ def __init__(self, window=None):
         self.DlList = Queue(self)
         self.titleid = ""
         self.proxies = {}
-        #handle sonys weak cert for their https server
-        self.https_session = requests.Session()
-        self.https_session.mount('https://a0.ww.np.dl.playstation.net', SonySSLContextAdapter())
-        self.pemfile = "./sony.pem"
 
         self.useDefaultConfig = True
         self.configDefaults = {}
@@ -57,6 +55,8 @@ def __init__(self, window=None):
         self.configDefaults["storageThresholdNew"] = 5
         self.configDefaults["currentLoc"] = "en"
         self.configDefaults["checkForNewRelease"] = True
+        self.configDefaults["bypass_ssl"] = True
+        self.configDefaults["add_all_updates_to_queue_automatically"] = False
         self.configDefaults["use_proxy"] = False
         self.configDefaults["proxy_ip"] = ""
         self.configDefaults["proxy_port"] = ""
@@ -66,6 +66,11 @@ def __init__(self, window=None):
         self.configDefaults["rename_pkgs"] = True
         self.configDefaults["update_titledb"] = True
 
+    def setupRequests(self):
+        self.https_session = requests.Session()
+        self.https_session.mount('https://a0.ww.np.dl.playstation.net', SonySSLContextAdapter(bypassSSL=self.getConfig("bypass_ssl")))
+        self.pemfile = "./sony.pem"
+
     def setWindow(self, window):
         self.logger.window = window
 
@@ -104,6 +109,8 @@ def loadConfig(self):
             self.logger.log(self.loc.getKey("msg_noConfigFile"))
             self.config = self.configDefaults
 
+        self.setupRequests()
+
     def setConfig(self, config):
         self.config = config
         self.saveConfig()
@@ -113,6 +120,7 @@ def saveConfig(self):
             f.write(json.dumps(self.config, sort_keys=True, indent=4, ensure_ascii=False))
         self.logger.log(self.loc.getKey("msg_configFileSaved"))
         self.useDefaultConfig = False 
+        self.setupRequests()
 
     def setProxyCredentials(self, pwd, user):
         keyring.set_password("ps3gud", "proxy_pass", pwd)
@@ -192,7 +200,7 @@ def getUpdates(self):
     def checkForUpdates(self, titleid):    
         #check given id
         check = False
-        titleid = titleid.upper()
+        titleid = titleid.upper().replace("-", "")
         for item in self.titledb:
             if titleid == item["id"]:
                 check = True
@@ -208,9 +216,14 @@ def checkForUpdates(self, titleid):
         updates = []
         url = urllib.parse.urljoin(urllib.parse.urljoin("https://a0.ww.np.dl.playstation.net/tpl/np/", self.titleid+"/"), self.titleid+"-ver.xml")
         try:
-            resp = self.https_session.get(url, verify=self.pemfile, proxies=self.proxies)
+            if self.getConfig("bypass_ssl"):
+                verify_value = False
+            else:
+                verify_value = self.pemfile
+
+            resp = self.https_session.get(url, verify=verify_value, proxies=self.proxies)
         except requests.exceptions.ConnectionError:
-            self.logger.log(self.loc.getKey("msg_metaNotAvailable"), "e")
+            self.logger.log(f"{self.loc.getKey('msg_metaNotAvailable')} ({url})")
             if self.getConfig("use_proxy"):
                 self.logger.log(self.loc.getKey("msg_checkProxySettings"))
             self.titleid = ""
@@ -256,9 +269,9 @@ def downloadFiles(self, window):
             sha1 = dl["sha1"]
             size = dl["size"]
             id = dl["gameid"]
-            fdir = os.path.join(self.getConfig("dldir")+"/", utils.filterIllegalCharsFilename(self.getTitleNameFromId(id))+"["+id+"]/")
+            fdir = os.path.join(self.getConfig("dldir")+"/", utils.filterIllegalCharsFilename(self.getTitleNameFromId(id))+" ["+id+"]/")
             if self.getConfig("rename_pkgs") == True:
-                fname = os.path.join(fdir, utils.filterIllegalCharsFilename(self.getTitleNameFromId(id)+"_["+id+"]_"+dl["version"]+".pkg"))
+                fname = os.path.join(fdir, utils.filterIllegalCharsFilename(self.getTitleNameFromId(id)+" ["+id+"] "+dl["version"]+".pkg"))
             else:
                 fname = os.path.join(fdir, utils.filterIllegalCharsFilename(os.path.basename(url)))
 
@@ -317,7 +330,7 @@ def _download_file(self, url, local_filename, size, window, num):
         text = window["window_main_progress_label"]
         bar = window["window_main_progress_bar"]
         size = int(size)
-        chunk_size=8192
+        chunk_size=98304
         count = 0
         already_loaded = 0
         with requests.get(url, stream=True, proxies=self.proxies) as r:
@@ -461,14 +474,24 @@ def exportQueue(self, exportFile):
         for id, data in games.items():
             s += f"{self.ps3.getTitleNameFromId(id)} [{id}]:\n\n"
             for url, version in data:
-                s += f"\t{url+'#name='+urllib.parse.quote(utils.filterIllegalCharsFilename(self.ps3.getTitleNameFromId(id)+'_['+id+']_'+version+'.pkg'))}\n"
+                filename = urllib.parse.quote(utils.filterIllegalCharsFilename(self.ps3.getTitleNameFromId(id)+' ['+id+'] '+version+'.pkg'))
+                folder = urllib.parse.quote(utils.filterIllegalCharsFilename(self.ps3.getTitleNameFromId(id)+' ['+id+']'))
+                s += f"\t{url+'#folder='+folder+'#name='+filename}\n"
             s += "\n"
         with open(exportFile, "w", encoding="utf8") as f:
             f.write(s)
 
 class SonySSLContextAdapter(requests.adapters.HTTPAdapter):
+    def __init__(self, bypassSSL=False, **kwargs):
+        self.bypassSSL = bypassSSL
+        super(SonySSLContextAdapter, self).__init__(**kwargs)
+
     def init_poolmanager(self, *args, **kwargs):
         context = ssl.create_default_context()
-        context.set_ciphers('DEFAULT:@SECLEVEL=1')
+        if self.bypassSSL:
+            context.check_hostname = False
+            context.verify_mode = ssl.CERT_NONE
+        else:
+            context.set_ciphers('DEFAULT:@SECLEVEL=1')
         kwargs['ssl_context'] = context
-        return super(SonySSLContextAdapter, self).init_poolmanager(*args, **kwargs)
+        return super(SonySSLContextAdapter, self).init_poolmanager(*args, **kwargs)
diff --git a/README.md b/README.md
@@ -15,7 +15,9 @@ Grab the latest release [here](https://github.com/shinrax2/PS3GameUpdateDownload
 Sony has announced that they will be closing the PS3 store on 02.07.21 and later reversed that decision. It seems that some game updates got deleted in the time between the announcement and its reversal. I recommend you backup your downloaded game updates.
 
 # JDownloader PKG Renaming
-If you want to use the queue export feature and download the updates with JDownloader import `ps3gud_jdownloader_renamepkgs.packagizer` into your packagizer settings
+If you want to use the queue export feature and download the updates with [JDownloader](https://jdownloader.org/download/index) import `ps3gud_jdownloader_renamepkgs.packagizer` into your packagizer settings and make sure to set the following rule order:
+
+![Main window](https://github.com/shinrax2/PS3GameUpdateDownloader/raw/master/screenshots/jdownloader_order.png)
 
 # Contributions
 Contributions in form of translations and bug fixes are welcome.

diff --git a/gui.py b/gui.py
@@ -199,6 +199,8 @@ def configWin(self, nocancel=False):
             [sg.Text(self.loc.getKey("window_config_storageThreshold_label"), size=(40,1)), sg.In(self.ps3.getConfig("storageThresholdNew"), key="storageThreshold")],
             [sg.Text(self.loc.getKey("window_config_currentLoc_label"), size=(40,1)), sg.OptionMenu(locChoices, size=(8, 15), key="currentLoc", default_value=self.loc.getKey("language_name"))],
             [sg.Text(self.loc.getKey("window_config_renamepkgs_label"), size=(40,1), tooltip=self.loc.getKey("window_config_renamepkgs_tooltip")), sg.Checkbox("", default=self.ps3.getConfig("rename_pkgs"), key="rename_pkgs", tooltip=self.loc.getKey("window_config_renamepkgs_tooltip"))],
+            [sg.Text(self.loc.getKey("window_config_add_all_updates_to_queue_automatically_label"), size=(40,1), tooltip=self.loc.getKey("window_config_add_all_updates_to_queue_automatically_tooltip")), sg.Checkbox("", default=self.ps3.getConfig("add_all_updates_to_queue_automatically"), key="add_all_updates_to_queue_automatically", tooltip=self.loc.getKey("window_config_add_all_updates_to_queue_automatically_tooltip"))],
+            [sg.Text(self.loc.getKey("window_config_bypass_ssl_label"), size=(40,1), tooltip=self.loc.getKey("window_config_bypass_ssl_tooltip")), sg.Checkbox("", default=self.ps3.getConfig("bypass_ssl"), key="bypass_ssl", tooltip=self.loc.getKey("window_config_bypass_ssl_tooltip"))],
             [sg.Text(self.loc.getKey("window_config_updatetitledb_label"), size=(40,1)), sg.Checkbox("", default=self.ps3.getConfig("update_titledb"), key="update_titledb")],
             [sg.Text(self.loc.getKey("window_config_useproxy_label"), size=(40,1)), sg.Checkbox("", default=self.ps3.getConfig("use_proxy"), key="use_proxy", enable_events=True)],
             [sg.Text(self.loc.getKey("window_config_proxyip_label"), size=(40,1)), sg.In(self.ps3.getConfig("proxy_ip"), key="proxy_ip")],
@@ -243,7 +245,7 @@ def configWin(self, nocancel=False):
                 for l in ll:
                     if cL == l["language_name"]:
                         cL = l["language_short"]
-                config = { "dldir": valConfig["dldir"], "verify": valConfig["verify"], "checkIfAlreadyDownloaded": valConfig["checkIfAlreadyDownloaded"], "storageThresholdNew": valConfig["storageThreshold"], "currentLoc": cL , "proxy_ip": valConfig["proxy_ip"], "proxy_port": valConfig["proxy_port"], "use_proxy": valConfig["use_proxy"], "rename_pkgs": valConfig["rename_pkgs"], "update_titledb": valConfig["update_titledb"]}
+                config = { "dldir": valConfig["dldir"], "verify": valConfig["verify"], "checkIfAlreadyDownloaded": valConfig["checkIfAlreadyDownloaded"], "storageThresholdNew": valConfig["storageThreshold"], "currentLoc": cL , "proxy_ip": valConfig["proxy_ip"], "proxy_port": valConfig["proxy_port"], "use_proxy": valConfig["use_proxy"], "rename_pkgs": valConfig["rename_pkgs"], "update_titledb": valConfig["update_titledb"], "bypass_ssl": valConfig["bypass_ssl"], "add_all_updates_to_queue_automatically": valConfig["add_all_updates_to_queue_automatically"]}
                 self.ps3.setConfig(config)
                 if self.ps3.getConfig("use_proxy") == True:
                     self.ps3.setProxyCredentials(valConfig["proxy_pass"], valConfig["proxy_user"])
@@ -261,61 +263,69 @@ def selectWin(self):
             updlen = 0
             self.tryDl = False
         if updlen > 0:
-            self.mainWindow.hide()
-            data = self.updatePackToTable(self.ps3.getUpdates())
-            lay2 = [
-                [sg.Text(self.loc.getKey("window_select_text_label", [self.ps3.getTitleNameFromId(), self.ps3.titleid]))],
-                [sg.Table(values=data, headings=[self.loc.getKey("window_select_table_ver"), self.loc.getKey("window_select_table_size"), self.loc.getKey("window_select_table_sysver")], key="Table", enable_events=True)],
-                [sg.Text(self.loc.getKey("window_select_help_label"), size=(45,2))],
-                [sg.Button(self.loc.getKey("window_select_download_btn"), key="OK", disabled=True),sg.Button(self.loc.getKey("window_select_queue_btn"), key="Queue", disabled=True), sg.Button(self.loc.getKey("window_select_cancel_btn"), key="Cancel")]
-            ]
-            self.selectWindow = sg.Window(self.loc.getKey("window_select_title"), lay2, icon=self.iconpath)
-            while True:
-                ev2, val2 = self.selectWindow.Read()
-                if len(val2["Table"]) > 0:
-                    self.selectWindow["OK"].Update(disabled=False)
-                    self.selectWindow["Queue"].Update(disabled=False)
-                if len(val2["Table"]) == 0:
-                    self.selectWindow["OK"].Update(disabled=True)
-                    self.selectWindow["Queue"].Update(disabled=True)
-                if ev2 in (None, "Exit"):
-                    self.selectWindow.Close()
-                    self.mainWindow.UnHide()
-                    self.tryDl = False
-                    break
-                if ev2 == "Cancel":
-                    self.selectWindow.Close()
-                    self.mainWindow.UnHide()
-                    self.tryDl = False
-                    break
-                if ev2 == "OK" and len(val2["Table"]) > 0:
-                    if len(val2["Table"]) == 1:
-                        self.ps3.DlList.addEntry(self.ps3.getUpdates()[val2["Table"][0]])
-                        self.selectWindow.Close()
-                        self.mainWindow.UnHide()
-                        self.tryDl = True
-                        break
-                    if len(val2["Table"]) > 1:
-                        for row in val2["Table"]:
-                            self.ps3.DlList.addEntry(self.ps3.getUpdates()[row])
-                        self.selectWindow.Close()
-                        self.mainWindow.UnHide()
-                        self.tryDl = True
-                        break
-                if ev2 == "Queue" and len(val2["Table"]) > 0:
-                    if len(val2["Table"]) == 1:    
-                        self.ps3.DlList.addEntry(self.ps3.getUpdates()[val2["Table"][0]])
+            updates = self.ps3.getUpdates()
+
+            if self.ps3.getConfig("add_all_updates_to_queue_automatically"):
+                for update in updates:
+                    self.ps3.DlList.addEntry(update)
+
+                self.ps3.logger.log(f"Added {updlen} updates to queue")
+            else:
+                self.mainWindow.hide()
+                data = self.updatePackToTable(updates)
+                lay2 = [
+                    [sg.Text(self.loc.getKey("window_select_text_label", [self.ps3.getTitleNameFromId(), self.ps3.titleid]))],
+                    [sg.Table(values=data, headings=[self.loc.getKey("window_select_table_ver"), self.loc.getKey("window_select_table_size"), self.loc.getKey("window_select_table_sysver")], key="Table", enable_events=True)],
+                    [sg.Text(self.loc.getKey("window_select_help_label"), size=(45,2))],
+                    [sg.Button(self.loc.getKey("window_select_download_btn"), key="OK", disabled=True),sg.Button(self.loc.getKey("window_select_queue_btn"), key="Queue", disabled=True), sg.Button(self.loc.getKey("window_select_cancel_btn"), key="Cancel")]
+                ]
+                self.selectWindow = sg.Window(self.loc.getKey("window_select_title"), lay2, icon=self.iconpath)
+                while True:
+                    ev2, val2 = self.selectWindow.Read()
+                    if len(val2["Table"]) > 0:
+                        self.selectWindow["OK"].Update(disabled=False)
+                        self.selectWindow["Queue"].Update(disabled=False)
+                    if len(val2["Table"]) == 0:
+                        self.selectWindow["OK"].Update(disabled=True)
+                        self.selectWindow["Queue"].Update(disabled=True)
+                    if ev2 in (None, "Exit"):
                         self.selectWindow.Close()
                         self.mainWindow.UnHide()
                         self.tryDl = False
                         break
-                    if len(val2["Table"]) > 1:
-                        for row in val2["Table"]:
-                            self.ps3.DlList.addEntry(self.ps3.getUpdates()[row])
+                    if ev2 == "Cancel":
                         self.selectWindow.Close()
                         self.mainWindow.UnHide()
                         self.tryDl = False
                         break
+                    if ev2 == "OK" and len(val2["Table"]) > 0:
+                        if len(val2["Table"]) == 1:
+                            self.ps3.DlList.addEntry(self.ps3.getUpdates()[val2["Table"][0]])
+                            self.selectWindow.Close()
+                            self.mainWindow.UnHide()
+                            self.tryDl = True
+                            break
+                        if len(val2["Table"]) > 1:
+                            for row in val2["Table"]:
+                                self.ps3.DlList.addEntry(self.ps3.getUpdates()[row])
+                            self.selectWindow.Close()
+                            self.mainWindow.UnHide()
+                            self.tryDl = True
+                            break
+                    if ev2 == "Queue" and len(val2["Table"]) > 0:
+                        if len(val2["Table"]) == 1:    
+                            self.ps3.DlList.addEntry(self.ps3.getUpdates()[val2["Table"][0]])
+                            self.selectWindow.Close()
+                            self.mainWindow.UnHide()
+                            self.tryDl = False
+                            break
+                        if len(val2["Table"]) > 1:
+                            for row in val2["Table"]:
+                                self.ps3.DlList.addEntry(self.ps3.getUpdates()[row])
+                            self.selectWindow.Close()
+                            self.mainWindow.UnHide()
+                            self.tryDl = False
+                            break
 
 
     def queueWin(self):

diff --git a/loc/en.json b/loc/en.json
@@ -70,6 +70,22 @@
     "window_config_renamepkgs_tooltip": {
         "string": "Renames PKGs to $GAMENAME_[$GAMEID]_$GAMEVERSION",
         "comment": ""
+    },
+	"window_config_add_all_updates_to_queue_automatically_label": {
+        "string": "Add all updates to queue automatically:",
+        "comment": ""
+    },
+    "window_config_add_all_updates_to_queue_automatically_tooltip": {
+        "string": "Automatically add all game updates to queue without displaying the confirmation and selection window",
+        "comment": ""
+    },
+	"window_config_bypass_ssl_label": {
+        "string": "Bypass SSL:",
+        "comment": ""
+    },
+    "window_config_bypass_ssl_tooltip": {
+        "string": "Do not attempt to resolve and validate SSL when downloading",
+        "comment": ""
     },
     "window_config_updatetitledb_label": {
         "string": "Update TitleDB on startup:",