Dealing with potentially transient 400 responses in YT API

medialab · Nov 8, 2023 · 81cffb4 · 81cffb4
1 parent 7d9f0fd
commit 81cffb4
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 4 deletions.
diff --git a/minet/youtube/client.py b/minet/youtube/client.py
@@ -37,6 +37,7 @@
     YouTubeNotFoundError,
     YouTubeInvalidAPIKeyError,
     YouTubeInvalidAPICallError,
+    YouTubePotentiallyTransientInvalidAPICallError,
     YouTubeInvalidVideoTargetError,
     YouTubeInvalidChannelTargetError,
     YouTubeExclusiveMemberError,
@@ -81,7 +82,10 @@ def __init__(self, key):
         self.scraper = YouTubeScraper()
 
         # YouTube's API is known to crash sometimes...
-        self.retryer = create_request_retryer(retry_on_statuses=(503,))
+        self.retryer = create_request_retryer(
+            retry_on_statuses=(503,),
+            additional_exceptions=[YouTubePotentiallyTransientInvalidAPICallError],
+        )
         self.url_formatter = YouTubeAPIURLFormatter()
 
     @retrying_method()
@@ -134,11 +138,19 @@ def request_json(self, url):
                 raise YouTubeNotFoundError
 
             if response.status >= 400:
-                if data is not None and "API key not valid" in getpath(
-                    data, ["error", "message"], ""
-                ):
+                error_message = ""
+
+                if data is not None:
+                    error_message = getpath(data, ["error", "message"], "").lower()
+
+                if "api key not valid" in error_message:
                     raise YouTubeInvalidAPIKeyError
 
+                if "transient" in error_message:
+                    raise YouTubePotentiallyTransientInvalidAPICallError(
+                        url, response.status, data
+                    )
+
                 raise YouTubeInvalidAPICallError(url, response.status, data)
 
             return data

diff --git a/minet/youtube/exceptions.py b/minet/youtube/exceptions.py
@@ -17,6 +17,21 @@ class YouTubeAPILimitReached(YouTubeError):
     pass
 
 
+class YouTubePotentiallyTransientInvalidAPICallError(YouTubeError):
+    def __init__(self, url, status, data):
+        super().__init__()
+        self.url = url
+        self.status = status
+        self.data = data
+
+    def __str__(self):
+        return super().__str__() + ", Url: %s, Status: %i, Data: %s" % (
+            self.url,
+            self.status,
+            self.data,
+        )
+
+
 class YouTubeInvalidAPICallError(YouTubeError):
     def __init__(self, url, status, data):
         super().__init__()