Fixing yt captions when encountering inconsistent transcripts

Sometimes duration is not given, we default it to 0

Author: Yomguithereal

minet/scrape/soup.py

@@ -1,5 +1,5 @@
 import re
-from typing import List, Optional, cast
+from typing import List, Optional, cast, overload
 
 import warnings
 from contextlib import contextmanager
@@ -94,8 +94,16 @@ def get_outer_html(self) -> str:
     def __getitem__(self, name: str) -> str:
         return cast(str, super().__getitem__(name))
 
-    def get(self, name: str) -> Optional[str]:
-        return cast(Optional[str], super().get(name))
+    @overload
+    def get(self, name: str, default: str = ...) -> str:
+        ...
+
+    @overload
+    def get(self, name: str, default: None = ...) -> Optional[str]:
+        ...
+
+    def get(self, name: str, default: Optional[str] = None) -> Optional[str]:
+        return cast(Optional[str], super().get(name, default))
 
     def get_list(self, name: str) -> List[str]:
         value = super().get(name)

minet/youtube/scraper.py

@@ -101,10 +101,14 @@ def get_video_captions(
         captions = []
 
         for item in soup.select("text"):
+
+            # NOTE: sometimes duration is absent. I don't really
+            # know what is the best solution there (merging with
+            # previous item?). So for now, we default duration to 0.
             captions.append(
                 YouTubeCaptionLine(
                     float(item["start"]),
-                    float(item["dur"]),
+                    float(item.get("dur", "0")),
                     unescape(item.get_text().strip()),
                 )
             )