Fix/parser/add attribute (#179)

* increase ratelimits on second_limiters

* Iceberg works. All scrapers working together. Symlinking works.

---------

Co-authored-by: Spoked <Spoked@localhost>

Author: dreulavelle

backend/program/__init__.py

@@ -58,9 +58,13 @@ def validate(self):
         return all(service.initialized for service in self.core_manager.services)
 
     def stop(self):
-        for service in self.core_manager.services:
-            if getattr(service, "running", False):
-                service.stop()
-        self.pickly.stop()
-        settings.save()
-        self.running = False
+        try:
+            for service in self.core_manager.services:
+                if getattr(service, "running", False):
+                    service.stop()
+            self.pickly.stop()
+            settings.save()
+            self.running = False
+        except Exception as e:
+            logger.error("Iceberg stopping with exception: %s", e)
+            pass

backend/program/content/plex_watchlist.py

@@ -29,26 +29,32 @@ def __init__(self, media_items: MediaItemContainer):
         self.media_items = media_items
         self.prev_count = 0
         self.updater = Trakt()
+        self.not_found_ids = []
 
     def validate_settings(self):
         if not self.settings.enabled:
             logger.debug("Plex Watchlists is set to disabled.")
             return False
         if self.settings.rss:
+            logger.info("Found Plex RSS URL. Validating...")
             try:
                 response = ping(self.settings.rss)
                 if response.ok:
                     self.rss_enabled = True
+                    logger.info("Plex RSS URL is valid.")
                     return True
                 else:
-                    logger.warn(f"Plex RSS URL is not reachable. Falling back to normal Watchlist.")
+                    logger.info(f"Plex RSS URL is not valid. Falling back to watching user Watchlist.")
                     return True
             except HTTPError as e:
                 if e.response.status_code in [404]:
-                    logger.error("Plex RSS URL is invalid. Falling back to normal Watchlist.")
+                    logger.warn("Plex RSS URL is Not Found. Falling back to watching user Watchlist.")
                     return True
-                if e.response.status_code >= 400 and e.response.status_code <= 500:
-                    logger.warn(f"Plex RSS URL is not reachable. Falling back to normal Watchlist.")
+                if e.response.status_code >= 400 and e.response.status_code <= 499:
+                    logger.warn(f"Plex RSS URL is not reachable. Falling back to watching user Watchlist.")
+                    return True
+                if e.response.status_code >= 500:
+                    logger.error(f"Plex is having issues validating RSS feed. Falling back to watching user Watchlist.")
                     return True
             except Exception as e:
                 logger.exception("Failed to validate Plex RSS URL: %s", e)
@@ -60,6 +66,13 @@ def run(self):
         if they are not already there"""
         items = self._create_unique_list()
         new_items = [item for item in items if item not in self.media_items] or []
+        if len(new_items) == 0:
+            logger.debug("No new items found in Plex Watchlist")
+            return
+        for check in new_items:
+            if check is None:
+                new_items.remove(check)
+                self.not_found_ids.append(check)
         container = self.updater.create_items(new_items)
         for item in container:
             item.set("requested_by", "Plex Watchlist")
@@ -76,6 +89,9 @@ def run(self):
                 logger.info("Added %s", item.log_string)
         elif length > 5:
             logger.info("Added %s items", length)
+        if len(self.not_found_ids) >= 1 and len(self.not_found_ids) <= 5:
+            for item in self.not_found_ids:
+                logger.info("Failed to add %s", item)
 
     def _create_unique_list(self):
         """Create a unique list of items from Plex RSS and Watchlist"""
@@ -88,7 +104,7 @@ def _create_unique_list(self):
     def _get_items_from_rss(self) -> list:
         """Fetch media from Plex RSS Feed"""
         try:
-            response_obj = get(self.settings.rss, timeout=30)
+            response_obj = get(self.settings.rss, timeout=60)
             data = json.loads(response_obj.response.content)
             items = data.get("items", [])
             ids = [

backend/program/media/container.py

@@ -2,6 +2,7 @@
 import threading
 import dill
 from typing import List, Optional
+from utils.logger import logger
 from program.media.item import MediaItem
 
 
@@ -24,7 +25,12 @@ def __iadd__(self, other):
         return self
 
     def sort(self, by, reverse):
-        self.items.sort(key=lambda item: item.get(by), reverse=reverse)
+        """Sort container by given attribute"""
+        try:
+            self.items.sort(key=lambda item: item.get(by), reverse=reverse)
+        except AttributeError as e:
+            logger.error("Failed to sort container: %s", e)
+            pass
 
     def __len__(self):
         """Get length of container"""

backend/program/media/item.py

@@ -1,5 +1,5 @@
-from datetime import datetime
 import threading
+from datetime import datetime
 from program.media.state import (
     Unknown,
     Content,
@@ -112,19 +112,6 @@ def to_extended_dict(self):
         dict["language"] = (self.language if hasattr(self, "language") else None,)
         dict["country"] = (self.country if hasattr(self, "country") else None,)
         dict["network"] = (self.network if hasattr(self, "network") else None,)
-        dict["streams"] = (self.streams if hasattr(self, "streams") else None,)
-        dict["active_stream"] = (
-            self.active_stream if hasattr(self, "active_stream") else None
-        ,)
-        dict["symlinked"] = (self.symlinked if hasattr(self, "symlinked") else None,)
-        dict["parsed"] = (self.parsed if hasattr(self, "parsed") else None,)
-        dict["parsed_data"] = (self.parsed_data if hasattr(self, "parsed_data") else None,)
-        dict["is_anime"] = (self.is_anime if hasattr(self, "is_anime") else None,)
-        dict["update_folder"] = (
-            self.update_folder if hasattr(self, "update_folder") else None
-        ,)
-        dict["file"] = (self.file if hasattr(self, "file") else None,)
-        dict["folder"] = (self.folder if hasattr(self, "folder") else None,)
         return dict
 
     def __iter__(self):

backend/program/plex.py

@@ -7,7 +7,7 @@
 from datetime import datetime
 from typing import Optional
 from plexapi.server import PlexServer
-from plexapi.exceptions import BadRequest
+from plexapi.exceptions import BadRequest, Unauthorized
 from pydantic import BaseModel
 # from program.updaters.trakt import get_imdbid_from_tvdb
 from utils.logger import logger
@@ -46,11 +46,14 @@ def __init__(self, media_items: MediaItemContainer):
             self.plex = PlexServer(
                 self.settings.url, self.settings.token, timeout=60
             )
+        except Unauthorized:
+            logger.warn("Plex is not authorized!")
+            return
         except BadRequest as e:
             logger.error("Plex is not configured correctly: %s", e)
             return
-        except Exception:
-            logger.error("Plex is not configured!")
+        except Exception as e:
+            logger.error("Plex exception thrown: %s", e)
             return
         self.running = False
         self.log_worker_count = False

backend/program/realdebrid.py

@@ -130,6 +130,9 @@ def chunks(lst, n):
                                 "active_stream",
                                 {"hash": stream_hash, "files": wanted_files, "id": None},
                             )
+                            all_filenames = [file_info["filename"] for file_info in wanted_files.values()]
+                            for file in all_filenames:
+                                logger.debug(f"Found cached file {file} for {item.log_string}")
                             return True
                         item.streams[stream_hash] = None
         return False

backend/program/scrapers/__init__.py

@@ -19,7 +19,7 @@ def __init__(self, _):
         self.key = "scraping"
         self.initialized = False
         self.settings = ScrapingConfig(**settings.get(self.key))
-        self.sm = ServiceManager(None, False, Torrentio, Orionoid, Jackett)
+        self.sm = ServiceManager(None, False, Orionoid, Torrentio, Jackett)
         if not any(service.initialized for service in self.sm.services):
             logger.error(
                 "You have no scraping services enabled, please enable at least one!"

backend/program/scrapers/jackett.py

@@ -1,11 +1,12 @@
 """ Jackett scraper module """
+import traceback
 from typing import Optional
 from pydantic import BaseModel
 from requests import ReadTimeout, RequestException
 from utils.logger import logger
 from utils.settings import settings_manager
 from utils.parser import parser
-from utils.request import RateLimitExceeded, get, RateLimiter
+from utils.request import RateLimitExceeded, get, RateLimiter, ping
 
 
 class JackettConfig(BaseModel):
@@ -24,9 +25,9 @@ def __init__(self, _):
         self.initialized = self.validate_settings()
         if not self.initialized and not self.api_key:
             return
-        self.parse_logging = False
         self.minute_limiter = RateLimiter(max_calls=60, period=60, raise_on_limit=True)
         self.second_limiter = RateLimiter(max_calls=1, period=10)
+        self.parse_logging = False
         logger.info("Jackett initialized!")
 
     def validate_settings(self) -> bool:
@@ -38,8 +39,8 @@ def validate_settings(self) -> bool:
             self.api_key = self.settings.api_key
             try:
                 url = f"{self.settings.url}/api/v2.0/indexers/!status:failing,test:passed/results/torznab?apikey={self.api_key}&cat=2000&t=movie&q=test"
-                response = get(url=url, retry_if_failed=False, timeout=60)
-                if response.is_ok:
+                response = ping(url=url, timeout=60)
+                if response.ok:
                     return True
             except ReadTimeout:
                 return True
@@ -65,18 +66,22 @@ def validate_settings(self) -> bool:
 
     def run(self, item):
         """Scrape Jackett for the given media items"""
+        if item is None or not self.initialized:
+            return
         try:
             self._scrape_item(item)
-        except RequestException:
+        except RateLimitExceeded as e:
             self.minute_limiter.limit_hit()
-            logger.debug("Jackett connection timeout for item: %s", item.log_string)
+            logger.warn("Jackett rate limit hit for item: %s", item.log_string)
             return
-        except RateLimitExceeded:
+        except RequestException as e:
             self.minute_limiter.limit_hit()
-            logger.debug("Jackett rate limit hit for item: %s", item.log_string)
+            logger.exception("Jackett request exception: %s", e, exc_info=True)
             return
         except Exception as e:
-            logger.exception("Jackett exception for item: %s - Exception: %s", item.log_string, e)
+            self.minute_limiter.limit_hit()
+            # logger.debug("Jackett exception for item: %s - Exception: %s", item.log_string, e.args[0], exc_info=True)
+            # logger.debug("Exception details: %s", traceback.format_exc())
             return
 
     def _scrape_item(self, item):
@@ -105,8 +110,11 @@ def api_scrape(self, item):
             if response.is_ok:
                 data = {}
                 streams = response.data["rss"]["channel"].get("item", [])
-                parsed_data_list = [parser.parse(item, stream.get("title")) for stream in streams]
+                parsed_data_list = [parser.parse(item, stream.get("title")) for stream in streams if type(stream) != str]
                 for stream, parsed_data in zip(streams, parsed_data_list):
+                    if type(stream) == str:
+                        logger.debug("Found another string: %s", stream)
+                        continue
                     if parsed_data.get("fetch", True) and parsed_data.get("title_match", False):
                         attr = stream.get("torznab:attr", [])
                         infohash_attr = next((a for a in attr if a.get("@name") == "infohash"), None)
@@ -118,6 +126,5 @@ def api_scrape(self, item):
                         logger.debug("Jackett Fetch: %s - Parsed item: %s", parsed_data["fetch"], parsed_data["string"])
                 if data:
                     item.parsed_data.extend(parsed_data_list)
-                    item.parsed_data.append({self.key: True})
                     return data, len(streams)
-                return {}, len(streams) or 0
+                return {}, 0

backend/program/scrapers/orionoid.py

@@ -29,8 +29,10 @@ def __init__(self, _):
             self.initialized = True
         else:
             return
+        self.orionoid_limit = 0
+        self.orionoid_remaining = 0
         self.parse_logging = False
-        self.max_calls = 50 if not self.is_premium else 60
+        self.max_calls = 100 if not self.is_premium else 60
         self.period = 86400 if not self.is_premium else 60
         self.minute_limiter = RateLimiter(max_calls=self.max_calls, period=self.period, raise_on_limit=True)
         self.second_limiter = RateLimiter(max_calls=1, period=10)
@@ -43,8 +45,20 @@ def validate_settings(self) -> bool:
             return False
         if self.settings.api_key:
             return True
-        logger.info("Orionoid is not configured and will not be used.")
-        return False
+        try:
+            url = f"https://api.orionoid.com?keyapp={KEY_APP}&keyuser={self.settings.api_key}&mode=user&action=retrieve"
+            response = get(url, retry_if_failed=False)
+            if response.is_ok:
+                return True
+            if not response.data.result.status == "success":
+                logger.error(f"Orionoid API Key is invalid. Status: {response.data.result.status}")
+                return False
+            if not response.is_ok:
+                logger.error(f"Orionoid Status Code: {response.status_code}, Reason: {response.reason}")
+                return False
+        except Exception as e:
+            logger.exception("Orionoid failed to initialize: %s", e)
+            return False
 
     def check_premium(self) -> bool:
         """
@@ -73,15 +87,19 @@ def run(self, item):
             self._scrape_item(item)
         except ConnectTimeout:
             self.minute_limiter.limit_hit()
-            logger.debug("Orionoid connection timeout for item: %s", item.log_string)
+            logger.warn("Orionoid connection timeout for item: %s", item.log_string)
             return
-        except RequestException:
+        except RequestException as e:
             self.minute_limiter.limit_hit()
-            logger.debug("Orionoid request exception for item: %s", item.log_string)
+            logger.exception("Orionoid request exception: %s", e)
             return
         except RateLimitExceeded:
             self.minute_limiter.limit_hit()
-            logger.debug("Orionoid rate limit hit for item: %s", item.log_string)
+            logger.warn("Orionoid rate limit hit for item: %s", item.log_string)
+            return
+        except Exception as e:
+            self.minute_limiter.limit_hit()
+            logger.exception("Orionoid exception for item: %s - Exception: %s", item.log_string, e)
             return
 
     def _scrape_item(self, item):
@@ -133,7 +151,14 @@ def api_scrape(self, item):
 
             with self.second_limiter:
                 response = get(url, retry_if_failed=False, timeout=60)
-            if response.is_ok and len(response.data.data.streams) > 0:
+            if response.is_ok and hasattr(response.data, "data"):
+
+                # Check and log Orionoid API limits
+                # self.orionoid_limit = response.data.data.requests.daily.limit
+                # self.orionoid_remaining = response.data.data.requests.daily.remaining
+                # if self.orionoid_remaining < 10:
+                #     logger.warning(f"Orionoid API limit is low. Limit: {self.orionoid_limit}, Remaining: {self.orionoid_remaining}")
+
                 parsed_data_list = [
                     parser.parse(item, stream.file.name)
                     for stream in response.data.data.streams
@@ -149,6 +174,5 @@ def api_scrape(self, item):
                         logger.debug("Orionoid Fetch: %s - Parsed item: %s", parsed_data["fetch"], parsed_data["string"])
                 if data:
                     item.parsed_data.extend(parsed_data_list)
-                    item.parsed_data.append({self.key: True})
                     return data, len(response.data.data.streams)
-            return {}, len(response.data.data.streams) or 0
+            return {}, 0