Merge pull request #362 from meisnate12/develop

v1.12.0

.dockerignore

@@ -7,4 +7,13 @@
 README.md
 LICENSE
 .gitignore
+.dockerignore
 .git
+.github
+*.psd
+config/**/*
+config
+Dockerfile
+venv
+.idea
+test.py

Dockerfile

@@ -1,20 +1,15 @@
-FROM python:3-slim
-VOLUME /config
+FROM python:3.9-slim
+RUN echo "**** install system packages ****" \
+ && apt-get update \
+ && apt-get upgrade -y --no-install-recommends \
+ && apt-get install -y tzdata --no-install-recommends \
+ && apt-get install -y gcc g++ libxml2-dev libxslt-dev libz-dev
+COPY requirements.txt /
+RUN echo "**** install python packages ****" \
+ && pip3 install --no-cache-dir --upgrade --requirement /requirements.txt \
+ && apt-get autoremove -y \
+ && apt-get clean \
+ && rm -rf /requirements.txt /tmp/* /var/tmp/* /var/lib/apt/lists/*
 COPY . /
-RUN \
-	echo "**** install system packages ****" && \
-		apt-get update && \
-		apt-get upgrade -y --no-install-recommends && \
-		apt-get install -y tzdata --no-install-recommends && \
-		apt-get install -y gcc g++ libxml2-dev libxslt-dev libz-dev && \
-	echo "**** install python packages ****" && \
-		pip3 install --no-cache-dir --upgrade --requirement /requirements.txt && \
-	echo "**** cleanup ****" && \
-		apt-get autoremove -y && \
-		apt-get clean && \
-		rm -rf \
-			/requirements.txt \
-			/tmp/* \
-			/var/tmp/* \
-			/var/lib/apt/lists/*
+VOLUME /config
 ENTRYPOINT ["python3", "plex_meta_manager.py"]

config/config.yml.template

@@ -89,3 +89,6 @@ mal:
     token_type:
     expires_in:
     refresh_token:
+anidb:          # Optional
+  username: ######
+  password: ######

modules/anidb.py

@@ -1,42 +1,54 @@
-import logging, requests
-from lxml import html
+import logging, time
 from modules import util
 from modules.util import Failed
-from retrying import retry
 
 logger = logging.getLogger("Plex Meta Manager")
 
-builders = ["anidb_id", "anidb_relation", "anidb_popular"]
+builders = ["anidb_id", "anidb_relation", "anidb_popular", "anidb_tag"]
+base_url = "https://anidb.net"
+urls = {
+    "anime": f"{base_url}/anime",
+    "popular": f"{base_url}/latest/anime/popular/?h=1",
+    "relation": "/relation/graph",
+    "tag": f"{base_url}/tag",
+    "login": f"{base_url}/perl-bin/animedb.pl"
+}
 
 class AniDB:
-    def __init__(self, config):
+    def __init__(self, config, params):
         self.config = config
-        self.urls = {
-            "anime": "https://anidb.net/anime",
-            "popular": "https://anidb.net/latest/anime/popular/?h=1",
-            "relation": "/relation/graph"
-        }
+        self.username = params["username"] if params else None
+        self.password = params["password"] if params else None
+        if params and not self._login(self.username, self.password).xpath("//li[@class='sub-menu my']/@title"):
+            raise Failed("AniDB Error: Login failed")
 
-    @retry(stop_max_attempt_number=6, wait_fixed=10000)
-    def _request(self, url, language):
-        return html.fromstring(requests.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content)
+    def _request(self, url, language=None, post=None):
+        if post:
+            return self.config.post_html(url, post, headers=util.header(language))
+        else:
+            return self.config.get_html(url, headers=util.header(language))
+
+    def _login(self, username, password):
+        data = {"show": "main", "xuser": username, "xpass": password, "xdoautologin": "on"}
+        return self._request(urls["login"], post=data)
 
     def _popular(self, language):
-        response = self._request(self.urls["popular"], language)
+        response = self._request(urls["popular"], language=language)
         return util.get_int_list(response.xpath("//td[@class='name anime']/a/@href"), "AniDB ID")
 
     def _relations(self, anidb_id, language):
-        response = self._request(f"{self.urls['anime']}/{anidb_id}{self.urls['relation']}", language)
+        response = self._request(f"{urls['anime']}/{anidb_id}{urls['relation']}", language=language)
         return util.get_int_list(response.xpath("//area/@href"), "AniDB ID")
 
     def _validate(self, anidb_id, language):
-        response = self._request(f"{self.urls['anime']}/{anidb_id}", language)
+        response = self._request(f"{urls['anime']}/{anidb_id}", language=language)
         ids = response.xpath(f"//*[text()='a{anidb_id}']/text()")
         if len(ids) > 0:
             return util.regex_first_int(ids[0], "AniDB ID")
         raise Failed(f"AniDB Error: AniDB ID: {anidb_id} not found")
 
-    def validate_anidb_list(self, anidb_list, language):
+    def validate_anidb_ids(self, anidb_ids, language):
+        anidb_list = util.get_int_list(anidb_ids, "AniDB ID")
         anidb_values = []
         for anidb_id in anidb_list:
             try:
@@ -47,20 +59,35 @@ def validate_anidb_list(self, anidb_list, language):
             return anidb_values
         raise Failed(f"AniDB Error: No valid AniDB IDs in {anidb_list}")
 
-    def get_items(self, method, data, language):
-        pretty = util.pretty_names[method] if method in util.pretty_names else method
+    def _tag(self, tag, limit, language):
+        anidb_ids = []
+        current_url = f"{urls['tag']}/{tag}"
+        while True:
+            response = self._request(current_url, language=language)
+            anidb_ids.extend(util.get_int_list(response.xpath("//td[@class='name main anime']/a/@href"), "AniDB ID"))
+            next_page_list = response.xpath("//li[@class='next']/a/@href")
+            if len(anidb_ids) >= limit or len(next_page_list) == 0:
+                break
+            time.sleep(2)
+            current_url = f"{base_url}{next_page_list[0]}"
+        return anidb_ids[:limit]
+
+    def get_anidb_ids(self, method, data, language):
         anidb_ids = []
         if method == "anidb_popular":
-            logger.info(f"Processing {pretty}: {data} Anime")
+            logger.info(f"Processing AniDB Popular: {data} Anime")
             anidb_ids.extend(self._popular(language)[:data])
+        elif method == "anidb_tag":
+            logger.info(f"Processing AniDB Tag: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Tag ID: {data['tag']}")
+            anidb_ids = self._tag(data["tag"], data["limit"], language)
+        elif method == "anidb_id":
+            logger.info(f"Processing AniDB ID: {data}")
+            anidb_ids.append(data)
+        elif method == "anidb_relation":
+            logger.info(f"Processing AniDB Relation: {data}")
+            anidb_ids.extend(self._relations(data, language))
         else:
-            logger.info(f"Processing {pretty}: {data}")
-            if method == "anidb_id":                            anidb_ids.append(data)
-            elif method == "anidb_relation":                    anidb_ids.extend(self._relations(data, language))
-            else:                                               raise Failed(f"AniDB Error: Method {method} not supported")
-        movie_ids, show_ids = self.config.Convert.anidb_to_ids(anidb_ids)
+            raise Failed(f"AniDB Error: Method {method} not supported")
         logger.debug("")
         logger.debug(f"{len(anidb_ids)} AniDB IDs Found: {anidb_ids}")
-        logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}")
-        logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}")
-        return movie_ids, show_ids
+        return anidb_ids