[pornhub] add gallery & user extractor (#282)

docs/supportedsites.rst

@@ -78,6 +78,7 @@ Pinterest            https://www.pinterest.com/          Boards, Pins, pin.it Li
 Pixiv                https://www.pixiv.net/              |pixiv-C|                                          Required
 Pixnet               https://www.pixnet.net/             |pixnet-C|
 Plurk                https://www.plurk.com/              Posts, Timelines
+Pornhub              https://www.pornhub.com/            Images from Users, Galleries
 Pornreactor          http://pornreactor.cc/              |pornreactor-C|
 PowerManga           https://read.powermanga.org/        Chapters, Manga
 Pururin              https://pururin.io/                 Galleries

gallery_dl/extractor/__init__.py

@@ -72,6 +72,7 @@
     "pixiv",
     "pixnet",
     "plurk",
+    "pornhub",
     "pururin",
     "reactor",
     "readcomiconline",

gallery_dl/extractor/pornhub.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2019 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://www.pornhub.com/"""
+
+from .common import Extractor, Message
+from .. import text, exception
+
+
+BASE_PATTERN = r"(?:https?://)?(?:[^.]+\.)?pornhub\.com"
+
+
+class PornhubExtractor(Extractor):
+    """Base class for pornhub extractors"""
+    category = "pornhub"
+    root = "https://www.pornhub.com"
+
+
+class PornhubGalleryExtractor(PornhubExtractor):
+    """Extractor for image galleries on pornhub.com"""
+    subcategory = "gallery"
+    directory_fmt = ("{category}", "{user}", "{gallery[id]} {gallery[title]}")
+    filename_fmt = "{num:>03}_{id}.{extension}"
+    archive_fmt = "{id}"
+    pattern = BASE_PATTERN + r"/album/(\d+)"
+    test = (
+        ("https://www.pornhub.com/album/1708982", {
+            "pattern": r"https://\w+.phncdn.com/pics/albums/\d+/\d+/\d+/\d+/",
+            "count": 93,
+            "keyword": {
+                "id": int,
+                "num": int,
+                "score": int,
+                "views": int,
+                "caption": str,
+                "user": "Unknown",
+                "gallery": {
+                    "id"   : 1708982,
+                    "score": int,
+                    "views": int,
+                    "tags" : list,
+                    "title": "Random Hentai",
+                },
+            },
+        }),
+        ("https://www.pornhub.com/album/37180171", {
+            "exception": exception.AuthorizationError,
+        }),
+    )
+
+    def __init__(self, match):
+        PornhubExtractor.__init__(self, match)
+        self.gallery_id = match.group(1)
+        self._first = None
+
+    def items(self):
+        data = self.metadata()
+        yield Message.Version, 1
+        yield Message.Directory, data
+        for num, image in enumerate(self.images(), 1):
+            url = image["url"]
+            image.update(data)
+            image["num"] = num
+            yield Message.Url, url, text.nameext_from_url(url, image)
+
+    def metadata(self):
+        url = "{}/album/{}".format(
+            self.root, self.gallery_id)
+        extr = text.extract_from(self.request(url).text)
+
+        title = extr("<title>", "</title>")
+        score = extr('<div id="albumGreenBar" style="width:', '"')
+        views = extr('<div id="viewsPhotAlbumCounter">', '<')
+        tags = extr('<div id="photoTagsBox"', '<script')
+        self._first = extr('<a href="/photo/', '"')
+        title, _, user = title.rpartition(" - ")
+
+        return {
+            "user" : text.unescape(user[:-14]),
+            "gallery": {
+                "id"   : text.parse_int(self.gallery_id),
+                "title": text.unescape(title),
+                "score": text.parse_int(score.partition("%")[0]),
+                "views": text.parse_int(views.partition(" ")[0]),
+                "tags" : text.split_html(tags)[2:],
+            },
+        }
+
+    def images(self):
+        url = "{}/album/show_album_json?album={}".format(
+            self.root, self.gallery_id)
+        response = self.request(url)
+
+        if response.content == b"Permission denied":
+            raise exception.AuthorizationError()
+        images = response.json()
+        key = end = self._first
+
+        while True:
+            img = images[key]
+            yield {
+                "url"    : img["img_large"],
+                "caption": img["caption"],
+                "id"     : text.parse_int(img["id"]),
+                "views"  : text.parse_int(img["times_viewed"]),
+                "score"  : text.parse_int(img["vote_percent"]),
+            }
+            key = img["next"]
+            if key == end:
+                return
+
+
+class PornhubUserExtractor(PornhubExtractor):
+    """Extractor for all galleries of a pornhub user"""
+    subcategory = "user"
+    pattern = (BASE_PATTERN + r"/(users|model)/([^/?&#]+)"
+               "(?:/photos(?:/(public|private|favorites))?)?/?$")
+    test = (
+        ("https://www.pornhub.com/users/flyings0l0/photos/public", {
+            "pattern": PornhubGalleryExtractor.pattern,
+            "count": ">= 8",
+        }),
+        ("https://www.pornhub.com/users/flyings0l0/"),
+        ("https://www.pornhub.com/users/flyings0l0/photos/public"),
+        ("https://www.pornhub.com/users/flyings0l0/photos/private"),
+        ("https://www.pornhub.com/users/flyings0l0/photos/favorites"),
+        ("https://www.pornhub.com/model/bossgirl/photos"),
+    )
+
+    def __init__(self, match):
+        PornhubExtractor.__init__(self, match)
+        self.type, self.user, self.cat = match.groups()
+
+    def items(self):
+        url = "{}/{}/{}/photos/{}/ajax".format(
+            self.root, self.type, self.user, self.cat or "public")
+        params = {"page": 1}
+        headers = {
+            "Referer": url[:-5],
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+        data = {"_extractor": PornhubGalleryExtractor}
+        yield Message.Version, 1
+        while True:
+            page = self.request(
+                url, method="POST", headers=headers, params=params).text
+            if not page:
+                return
+            for gid in text.extract_iter(page, 'id="albumphoto', '"'):
+                yield Message.Queue, self.root + "/album/" + gid, data
+            params["page"] += 1