diff --git a/README.rst b/README.rst index fa823b5bdb..272ac23aa1 100644 --- a/README.rst +++ b/README.rst @@ -214,8 +214,8 @@ a username & password pair. This is necessary for ``pixiv``, ``nijie``, and ``seiga`` and optional for ``aryion``, ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``, -``instagram``, ``luscious``, ``sankaku``, ``subscribestar``, ``tsumino``, -and ``twitter``. +``instagram``, ``luscious``, ``pinterest``, ``sankaku``, ``subscribestar``, +``tsumino``, and ``twitter``. You can set the necessary information in your configuration file (cf. gallery-dl.conf_) diff --git a/docs/configuration.rst b/docs/configuration.rst index daf4108e30..074125213a 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -267,17 +267,18 @@ Description * ``inkbunny`` * ``instagram`` * ``luscious`` + * ``pinterest`` * ``sankaku`` * ``subscribestar`` * ``tsumino`` * ``twitter`` - These values can also be set via the ``-u/--username`` and - ``-p/--password`` command-line options or by using a |.netrc|_ file. - (see Authentication_) + These values can also be specified via the + ``-u/--username`` and ``-p/--password`` command-line options or + by using a |.netrc|_ file. (see Authentication_) Note: The password values for ``danbooru`` and ``e621`` should be - the API keys found in your user profile, not your actual account + the API keys found in your user profile, not the actual account password. diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index dd42cf93f6..0326dada26 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -94,7 +94,7 @@ Patreon https://www.patreon.com/ Creators, Posts, User P Pawoo https://pawoo.net/ Images from Statuses, User Profiles Optional (`OAuth `__) Photobucket https://photobucket.com/ Albums, individual Images Piczel https://piczel.tv/ Folders, individual Images, User Profiles -Pinterest https://www.pinterest.com/ Boards, Pins, pin.it Links, related Pins, Sections +Pinterest https://www.pinterest.com/ Boards, Pins, pin.it Links, related Pins, Sections Optional Pixiv https://www.pixiv.net/ |pixiv-C| Required Pixnet https://www.pixnet.net/ Folders, individual Images, Sets, User Profiles Plurk https://www.plurk.com/ Posts, Timelines diff --git a/gallery_dl/extractor/pinterest.py b/gallery_dl/extractor/pinterest.py index cc89ac529d..aa1128970c 100644 --- a/gallery_dl/extractor/pinterest.py +++ b/gallery_dl/extractor/pinterest.py @@ -9,7 +9,8 @@ """Extractors for https://www.pinterest.com/""" from .common import Extractor, Message -from .. import text, exception +from .. import text, util, exception +from ..cache import cache import itertools import json @@ -28,6 +29,7 @@ def __init__(self, match): self.api = PinterestAPI(self) def items(self): + self.api.login() data = self.metadata() yield Message.Version, 1 yield Message.Directory, data @@ -98,6 +100,10 @@ class PinterestBoardExtractor(PinterestExtractor): "options": (("sections", True),), "count": 5, }), + # secret board (#1055) + ("https://www.pinterest.de/g1952849/secret/", { + "count": 2, + }), ("https://www.pinterest.com/g1952848/test/", { "exception": exception.GalleryDLException, }), @@ -230,16 +236,22 @@ class PinterestAPI(): "Accept" : "application/json, text/javascript, " "*/*, q=0.01", "Accept-Language" : "en-US,en;q=0.5", - "X-Pinterest-AppState": "active", - "X-APP-VERSION" : "b00dd49", + "Referer" : BASE_URL + "/", "X-Requested-With" : "XMLHttpRequest", + "X-APP-VERSION" : "7a20185", + "X-CSRFToken" : None, + "X-Pinterest-AppState": "active", "Origin" : BASE_URL, - "Referer" : BASE_URL + "/", } def __init__(self, extractor): self.extractor = extractor + csrf_token = util.generate_csrf_token() + self.headers = self.HEADERS.copy() + self.headers["X-CSRFToken"] = csrf_token + self.cookies = {"csrftoken": csrf_token} + def pin(self, pin_id): """Query information about a pin""" options = {"id": pin_id, "field_set_key": "detailed"} @@ -282,12 +294,45 @@ def board_related(self, board_id): options = {"board_id": board_id, "add_vase": True} return self._pagination("BoardRelatedPixieFeed", options) + def login(self): + """Login and obtain session cookies""" + username, password = self.extractor._get_auth_info() + if username: + self.cookies.update(self._login_impl(username, password)) + + @cache(maxage=180*24*3600, keyarg=1) + def _login_impl(self, username, password): + self.extractor.log.info("Logging in as %s", username) + + url = self.BASE_URL + "/resource/UserSessionResource/create/" + options = { + "username_or_email": username, + "password" : password, + } + data = {"data": json.dumps({"options": options}), "source_url": ""} + + try: + response = self.extractor.request( + url, method="POST", headers=self.headers, + cookies=self.cookies, data=data) + resource = response.json()["resource_response"] + except (exception.HttpError, ValueError, KeyError): + raise exception.AuthenticationError() + + if resource["status"] != "success": + raise exception.AuthenticationError() + return { + cookie.name: cookie.value + for cookie in response.cookies + } + def _call(self, resource, options): url = "{}/resource/{}Resource/get/".format(self.BASE_URL, resource) params = {"data": json.dumps({"options": options}), "source_url": ""} response = self.extractor.request( - url, params=params, headers=self.HEADERS, fatal=False) + url, params=params, headers=self.headers, + cookies=self.cookies, fatal=False) try: data = response.json() diff --git a/test/test_results.py b/test/test_results.py index a594032bc7..3bc950ade1 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -296,6 +296,7 @@ def wrap(obj): def setup_test_config(): name = "gallerydl" email = "gallerydl@openaliasbox.org" + email2 = "gallerydl@protonmail.com" config.clear() config.set(("cache",), "file", None) @@ -307,6 +308,7 @@ def setup_test_config(): config.set(("extractor", "nijie") , "username", email) config.set(("extractor", "seiga") , "username", email) + config.set(("extractor", "pinterest") , "username", email2) config.set(("extractor", "newgrounds"), "username", "d1618111") config.set(("extractor", "newgrounds"), "password", "d1618111")