From f1cab23724708ffc911e1fa7f3ca6be918d9bf7d Mon Sep 17 00:00:00 2001 From: Alice <38675581+alice945@users.noreply.github.com> Date: Mon, 28 Feb 2022 13:42:15 -0800 Subject: [PATCH] [skeb] add 'sent-requests' option (#2322) (#2330) * Update skeb.py * Update configuration.rst * flake8 --- docs/configuration.rst | 10 ++++++++++ gallery_dl/extractor/skeb.py | 26 ++++++++++++++++++-------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 211193c164..4fc5f93b21 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1979,6 +1979,16 @@ Description Download videos. +extractor.skeb.sent-requests +------------------------- +Type + ``bool`` +Default + ``false`` +Description + Download sent requests. + + extractor.skeb.thumbnails ------------------------- Type diff --git a/gallery_dl/extractor/skeb.py b/gallery_dl/extractor/skeb.py index 2c806ad6d3..24e39f91c2 100644 --- a/gallery_dl/extractor/skeb.py +++ b/gallery_dl/extractor/skeb.py @@ -22,10 +22,11 @@ def __init__(self, match): Extractor.__init__(self, match) self.user_name = match.group(1) self.thumbnails = self.config("thumbnails", False) + self.sent_requests = self.config("sent-requests", False) def items(self): - for post_num in self.posts(): - response, post = self._get_post_data(post_num) + for user_name, post_num in self.posts(): + response, post = self._get_post_data(user_name, post_num) yield Message.Directory, post for data in self._get_urls_from_post(response, post): url = data["file_url"] @@ -38,24 +39,33 @@ def _pagination(self): url = "{}/api/users/{}/works".format(self.root, self.user_name) params = {"role": "creator", "sort": "date", "offset": 0} headers = {"Referer": self.root, "Authorization": "Bearer null"} + do_requests = self.sent_requests while True: posts = self.request(url, params=params, headers=headers).json() for post in posts: post_num = post["path"].rpartition("/")[2] + user_name = post["path"].split("/")[1][1:] if post["private"]: - self.log.debug("Skipping %s (private)", post_num) + self.log.debug("Skipping @%s/%s (private)", + user_name, post_num) continue - yield post_num + yield user_name, post_num if len(posts) < 30: - return + if do_requests: + params["offset"] = 0 + params['role'] = "client" + do_requests = False + continue + else: + return params["offset"] += 30 - def _get_post_data(self, post_num): + def _get_post_data(self, user_name, post_num): url = "{}/api/users/{}/works/{}".format( - self.root, self.user_name, post_num) + self.root, user_name, post_num) headers = {"Referer": self.root, "Authorization": "Bearer null"} resp = self.request(url, headers=headers).json() creator = resp["creator"] @@ -130,7 +140,7 @@ def __init__(self, match): self.post_num = match.group(2) def posts(self): - return (self.post_num,) + return (self.user_name, self.post_num,) class SkebUserExtractor(SkebExtractor):