Skip to content

Commit

Permalink
[newgrounds] update and fix pagination (#2456)
Browse files Browse the repository at this point in the history
  • Loading branch information
mikf committed Apr 7, 2022
1 parent d78a2c7 commit c176897
Showing 1 changed file with 30 additions and 20 deletions.
50 changes: 30 additions & 20 deletions gallery_dl/extractor/newgrounds.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,16 +236,20 @@ def _video_fallback(formats):
yield fmt[1][0]["src"]

def _pagination(self, kind):
root = self.user_root
url = "{}/{}".format(self.user_root, kind)
params = {
"page": 1,
"isAjaxRequest": "1",
}
headers = {
"Accept": "application/json, text/javascript, */*; q=0.01",
"Referer": url,
"X-Requested-With": "XMLHttpRequest",
"Referer": root,
}
url = "{}/{}/page/1".format(root, kind)

while True:
with self.request(url, headers=headers, fatal=False) as response:
with self.request(
url, params=params, headers=headers,
fatal=False) as response:
try:
data = response.json()
except ValueError:
Expand All @@ -256,14 +260,17 @@ def _pagination(self, kind):
msg = ", ".join(text.unescape(e) for e in data["errors"])
raise exception.StopExtraction(msg)

for year in data["sequence"]:
for item in data["years"][str(year)]["items"]:
for year, items in data["items"].items():
for item in items:
page_url = text.extract(item, 'href="', '"')[0]
yield text.urljoin(root, page_url)
if page_url[0] == "/":
page_url = self.root + page_url
yield page_url

if not data["more"]:
more = data.get("load_more")
if not more or len(more) < 8:
return
url = text.urljoin(root, data["more"])
params["page"] += 1


class NewgroundsImageExtractor(NewgroundsExtractor):
Expand Down Expand Up @@ -301,9 +308,9 @@ class NewgroundsImageExtractor(NewgroundsExtractor):
"count": 2,
}),
# "adult" rated (#2456)
("https://www.newgrounds.com/art/view/bdoneart/vampire-booty", {
("https://www.newgrounds.com/art/view/kekiiro/red", {
"options": (("username", None),),
"count": 0,
"count": 1,
}),
)

Expand Down Expand Up @@ -374,7 +381,7 @@ class NewgroundsMediaExtractor(NewgroundsExtractor):
# "adult" rated (#2456)
("https://www.newgrounds.com/portal/view/717744", {
"options": (("username", None),),
"count": 0,
"count": 1,
}),
)

Expand Down Expand Up @@ -470,25 +477,28 @@ def posts(self):
)

def _pagination(self, kind):
num = 1
url = "{}/favorites/{}".format(self.user_root, kind)
params = {
"page": 1,
"isAjaxRequest": "1",
}
headers = {
"Accept": "application/json, text/javascript, */*; q=0.01",
"Referer": url,
"X-Requested-With": "XMLHttpRequest",
"Referer": self.user_root,
}

while True:
url = "{}/favorites/{}/{}".format(self.user_root, kind, num)
response = self.request(url, headers=headers)
response = self.request(url, params=params, headers=headers)
if response.history:
return

favs = self._extract_favorites(response.text)
data = response.json()
favs = self._extract_favorites(data.get("component") or "")
yield from favs

if len(favs) < 24:
return
num += 1
params["page"] += 1

def _extract_favorites(self, page):
return [
Expand Down

0 comments on commit c176897

Please sign in to comment.