Skip to content

Commit

Permalink
[nijie] improvements and fixes (#423)
Browse files Browse the repository at this point in the history 
- ignore unavailable image pages
- more metadata fields: artist_name, date, tags
- rename 'index' to 'num'
- improved code structure
  • Loading branch information
@mikf
mikf committed Sep 26, 2019
1 parent d1ea08c commit 20eb6c4
Showing 1 changed file with 87 additions and 69 deletions.
156 changes: 87 additions & 69 deletions gallery_dl/extractor/nijie.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,15 @@
from ..cache import cache


BASE_PATTERN = r"(?:https?://)?(?:www\.)?nijie\.info"


class NijieExtractor(AsynchronousMixin, Extractor):
"""Base class for nijie extractors"""
category = "nijie"
directory_fmt = ("{category}", "{user_id}")
filename_fmt = "{category}_{artist_id}_{image_id}_p{index:>02}.{extension}"
archive_fmt = "{image_id}_{index}"
filename_fmt = "{category}_{artist_id}_{image_id}_p{num:>02}.{extension}"
archive_fmt = "{image_id}_{num}"
cookiedomain = "nijie.info"
cookienames = ("nemail", "nlogin")
root = "https://nijie.info"
Expand All @@ -32,56 +35,64 @@ def __init__(self, match):

def items(self):
self.login()
data = self.get_job_metadata()

metadata = self.metadata()
yield Message.Version, 1
yield Message.Directory, data

for image_id in self.get_image_ids():
for image_url, image_data in self.get_image_data(image_id):
image_data.update(data)
if not image_data["extension"]:
image_data["extension"] = "jpg"
yield Message.Url, image_url, image_data
for image_id in self.image_ids():

response = self.request(self.view_url + image_id, fatal=False)
if response.status_code >= 400:
continue
page = response.text

def get_job_metadata(self):
data = self._extract_data(page)
data.update(metadata)
data["image_id"] = text.parse_int(image_id)
yield Message.Directory, data

for image in self._extract_images(page):
image.update(data)
if not image["extension"]:
image["extension"] = "jpg"
yield Message.Url, image["url"], image

def metadata(self):
"""Collect metadata for extractor-job"""
return {"user_id": text.parse_int(self.user_id)}

def get_image_ids(self):
def image_ids(self):
"""Collect all relevant image-ids"""

def get_image_data(self, image_id):
"""Get URL and metadata for images specified by 'image_id'"""
page = self.request(self.view_url + image_id).text
return self.extract_image_data(page, image_id)

def extract_image_data(self, page, image_id):
"""Get URL and metadata for images from 'page'"""
title, pos = text.extract(
page, '<meta property="og:title" content="', '"')
description, pos = text.extract(
page, '<meta property="og:description" content="', '"', pos)
artist_id, pos = text.extract(
page, '"sameAs": "https://nijie.info/members.php?id=', '"', pos)
images = list(text.extract_iter(
page, '<a href="./view_popup.php', '</a>', pos))

title = title.rpartition("|")[0].strip()
image_id = text.parse_int(image_id)
artist_id = text.parse_int(artist_id)

for index, image in enumerate(images):
return ()

@staticmethod
def _extract_data(page):
"""Extract image metadata from 'page'"""
extr = text.extract_from(page)
keywords = text.unescape(extr(
'name="keywords" content="', '"')).split(",")
return {
"title" : keywords[0].strip(),
"description": text.unescape(extr(
'"og:description" content="', '"')),
"date" : text.parse_datetime(extr(
'"datePublished": "', '"')[:-4] + "+0900",
"%a %d %b %Y %I:%M:%S %p%z"),
"artist_id" : text.parse_int(extr(
'"sameAs": "https://nijie.info/members.php?id=', '"')),
"artist_name": keywords[1],
"tags" : keywords[2:-1],
}

@staticmethod
def _extract_images(page):
"""Extract image URLs from 'page'"""
images = text.extract_iter(page, '<a href="./view_popup.php', '</a>')
for num, image in enumerate(images):
url = "https:" + text.extract(image, 'src="', '"')[0]
url = url.replace("/__rs_l120x120/", "/", 1)

yield url, text.nameext_from_url(url, {
"index": index,
"count": len(images),
"title": title,
"description": description,
"image_id": image_id,
"artist_id": artist_id,
url = url.replace("/__rs_l120x120/", "/")
yield text.nameext_from_url(url, {
"num": num,
"url": url,
})

def login(self):
Expand Down Expand Up @@ -117,82 +128,89 @@ def _pagination(self, path):
class NijieUserExtractor(NijieExtractor):
"""Extractor for works of a nijie-user"""
subcategory = "user"
pattern = (r"(?:https?://)?(?:www\.)?nijie\.info"
r"/members(?:_illust)?\.php\?id=(\d+)")
pattern = BASE_PATTERN + r"/members(?:_illust)?\.php\?id=(\d+)"
test = (
("https://nijie.info/members_illust.php?id=44", {
"url": "66c4ff94c6e77c0765dd88f2d8c663055fda573e",
"keyword": "d629c69e3172db1d7e026145e8eb640ac31ac16a",
"keyword": {
"artist_id": 44,
"artist_name": "ED",
"date": "type:datetime",
"description": str,
"extension": "jpg",
"filename": str,
"image_id": int,
"num": int,
"tags": list,
"title": str,
"url": r"re:https://pic.nijie.net/\d+/nijie_picture/.*jpg$",
"user_id": 44,
},
}),
("https://nijie.info/members_illust.php?id=43", {
"exception": exception.NotFoundError,
}),
("https://nijie.info/members.php?id=44"),
)

def get_image_ids(self):
def image_ids(self):
return self._pagination("members_illust")


class NijieDoujinExtractor(NijieExtractor):
"""Extractor for doujin entries of a nijie-user"""
subcategory = "doujin"
pattern = (r"(?:https?://)?(?:www\.)?nijie\.info/"
r"members_dojin\.php\?id=(\d+)")
pattern = BASE_PATTERN + r"/members_dojin\.php\?id=(\d+)"
test = ("https://nijie.info/members_dojin.php?id=6782", {
"count": ">= 18",
})

def get_image_ids(self):
def image_ids(self):
return self._pagination("members_dojin")


class NijieFavoriteExtractor(NijieExtractor):
"""Extractor for all favorites/bookmarks of a nijie-user"""
subcategory = "favorite"
directory_fmt = ("{category}", "bookmarks", "{user_id}")
archive_fmt = "f_{user_id}_{image_id}_{index}"
pattern = (r"(?:https?://)?(?:www\.)?nijie\.info"
r"/user_like_illust_view\.php\?id=(\d+)")
archive_fmt = "f_{user_id}_{image_id}_{num}"
pattern = BASE_PATTERN + r"/user_like_illust_view\.php\?id=(\d+)"
test = ("https://nijie.info/user_like_illust_view.php?id=44", {
"count": ">= 16",
})

def get_image_ids(self):
def image_ids(self):
return self._pagination("user_like_illust_view")


class NijieImageExtractor(NijieExtractor):
"""Extractor for a work/image from nijie.info"""
subcategory = "image"
pattern = (r"(?:https?://)?(?:www\.)?nijie\.info"
r"/view(?:_popup)?\.php\?id=(\d+)")
pattern = BASE_PATTERN + r"/view(?:_popup)?\.php\?id=(\d+)"
test = (
("https://nijie.info/view.php?id=70720", {
"url": "5497f897311397dafa188521258624346a0af2a3",
"keyword": "408393d010307c76d52cbd0a4368d6d357805aea",
"keyword": "7b7cfc89fa59652a100f94d4b765130d93281c66",
"content": "d85e3ea896ed5e4da0bca2390ad310a4df716ca6",
}),
("https://nijie.info/view.php?id=70724", {
"exception": exception.NotFoundError,
"count": 0,
}),
("https://nijie.info/view_popup.php?id=70720"),
)

def __init__(self, match):
NijieExtractor.__init__(self, match)
self.image_id = match.group(1)
self.page = ""

def get_job_metadata(self):
self.page = self.request(
self.view_url + self.image_id, notfound="image").text
self.user_id = text.extract(
self.page, '"sameAs": "https://nijie.info/members.php?id=', '"')[0]
return NijieExtractor.get_job_metadata(self)
def metadata(self):
return {}

def get_image_ids(self):
def image_ids(self):
return (self.image_id,)

def get_image_data(self, _):
return self.extract_image_data(self.page, self.image_id)
@staticmethod
def _extract_data(page):
data = NijieExtractor._extract_data(page)
data["user_id"] = data["artist_id"]
return data

0 comments on commit 20eb6c4

Please sign in to comment.