Skip to content

Commit

Permalink
[idolcomplex] fix metadata extraction
Browse files Browse the repository at this point in the history
- replace legacy 'id' vales with alphanumeric ones, since the former are
  no longer available
- approximate 'vote_average', since the real value is no longer
  available
- fix 'vote_count'
  • Loading branch information
mikf committed Mar 22, 2024
1 parent 77ab015 commit 32262a0
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 12 deletions.
11 changes: 5 additions & 6 deletions gallery_dl/extractor/idolcomplex.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,8 @@ def _extract_post(self, post_id):
page = self.request(url, retries=10).text
extr = text.extract_from(page)

pid_alnum = extr('/posts/', '"')
vavg = extr('itemprop="ratingValue">', "<")
vcnt = extr('itemprop="reviewCount">', "<")
vavg = extr('id="rating"', "</ul>")
vcnt = extr('>Votes</strong>:', "<")
pid = extr(">Post ID:", "<")
created = extr(' title="', '"')

Expand All @@ -120,10 +119,10 @@ def _extract_post(self, post_id):
rating = extr(">Rating:", "<br")

data = {
"id" : text.parse_int(pid),
"id_alnum" : pid_alnum,
"id" : pid.strip(),
"md5" : file_url.rpartition("/")[2].partition(".")[0],
"vote_average": text.parse_float(vavg),
"vote_average": (1.0 * vavg.count('class="star-full"') +
0.5 * vavg.count('class="star-half"')),
"vote_count" : text.parse_int(vcnt),
"created_at" : created,
"date" : text.parse_datetime(
Expand Down
11 changes: 5 additions & 6 deletions test/results/idolcomplex.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
},

{
"#url" : "https://idol.sankakucomplex.com/pools/show/145",
"#url" : "https://idol.sankakucomplex.com/en/pools/e9PMwnwRBK3",
"#category": ("booru", "idolcomplex", "pool"),
"#class" : idolcomplex.IdolcomplexPoolExtractor,
"#count" : 3,
Expand Down Expand Up @@ -72,16 +72,16 @@
"file_url" : r"re:https://i[sv]\.sankakucomplex\.com/data/50/9e/509eccbba54a43cea6b275a65b93c51d\.jpg\?",
"filename" : "509eccbba54a43cea6b275a65b93c51d",
"height" : 683,
"id" : 694215,
"id_alnum" : "vkr36qdOaZ4",
"id" : "vkr36qdOaZ4", # legacy ID: 694215
"md5" : "509eccbba54a43cea6b275a65b93c51d",
"rating" : "g",
"tags" : "lyumos the_witcher shani_(the_witcher) 1girl green_eyes non-asian redhead waistcoat wreath cosplay 3:2_aspect_ratio",
"tags_character": "shani_(the_witcher)",
"tags_copyright": "the_witcher",
"tags_general" : "1girl green_eyes non-asian redhead waistcoat wreath",
"tags_genre" : "cosplay",
"tags_idol" : "lyumos",
"tags_medium" : "cosplay 3:2_aspect_ratio",
"tags_medium" : "3:2_aspect_ratio",
"vote_average" : range(4, 5),
"vote_count" : range(25, 40),
"width" : 1024,
Expand Down Expand Up @@ -111,8 +111,7 @@
"#class" : idolcomplex.IdolcomplexPostExtractor,
"#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd",

"id" : 694215,
"id_alnum" : "vkr36qdOaZ4",
"id" : "vkr36qdOaZ4", # legacy ID: 694215
"tags_character": "shani_(the_witcher)",
"tags_copyright": "the_witcher",
"tags_idol" : str,
Expand Down

0 comments on commit 32262a0

Please sign in to comment.