[2ch] init

docs/supportedsites.md

@@ -13,6 +13,12 @@ Consider all listed sites to potentially be NSFW.
 </tr>
 </thead>
 <tbody valign="top">
+<tr>
+    <td>2ch</td>
+    <td>https://2ch.hk/</td>
+    <td>Boards, Threads</td>
+    <td></td>
+</tr>
 <tr>
     <td>2chen</td>
     <td>https://sturdychan.help/</td>

gallery_dl/extractor/2ch.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://2ch.hk/"""
+
+from .common import Extractor, Message
+from .. import text, util
+
+
+class _2chThreadExtractor(Extractor):
+    """Extractor for 2ch threads"""
+    category = "2ch"
+    subcategory = "thread"
+    root = "https://2ch.hk"
+    directory_fmt = ("{category}", "{board}", "{thread} {title}")
+    filename_fmt = "{tim}{filename:? //}.{extension}"
+    archive_fmt = "{board}_{thread}_{tim}"
+    pattern = r"(?:https?://)?2ch\.hk/([^/?#]+)/res/(\d+)"
+    example = "https://2ch.hk/a/res/12345.html"
+
+    def __init__(self, match):
+        Extractor.__init__(self, match)
+        self.board, self.thread = match.groups()
+
+    def items(self):
+        url = "{}/{}/res/{}.json".format(self.root, self.board, self.thread)
+        posts = self.request(url).json()["threads"][0]["posts"]
+
+        op = posts[0]
+        title = op.get("subject") or text.remove_html(op["comment"])
+
+        thread = {
+            "board" : self.board,
+            "thread": self.thread,
+            "title" : text.unescape(title)[:50],
+        }
+
+        yield Message.Directory, thread
+        for post in posts:
+            files = post.get("files")
+            if files:
+                post["post_name"] = post["name"]
+                post["date"] = text.parse_timestamp(post["timestamp"])
+                del post["files"]
+                del post["name"]
+
+                for file in files:
+                    file.update(thread)
+                    file.update(post)
+
+                    file["filename"] = file["fullname"].rpartition(".")[0]
+                    file["tim"], _, file["extension"] = \
+                        file["name"].rpartition(".")
+
+                    yield Message.Url, self.root + file["path"], file
+
+
+class _2chBoardExtractor(Extractor):
+    """Extractor for 2ch boards"""
+    category = "2ch"
+    subcategory = "board"
+    root = "https://2ch.hk"
+    pattern = r"(?:https?://)?2ch\.hk/([^/?#]+)/?$"
+    example = "https://2ch.hk/a/"
+
+    def __init__(self, match):
+        Extractor.__init__(self, match)
+        self.board = match.group(1)
+
+    def items(self):
+        # index page
+        url = "{}/{}/index.json".format(self.root, self.board)
+        index = self.request(url).json()
+        index["_extractor"] = _2chThreadExtractor
+        for thread in index["threads"]:
+            url = "{}/{}/res/{}.html".format(
+                self.root, self.board, thread["thread_num"])
+            yield Message.Queue, url, index
+
+        # pages 1..n
+        for n in util.advance(index["pages"], 1):
+            url = "{}/{}/{}.json".format(self.root, self.board, n)
+            page = self.request(url).json()
+            page["_extractor"] = _2chThreadExtractor
+            for thread in page["threads"]:
+                url = "{}/{}/res/{}.html".format(
+                    self.root, self.board, thread["thread_num"])
+                yield Message.Queue, url, page

gallery_dl/extractor/__init__.py

@@ -10,6 +10,7 @@
 import re
 
 modules = [
+    "2ch",
     "2chan",
     "2chen",
     "35photo",

test/results/2ch.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+gallery_dl = __import__("gallery_dl.extractor.2ch")
+_2ch = getattr(gallery_dl.extractor, "2ch")
+
+
+__tests__ = (
+{
+    "#url"     : "https://2ch.hk/a/res/6202876.html",
+    "#category": ("", "2ch", "thread"),
+    "#class"   : _2ch._2chThreadExtractor,
+    "#pattern" : r"https://2ch\.hk/a/src/6202876/\d+\.\w+",
+    "#count"   : range(450, 1000),
+
+    "banned"   : 0,
+    "board"    : "a",
+    "closed"   : 0,
+    "comment"  : str,
+    "date"     : "type:datetime",
+    "displayname": str,
+    "email"    : "",
+    "endless"  : 1,
+    "extension": str,
+    "filename" : str,
+    "fullname" : str,
+    "height"   : int,
+    "lasthit"  : 1705273977,
+    "md5"      : r"re:[0-9a-f]{32}",
+    "name"     : r"re:\d+\.\w+",
+    "num"      : int,
+    "number"   : range(1, 1000),
+    "op"       : 0,
+    "parent"   : int,
+    "path"     : r"re:/a/src/6202876/\d+\.\w+",
+    "post_name": "Аноним",
+    "size"     : int,
+    "sticky"   : 0,
+    "subject"  : str,
+    "thread"   : "6202876",
+    "thumbnail": str,
+    "tim"      : r"re:\d+",
+    "timestamp": int,
+    "title"    : "MP4/WEBM",
+    "tn_height": int,
+    "tn_width" : int,
+    "trip"     : "",
+    "type"     : int,
+    "views"    : int,
+    "width"    : int,
+},
+
+{
+    "#url"     : "https://2ch.hk/a/",
+    "#category": ("", "2ch", "board"),
+    "#class"   : _2ch._2chBoardExtractor,
+    "#pattern" : _2ch._2chThreadExtractor.pattern,
+    "#count"   : range(200, 300),
+},
+
+)

test/results/coomerparty.py

@@ -8,12 +8,19 @@
 
 
 __tests__ = (
+{
+    "#url"     : "https://coomer.su/onlyfans/user/alinity/post/125962203",
+    "#comment" : "coomer (#2100)",
+    "#category": ("", "coomerparty", "onlyfans"),
+    "#class"   : kemonoparty.KemonopartyPostExtractor,
+    "#urls"    : "https://coomer.su/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8.jpg",
+},
+
 {
     "#url"     : "https://coomer.party/onlyfans/user/alinity/post/125962203",
-    "#comment" : "coomer.party (#2100)",
     "#category": ("", "coomerparty", "onlyfans"),
     "#class"   : kemonoparty.KemonopartyPostExtractor,
-    "#pattern" : r"https://coomer\.party/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8\.jpg",
+    "#urls"    : "https://coomer.party/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8.jpg",
 },
 
 )

test/results/kemonoparty.py

@@ -297,6 +297,7 @@
     "#category": ("", "kemonoparty", "favorite"),
     "#class"   : kemonoparty.KemonopartyFavoriteExtractor,
     "#pattern" : kemonoparty.KemonopartyUserExtractor.pattern,
+    "#auth"    : True,
     "#count"   : 3,
     "#sha1_url": "f4b5b796979bcba824af84206578c79101c7f0e1",
 },
@@ -306,6 +307,7 @@
     "#category": ("", "kemonoparty", "favorite"),
     "#class"   : kemonoparty.KemonopartyFavoriteExtractor,
     "#pattern" : kemonoparty.KemonopartyPostExtractor.pattern,
+    "#auth"    : True,
     "#count"   : 3,
     "#sha1_url": "ecfccf5f0d50b8d14caa7bbdcf071de5c1e5b90f",
 },
@@ -315,6 +317,7 @@
     "#category": ("", "kemonoparty", "favorite"),
     "#class"   : kemonoparty.KemonopartyFavoriteExtractor,
     "#pattern" : kemonoparty.KemonopartyPostExtractor.pattern,
+    "#auth"    : True,
     "#count"   : 3,
     "#sha1_url": "4be8e84cb384a907a8e7997baaf6287b451783b5",
 },

test/test_results.py

@@ -38,6 +38,15 @@
     },
 }
 
+AUTH = {
+    "pixiv",
+    "nijie",
+    "horne",
+    "seiga",
+    "instagram",
+    "twitter",
+}
+
 
 class TestExtractorResults(unittest.TestCase):
 
@@ -76,6 +85,18 @@ def _run_test(self, result):
                 for key, value in result["#options"].items():
                     key = key.split(".")
                     config.set(key[:-1], key[-1], value)
+
+            requires_auth = result.get("#auth")
+            if requires_auth is None:
+                requires_auth = (result["#category"][1] in AUTH)
+            if requires_auth:
+                extr = result["#class"].from_url(result["#url"])
+                if not any(extr.config(key) for key in (
+                        "username", "cookies", "api-key", "client-id")):
+                    msg = "no auth"
+                    self._skipped.append((result["#url"], msg))
+                    self.skipTest(msg)
+
             if "#range" in result:
                 config.set((), "image-range"  , result["#range"])
                 config.set((), "chapter-range", result["#range"])
@@ -371,7 +392,7 @@ def load_test_config():
     except FileNotFoundError:
         pass
     except Exception as exc:
-        print("Error when loading {}: {}: {}".format(
+        sys.exit("Error when loading {}: {}: {}".format(
             path, exc.__class__.__name__, exc))
 
 
@@ -422,7 +443,7 @@ def test(self):
         setattr(TestExtractorResults, method.__name__, method)
 
 
-load_test_config()
 generate_tests()
 if __name__ == "__main__":
+    load_test_config()
     unittest.main(warnings="ignore")