Add new parameters for yaml-yugi-ko proposal, with CSV load for Rush

.github/workflows/merge.yaml

@@ -68,6 +68,8 @@ jobs:
           git rm --ignore-unmatch yugipedia*.json yugipedia*.yaml
           python3 ../../src/main_rush.py \
             ../../../yaml-yugipedia/wikitext/rush \
+            --ko-override ../../../yaml-yugi-ko/rush-override.csv \
+            --ko-prerelease ../../../yaml-yugi-ko/rush-prerelease.csv \
             --aggregate ../../../aggregate/rush.json
       - name: Transform (TCG Speed Duel Skills)
         working-directory: yaml-yugi/data/tcg-speed-skill

src/job_ocgtcg.py

@@ -211,14 +211,16 @@ def job(
     tcg_vector: Optional[Dict[str, int]] = None,
     ocg_vector: Optional[Dict[str, int]] = None,
     ko_file: Optional[str] = None,
-    ko_csv: Optional[str] = None,
+    ko_official_csv: Optional[str] = None,
+    ko_override_csv: Optional[str] = None,
+    ko_prerelease_csv: Optional[str] = None,
     return_results=False
 ) -> Optional[List[Dict[str, Any]]]:
     yaml = YAML()
     yaml.width = sys.maxsize
     assignments = load_assignments(yaml, assignment_file) if assignment_file else None
     ko_overrides = load_ko_overrides(ko_file) if ko_file else None
-    ko_official = load_ko_official(ko_csv) if ko_csv else None
+    ko_official = load_ko_official(ko_official_csv) if ko_official_csv else None
     results = []
     for i, filename in enumerate(filenames):
         filepath = os.path.join(wikitext_dir, filename)

src/job_rush.py

@@ -1,5 +1,6 @@
-# SPDX-FileCopyrightText: © 2022 Kevin Lu
+# SPDX-FileCopyrightText: © 2022–2023 Kevin Lu
 # SPDX-Licence-Identifier: AGPL-3.0-or-later
+from csv import DictReader
 import logging
 import os
 import sys
@@ -53,13 +54,30 @@ def write_output(yaml: YAML, logger: logging.Logger, document: Dict[str, Any]) -
     write(document, basename, yaml, logger)
 
 
+def load_ko_csv(key: str, filename: Optional[str]) -> Dict[int, Dict[str, str]] | None:
+    if not filename:
+        return
+    with open(filename, encoding="utf8") as f:
+        reader = DictReader(f)
+        return {
+            int(row[key]): row
+            for row in reader
+        }
+
+
 def job(
     wikitext_dir: str,
     filenames: List[str],
+    ko_official_csv: Optional[str] = None,
+    ko_override_csv: Optional[str] = None,
+    ko_prerelease_csv: Optional[str] = None,
     return_results=False
 ) -> Optional[List[Dict[str, Any]]]:
     yaml = YAML()
     yaml.width = sys.maxsize
+    ko_official = load_ko_csv("konami_id", ko_official_csv)
+    ko_override = load_ko_csv("konami_id", ko_override_csv)
+    ko_prerelease = load_ko_csv("yugipedia_page_id", ko_prerelease_csv)
     results = []
     for i, filename in enumerate(filenames):
         filepath = os.path.join(wikitext_dir, filename)

src/main_ocgtcg.py

@@ -49,8 +49,17 @@ def main() -> None:
         if os.path.isfile(os.path.join(args.wikitext_directory, filename))
     ]
 
+    arguments = (
+        args.zh_CN,
+        args.assignments,
+        tcg,
+        ocg,
+        args.ko,
+        args.ko_official,
+        args.aggregate is not None,
+    )
     if processes == 1:
-        cards = job(args.wikitext_directory, files, args.zh_CN, args.assignments, tcg, ocg, args.ko, args.ko_official, args.aggregate is not None)
+        cards = job(args.wikitext_directory, files, *arguments)
     else:
         size = math.ceil(len(files) / processes)
         partitions = [files[i:i+size] for i in range(0, len(files), size)]
@@ -59,8 +68,7 @@ def main() -> None:
         from multiprocessing import Pool
         with Pool(processes) as pool:
             jobs = [
-                pool.apply_async(job, (args.wikitext_directory, partition,
-                                       args.zh_CN, args.assignments, tcg, ocg, args.ko, args.ko_official, args.aggregate is not None))
+                pool.apply_async(job, (args.wikitext_directory, partition, *arguments))
                 for partition in partitions
             ]
             for result in jobs:

src/main_rush.py

@@ -11,6 +11,9 @@
 
 parser = ArgumentParser()
 parser.add_argument("wikitext_directory", help="yaml-yugipedia card texts")
+parser.add_argument("--ko-official", help="yaml-yugi-ko official database CSV")
+parser.add_argument("--ko-override", help="yaml-yugi-ko rush-override.csv")
+parser.add_argument("--ko-prerelease", help="yaml-yugi-ko rush-prerelease.csv")
 parser.add_argument("--generate-schema", action="store_true", help="output generated JSON schema file")
 parser.add_argument("--processes", type=int, default=0, help="number of worker processes, default ncpu")
 parser.add_argument("--aggregate", help="output aggregate JSON file")
@@ -32,8 +35,14 @@ def main() -> None:
         if os.path.isfile(os.path.join(args.wikitext_directory, filename))
     ]
 
+    arguments = (
+        args.ko_official,
+        args.ko_override,
+        args.ko_prerelease,
+        args.aggregate is not None,
+    )
     if processes == 1:
-        cards = job(args.wikitext_directory, files, args.aggregate is not None)
+        cards = job(args.wikitext_directory, files, *arguments)
     else:
         size = math.ceil(len(files) / processes)
         partitions = [files[i:i+size] for i in range(0, len(files), size)]
@@ -42,7 +51,7 @@ def main() -> None:
         from multiprocessing import Pool
         with Pool(processes) as pool:
             jobs = [
-                pool.apply_async(job, (args.wikitext_directory, partition, args.aggregate is not None))
+                pool.apply_async(job, (args.wikitext_directory, partition, *arguments))
                 for partition in partitions
             ]
             for result in jobs: