diff --git a/docs/taxonomy/141-noncard-census.py b/docs/taxonomy/141-noncard-census.py new file mode 100644 index 00000000..c65caf9a --- /dev/null +++ b/docs/taxonomy/141-noncard-census.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +""" +#141 — non-card taxonomy node census (read-only, reproducible). + +Counterpart to the #141 issue ("Iterate on non-card taxonomy nodes + adapt +GPT-4 enrichment script"). Grounds the proposal in the ACTUAL current state +of the taxonomy CSVs (not the 2-year-old issue text) — same reproducibility +discipline as #600/#606 (read-only, deterministic, 0 write under Cards/). + +WHAT IT MEASURES + 1. Non-card node count per dataset (rows where the card flag is empty = + family/subfamily/subsubfamily headers + order groupings, not leaf cards). + 2. Text-enrichment coverage on non-card nodes: description/example/title + x {fr, en, ru, pt, es, ar, zh, fa} — the original #141 scope item 3. + 3. Cross-reference coverage: crossLink_* (8 AIF relationship types) + + AIF_skos* (4 SKOS semantic mappings) — the cross-reference target, + schema-ready columns whose fill drives #130 (OWL) + #136 (2sxc). + +Re-run anytime from repo root: python docs/taxonomy/141-noncard-census.py +""" +import csv +from collections import Counter + +LANGS = ["fr", "en", "ru", "pt", "es", "ar", "zh", "fa"] + +# (dataset, path, card_flag_col, [(stem_fr, stem_en)], extra_text_cols) +# text stems = fields whose _ variants carry enrichment content. +DATASETS = [ + ("Fallacies", "Cards/Fallacies/Argumentum Fallacies - Taxonomy.csv", "carte", + [("desc", "desc"), ("example", "example")], ["text", "title"], LANGS), + ("Virtues", "Cards/Fallacies/Argumentum Virtues - Taxonomy.csv", "card", + [("description", "description"), ("remark", "remark")], ["title"], LANGS), +] + +CROSSLINK_PREFIXES = ["crossLink_", "AIF_skos"] + + +def load(path): + with open(path, encoding="utf-8-sig", newline="") as f: + return list(csv.DictReader(f)) + + +def pct(n, d): + return 0 if not d else 100.0 * n / d + + +def text_coverage(rows, stems, extra, langs): + """Return list of (field, filled, total, pct) for every _ + extras.""" + out = [] + cols = rows[0].keys() if rows else [] + for stem in [s[1] for s in stems]: # en-stem is the lang-suffixed form + for lang in langs: + col = "%s_%s" % (stem, lang) + if col in cols: + filled = sum(1 for r in rows if (r.get(col) or "").strip()) + out.append((col, filled, len(rows))) + return out + + +def crosslink_coverage(rows): + """crossLink_* + AIF_skos* fill, split by prefix.""" + cols = rows[0].keys() if rows else [] + by_prefix = {} + for col in cols: + for pre in CROSSLINK_PREFIXES: + if col.startswith(pre): + filled = sum(1 for r in rows if (r.get(col) or "").strip()) + by_prefix.setdefault(pre, []).append((col, filled, len(rows))) + return by_prefix + + +def report(name, path, card_col, stems, extra, langs): + try: + rows = load(path) + except FileNotFoundError: + print("== %s: %s NOT FOUND, skip ==\n" % (name, path)) + return + cards = [r for r in rows if (r.get(card_col) or "").strip()] + noncards = [r for r in rows if not (r.get(card_col) or "").strip()] + print("=" * 78) + print("== %s (%s) ==" % (name, path)) + print("=" * 78) + print(" total rows: %d | cards: %d | NON-CARD nodes: %d" % + (len(rows), len(cards), len(noncards))) + print(" card-flag '%s' values: %s" % (card_col, dict(Counter((r.get(card_col) or "").strip() for r in rows)))) + print(" depth distribution: %s" % dict(Counter(r.get("depth", "").strip() for r in noncards))) + + print("\n -- TEXT enrichment on NON-CARD nodes (original #141 scope item 3) --") + tc = text_coverage(noncards, stems, extra, langs) + groups = {} + for col, filled, total in tc: + stem = col.rsplit("_", 1)[0] + groups.setdefault(stem, []).append((filled, total)) + for stem, vals in sorted(groups.items()): + mf = min(v[0] for v in vals) + avg = pct(sum(v[0] for v in vals), sum(v[1] for v in vals) * len(vals) / len(vals)) if vals else 0 + mn = min(pct(f, t) for f, t in vals) + mx = max(pct(f, t) for f, t in vals) + print(" %-16s %2d langs, fill %.0f%%-%.0f%% (min %d/%d)" % + (stem + "_", len(vals), mn, mx, mf, vals[0][1])) + + print("\n -- CROSS-REFERENCES (crossLink_* + AIF_skos*, schema-ready, feeds #130/#136) --") + cl = crosslink_coverage(noncards) + for pre, items in cl.items(): + tot_filled = sum(f for _, f, _ in items) + cap = sum(t for _, _, t in items) + print(" %s : %d cols, %d/%d cells filled (%.1f%%)" % + (pre, len(items), tot_filled, cap, pct(tot_filled, cap))) + for col, filled, total in sorted(items, key=lambda x: x[1]): + print(" %-26s %4d/%d (%.0f%%)" % (col, filled, total, pct(filled, total))) + + +def main(): + print("#141 non-card taxonomy node census (read-only)") + print("Base: run from repo root. 0 write under Cards/.\n") + for ds in DATASETS: + report(*ds) + print("\n" + "=" * 78) + print("READING GUIDE:") + print(" - TEXT enrichment (desc/example/title x 8 langs) ~100%% => #141 scope item 3 DONE.") + print(" - crossLink_* + AIF_skos* ~0%% => the REAL open gap (AIF cross-reference graph).") + print(" - Schema is READY (columns exist); content is the gap. See 141-noncard-enrichment-census.md.") + print("=" * 78) + + +if __name__ == "__main__": + main() diff --git a/docs/taxonomy/141-noncard-enrichment-census.md b/docs/taxonomy/141-noncard-enrichment-census.md new file mode 100644 index 00000000..5d957ac1 --- /dev/null +++ b/docs/taxonomy/141-noncard-enrichment-census.md @@ -0,0 +1,112 @@ +# #141 — Non-card Taxonomy Node Census & Enrichment Proposal + +**Author**: po-2024 (worker) · **Date**: 2026-06-29 · **Base**: master `ba8e4a6c` +**Status**: **PROPOSAL / CENSUS** — grounds #141 in the *current* taxonomy state (the issue text is ~2 years old; the DatasetUpdater + translation PRs have moved the state considerably since). +**Scope**: docs + read-only script. **0 write under `Cards/`** (pre-tag freeze). master stays `ba8e4a6c`. +**Reproducibility**: [`141-noncard-census.py`](141-noncard-census.py) — re-run anytime: `python docs/taxonomy/141-noncard-census.py`. + +--- + +## TL;DR — #141 reframed by the census + +The issue's **original enrichment target (scope item 3: descriptions + examples + translations on non-card nodes) is DONE** — 100 % across 8 languages on both Fallacies and Virtues non-card nodes. The "GPT-4 enrichment script" it asks to adapt **is already adapted** — it is the modern `DatasetUpdater` (OpenAI SDK v2.10.0 via PR #210, gpt-5.5 primary, multi-provider). + +**The genuinely-open residue of #141 is the AIF cross-reference graph** (`crossLink_*` relationship columns + `AIF_skos*` semantic mappings): schema is **ready** (columns exist) but content is **~0 % populated** on non-card nodes. This is exactly scope item 3's "cross-references between related fallacies", and it feeds the dependencies #141 declares (#130 OWL, #136 2sxc). It overlaps the **#498 AIF scale-up** pilots, which produced the few filled cells that exist. + +→ **Recommendation**: re-scope #141 from "enrich empty descriptions" (done) to "**populate the AIF cross-reference graph**", via gpt-5.5 candidate generation + an expert validation gate (AIF/Walton mappings are specialised — not auto-writable). + +--- + +## Census (measured on `ba8e4a6c`, 2026-06-29) + +### Non-card node counts + +| Dataset | Total rows | Cards | **Non-card nodes** | Card flag | +|---|---|---|---|---| +| Fallacies | 1408 | 176 | **1232** | `carte` ∈ {1, 2} | +| Virtues | 223 | 113 | **110** | `card` = 1 | + +Non-card = family / subfamily / subsubfamily headers + order groupings (rows whose card flag is empty). Scenarii and Rules have no family hierarchy nor crossLink/AIF columns → out of #141 scope. + +### Text enrichment on non-card nodes (original #141 scope item 3) — ✅ DONE + +| Dataset | desc/description ×8 langs | example/remark ×8 langs | +|---|---|---| +| Fallacies | **100 %** (1232/1232) | 97–100 % (min 1192/1232 `example_fr`) | +| Virtues | **100 %** (110/110) | **100 %** (110/110) | + +Titles, descriptions, examples and remarks are fully populated across fr/en/ru/pt/es/ar/zh/fa. **There is no text-enrichment gap to script.** + +### Cross-references (`crossLink_*` + `AIF_skos*`) — the REAL open gap + +Format observed: `crossLink_*` holds **`decimal_path` references** to other nodes (e.g. `5.1.2.2.2`); `AIF_skos*` holds **AIF-scheme refs + SKOS mapping types** (e.g. `GeneralAcceptanceDoubt_Conflict`, `skos:broadMatch`). + +| Dataset | `crossLink_*` (8 cols) | `AIF_skos*` (4 cols) | Notes | +|---|---|---|---| +| Fallacies non-card | **15 / 9856 (0.2 %)** | 26 / 4928 (0.5 %) | AIF barely started on non-cards | +| Virtues non-card | 110 / 880 (12.5 %) | 220 / 440 (50 %) | Partial via #498 pilots | + +**Virtues detail** (asymmetry from #498): `crossLink_Opposes` 100 %, `AIF_skosDirectRef` 100 %, `AIF_skosMappingType` 100 % — but the **7 other `crossLink_*` relationship types are 0 %**. So even where #498 ran, only a slice of the relationship graph is populated. + +**Fallacies detail**: `crossLink_PredatesOn` 8, `Leverages`/`Opposes` 2, `IsRelatedTo`/`Allows`/`Denounces` 1, `Inverts`/`Mirrors` 0 — effectively unstarted on non-cards. + +**Aggregate**: ~16700 empty cross-reference cells across the two datasets where the schema is ready and waiting. + +--- + +## Why this is a proposal, not an auto-write + +1. **Pre-tag freeze** — `Cards/` is frozen (release coupled to DNN). Any cross-reference write is post-release. +2. **AIF/Walton is specialised** — the 8 `crossLink_*` verbs (`Inverts`, `Mirrors`, `Allows`, `Denounces`, `IsRelatedTo`, `Leverages`, `Opposes`, `PredatesOn`) and the AIF scheme refs encode an argumentation-theory judgement. An LLM can propose candidates but **must not be ratified as authoritative without expert/native review** (cf. project memory: *"Anti-Fab Validator: Check 'Walton scheme' = WARN"* — LLM-fabricated AIF mappings are a known failure mode). This mirrors the #192 native-ratification discipline. +3. **Schema decisions still open** — should `crossLink_*` hold one `decimal_path`, a list, or a structured `{target, note}`? The few filled cells use bare `decimal_path`, but a richer encoding may serve #136 (2sxc entities) better. + +--- + +## Proposed enrichment method (gpt-5.5 assist + expert gate) + +**Stage 0 — ground truth (done by this census).** Confirm text is complete; isolate the cross-reference gap. + +**Stage 1 — gpt-5.5 candidate generation (assist only).** For each non-card node, give the model: +- the node's FR/EN description + example (already 100 % populated — the context), +- the list of sibling/cousin nodes (same parent's children) with their `decimal_path` + label, +- the AIF scheme reference list (external, academic — Walton's argumentation schemes), +- the 8 `crossLink_*` verb definitions, +and ask it to **propose** 0–N candidate relationships as `{verb, target_decimal_path, confidence, one-line rationale}`. Use `/v1/responses` + `reasoning.effort=low` (memory `gpt55-responses-api-effort-low`); `max_output_tokens=7000`. + +**Stage 2 — dry-run + diff review (drift-free method #595).** Write candidates to a **sidecar** report (not `Cards/`). Re-runs verified cell-by-cell (no drift). + +**Stage 3 — expert/jsboige ratification gate.** A human reviews the candidate pairs per node (like the #192 `RATIFY` checklist). Only ratified pairs are written to `Cards/`, post-release, via the `DatasetUpdater` (point it at the `crossLink_*` fields with a dedicated prompt + JSON schema — the adaptation the issue asked for). + +**Stage 4 — OWL / 2sxc export.** Once ratified, the AIF graph flows into #130 (OWL/SKOS `skos:related`, `skos:broadMatch`) and #136 (2sxc entities) — the dependencies #141 declares. + +--- + +## What "adapt the GPT-4 enrichment script" means now + +The original ~2-year-old GPT-4 script's **modern counterpart already exists**: [`DatasetUpdater`](../../Generation/Converters/Argumentum.AssetConverter/DatasetUpdater) (8 C# files, 35+ prompts, OpenAI SDK v2.10.0, function-calling via manual `FunctionToolDef` + JSON schema, gpt-5.5 primary). The adaptation the issue asks for is **not** a rewrite — it is: + +1. **Add a `crossLink` prompt** (new `PromptCrossLink*User/Assistant/System` in `DatasetUpdater/Resources/`) that frames the relationship-proposal task above. +2. **Add a `crossLink` task config** in `DatasetUpdaterRootConfig.cs` (7 task configs exist; all `Enabled = false` — same gated pattern). +3. **Point it at the `crossLink_*` + `AIF_skos*` columns** with the Stage 1–2 assist-then-gate flow. + +This is **post-release work** (touches `Cards/` content + the AssetConverter project → pre-tag regression risk). The deliverable *now* is this proposal + the census script; the prompt/config adaptation is gated on (a) jsboige GO and (b) the schema decision (§"Why this is a proposal"). + +--- + +## Dependencies & overlaps + +- **#498 (AIF scale-up)** — phases 1–3 + generative pilot (branches `feat/498-aif-scaleup-phase{1,2,3}`, `docs/498-aif-closure`) produced the few filled AIF/crossLink cells. #141 cross-reference work should **resume from #498's results**, not restart. +- **#130 (OWL ontology)** — AIF graph → SKOS semantic relations. Dependency declared by #141. +- **#136 (2sxc taxonomy entities)** — AIF graph → website entity relationships. Dependency declared by #141. +- **#192** — i18n harmonisation (closed for the ratifiable subset; native-ratification residue tracked separately). Orthogonal: #192 is *labels*, #141 is *relationships*. + +--- + +## Scope of THIS PR + +- ✅ `docs/taxonomy/141-noncard-census.py` — read-only, reproducible census (0 write). +- ✅ `docs/taxonomy/141-noncard-enrichment-census.md` — this proposal. +- ✅ **0 write under `Cards/`**, **0 AssetConverter code change** (pre-tag safe). +- ✅ Base `ba8e4a6c`. + +Relates to #130, #136, #498. Reframes #141: the text-enrichment scope is complete; the cross-reference graph is the open work, gated post-release on jsboige GO + schema decision. diff --git a/tools/link-langlinks-resolve.md b/tools/link-langlinks-resolve.md new file mode 100644 index 00000000..a8771346 --- /dev/null +++ b/tools/link-langlinks-resolve.md @@ -0,0 +1,45 @@ +# `link-langlinks-resolve.py` — `link_*` candidate-URL resolver (sidecar, 0 write) + +Companion tool to [`docs/taxonomy/192-link-coverage-langlinks-probe.py`](../docs/taxonomy/192-link-coverage-langlinks-probe.py). Extends the #600/#606 coverage track from **measurement** (the probe answers *"is this cell resolvable?"*) to **resolution** (this tool answers *"what is the URL?"* and emits the candidate-fill list). + +This is **step 1 of the #600 §6 fill methodology**. It never writes under `Cards/`. + +## What it produces + +A sidecar report (stdout, or `--out `): + +``` +dataset,key,link_lang,resolved_url +fallacies,52,pt,https://pt.wikipedia.org/wiki/Fal%C3%A1cia_da_proje%C3%A7%C3%A3o_mental +fallacies,1260,ar,https://ar.wikipedia.org/wiki/... +``` + +For every node that has an `en.wikipedia.org/wiki/` `link_en` and is **missing** `link_<lang>`, it queries the MediaWiki `langlinks` API, captures the target-language title, and emits the resolved URL. The probe measured **~2919 resolvable cells (57 %)** (#600 §5.1); this tool materializes the candidate URLs behind that ceiling. + +## Usage + +```bash +# full run, fallacies, stdout +python tools/link-langlinks-resolve.py + +# strided sample of 50, virtues +python tools/link-langlinks-resolve.py 50 virtues + +# full run, fallacies, sidecar file +python tools/link-langlinks-resolve.py 0 fallacies --out tmp/link-resolve-fallacies.csv +``` + +Args: `[sample_size] [dataset] [--out path]`. `sample_size=0` = full run. Dataset ∈ {fallacies, virtues}. + +## Safety / scope + +- **0 write under `Cards/`** — sidecar only (stdout or `--out`). Pre-tag freeze respected. +- Public MediaWiki API, no key, ~0.3 s throttle, descriptive User-Agent (default urllib UA is 403-forbidden). +- **RTL/CJK homonym risk** (#600 §6.4): AR/FA/ZH resolved URLs are **candidates**, not authoritative — human spot-validation is non-optional before any write. The sample run already surfaces this (e.g. an English "Engagement" homonym resolved for a fallacy node). +- Resolves **from** `link_en` (Wikipedia URLs only). Non-Wikipedia `link_en` (rationalwiki, yourlogicalfallacyis, etc. — 433 Fallacies / 9 Virtues) are curated sources, excluded and preserved as-is (#600 §6.2). + +## Next step (post-release, gated) + +A follow-up PR consumes this sidecar: apply the candidate URLs cell-by-cell (drift-free `QUOTE_MINIMAL` + CRLF, method #595), **skip non-empty cells** (preserve curated links), then human spot-validate the ~5 % residue (AR/FA/ZH priority, ~150 cells). + +Relates to #600, #606, memory `i18n-coverage-gap-is-link-urls`. diff --git a/tools/link-langlinks-resolve.py b/tools/link-langlinks-resolve.py new file mode 100644 index 00000000..1f352c5f --- /dev/null +++ b/tools/link-langlinks-resolve.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +""" +#600/#606 follow-up — link_* langlinks RESOLVER (sidecar, 0 write under Cards/). + +Extends docs/taxonomy/192-link-coverage-langlinks-probe.py (which MEASURES the +resolvable ceiling) into the RESOLUTION step: it captures the target-language +article TITLE (the probe discards it) and builds the candidate fill URLs, +emitting a sidecar report for human spot-validation. This is step 1 of the +#600 §6 fill methodology — without ever touching Cards/. + +WHAT IT DOES + For every node that has an en.wikipedia.org/wiki/<Title> link_en and is + MISSING link_<lang>, query the MediaWiki langlinks API, capture the target + title, and emit: + <dataset>, <pk/decimal_path>, <lang>, <resolved_url> + The report goes to STDOUT by default, or to a sidecar file (--out). It is + NEVER written into Cards/ (pre-tag freeze + #600 §6 human-validation gate). + +WHY A SEPARATE TOOL (not just run the probe) + The probe returns {set of lang codes} -> answers "is it resolvable?". + This tool returns {lang -> target_title} -> answers "what is the URL?" and + produces the candidate-fill list. The measured ceiling (2919 cells, #600 §5.1) + is the budget; this tool materializes the candidate URLs behind that budget. + +SAFETY + - 0 write under Cards/. Sidecar only (stdout or --out path). + - Public MediaWiki API, no key, rate-limited (API_DELAY), descriptive UA. + - RTL/CJK homonym risk (#600 §6.4): AR/FA/ZH resolved URLs are CANDIDATES, + not authoritative — human spot-validation is non-optional before any write. + +USAGE (from repo root) + python tools/link-langlinks-resolve.py # full, fallacies, stdout + python tools/link-langlinks-resolve.py 50 virtues # strided sample of 50, virtues + python tools/link-langlinks-resolve.py 0 fallacies --out tmp/link-resolve-fallacies.csv +""" +import csv +import json +import sys +import time +import urllib.parse +import urllib.request + +DATASETS = { + "fallacies": "Cards/Fallacies/Argumentum Fallacies - Taxonomy.csv", + "virtues": "Cards/Fallacies/Argumentum Virtues - Taxonomy.csv", +} +# fr is ~45-97% (partly filled, source mix); en is the SOURCE we resolve FROM. +# Resolve the 6 under-filled targets (matches the #600 probe). +TARGET_LANGS = ["ru", "pt", "es", "ar", "fa", "zh"] +DEFAULT_DATASET = "fallacies" +SAMPLE_SIZE = 0 # 0 = full run; >0 = deterministic-strided sample +API_DELAY = 0.3 # polite throttle (MediaWiki best practice) +API_TIMEOUT = 25 +EN_WIKI_PREFIX = "https://en.wikipedia.org/wiki/" +USER_AGENT = "ArgumentumLinkResolver/1.0 (educational card game taxonomy; contact: jsboige@gmail.com)" + +# per-dataset: (pk column for the sidecar key, link_en column) +DATASET_KEYS = { + "fallacies": ("PK", "decimal_path"), + "virtues": ("pk", "decimal_path_padded"), +} + + +def extract_wiki_title(link_en): + url = (link_en or "").strip() + if not url.startswith(EN_WIKI_PREFIX): + return None + title = url[len(EN_WIKI_PREFIX):] + title = urllib.parse.unquote(title).replace("_", " ").strip() + return title or None + + +def build_url(lang, title): + """Build the canonical <lang>.wikipedia.org URL from a resolved title.""" + return "https://%s.wikipedia.org/wiki/%s" % (lang, urllib.parse.quote(title.replace(" ", "_"))) + + +def query_langlinks_titles(title): + """Query langlinks for one article. Return ({lang: target_title}, error).""" + api = ("https://en.wikipedia.org/w/api.php?action=query&prop=langlinks" + "&titles=%s&lllimit=500&format=json" % urllib.parse.quote(title)) + try: + req = urllib.request.Request(api, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req, timeout=API_TIMEOUT) as r: + data = json.load(r) + except Exception as e: + return None, "error: %s" % e + pages = data.get("query", {}).get("pages", {}) + out = {} + for p in pages.values(): + for ll in p.get("langlinks", []): + lang = ll.get("lang") + target = ll.get("*") + if lang and target: + out[lang] = target + return out, None + + +def main(): + sample_size = SAMPLE_SIZE + dataset = DEFAULT_DATASET + out_path = None + args = sys.argv[1:] + i = 0 + while i < len(args): + a = args[i] + if a == "--out": + out_path = args[i + 1] if i + 1 < len(args) else None + i += 2 + elif a.isdigit() or a == "0": + sample_size = int(a) + i += 1 + elif a.lower() in DATASETS: + dataset = a.lower() + i += 1 + else: + i += 1 + csv_path = DATASETS[dataset] + pk_cols = DATASET_KEYS[dataset] + + with open(csv_path, encoding="utf-8-sig") as f: + rows = list(csv.DictReader(f)) + + # categorize link_en + build candidate (row -> title) for missing target cells + wiki_title_of = {} # id(row) -> en title + for row in rows: + t = extract_wiki_title((row.get("link_en") or "").strip()) + if t: + wiki_title_of[id(row)] = t + + # candidate_missing[lang] = list of (row, title) + candidate_missing = {lang: [] for lang in TARGET_LANGS} + for row in rows: + title = wiki_title_of.get(id(row)) + if not title: + continue + for lang in TARGET_LANGS: + if not (row.get("link_" + lang) or "").strip(): + candidate_missing[lang].append((row, title)) + + all_titles = sorted({t for lst in candidate_missing.values() for _, t in lst}) + if sample_size and len(all_titles) > sample_size: + stride = len(all_titles) / sample_size + sample = [all_titles[int(i * stride)] for i in range(sample_size)] + else: + sample = all_titles + + print("=== link_* RESOLVER: %s (%s) | %d/%d unique articles ===" % + (dataset, csv_path, len(sample), len(all_titles)), file=sys.stderr) + print(" candidates missing: %s" % + {lang: len(lst) for lang, lst in candidate_missing.items()}, file=sys.stderr) + + # resolve: title -> {lang: target_title} + title_targets = {} + errors = 0 + for idx, title in enumerate(sample): + targets, err = query_langlinks_titles(title) + if err: + errors += 1 + title_targets[title] = {} + else: + title_targets[title] = targets or {} + time.sleep(API_DELAY) + if (idx + 1) % 10 == 0: + print(" ...%d/%d (errors=%d)" % (idx + 1, len(sample), errors), file=sys.stderr) + + # emit sidecar: dataset, key, lang, resolved_url (only resolvable candidates) + def row_key(row): + for c in pk_cols: + v = (row.get(c) or "").strip() + if v: + return v + return "" + + lines = ["dataset,key,link_lang,resolved_url"] + resolved_count = {lang: 0 for lang in TARGET_LANGS} + for lang in TARGET_LANGS: + for row, title in candidate_missing[lang]: + if title not in title_targets: + continue + targets = title_targets[title] + if lang not in targets: + continue + url = build_url(lang, targets[lang]) + lines.append("%s,%s,%s,%s" % (dataset, row_key(row), lang, url)) + resolved_count[lang] += 1 + + payload = "\n".join(lines) + "\n" + if out_path: + import os + os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True) + with open(out_path, "w", encoding="utf-8", newline="") as f: + f.write(payload) + print("wrote sidecar: %s (%d candidate fills)" % (out_path, len(lines) - 1), file=sys.stderr) + else: + sys.stdout.write(payload) + + total_resolved = sum(resolved_count.values()) + print("\n=== SUMMARY (%s, %d articles probed) ===" % (dataset, len(sample)), file=sys.stderr) + for lang in TARGET_LANGS: + print(" link_%s: %d resolved URLs" % (lang, resolved_count[lang]), file=sys.stderr) + print(" TOTAL candidate fills: %d (probe ceiling was ~57%%; this materializes them)" % + total_resolved, file=sys.stderr) + print(" errors: %d/%d" % (errors, len(sample)), file=sys.stderr) + print(" SAFETY: sidecar only. 0 write under Cards/. AR/FA/ZH need human spot-validation.", file=sys.stderr) + + +if __name__ == "__main__": + main()