Reorganization of cg

CHANGELOG.md

@@ -22,6 +22,7 @@ Unreleased
 -   Added `data/cg/tsv/jpn_hira.tsv`. (\#384)
 -   Enforced final newlines. (\#387)
 -   Adds all UniMorph languages to morphology. (\#393)
+-   Added `data/covering_grammar/lib/make_test_file.py` (\#396)
 
 #### Changed
 
@@ -45,6 +46,7 @@ Unreleased
 -   Fixed phone counting in `data/src/generate_phones_summary.py` (\#390, \#392)
 -   Reorganizes scraping scripts under `data/scrape` (\#394)
 -   Reorganizes `.phones` files and related scripts under `data/phones` (\#395)
+-   Reorganizes CG files and related scripts under `data/covering_grammar` (\#395)
 
 ### Under `wikipron/` and elsewhere
 

data/covering_grammar/README.md

@@ -0,0 +1 @@
+(TEMPORARY)

data/covering_grammar/lib/make_test_file.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+"""Makes test file.
+
+Takes the gold data and the model output, and creates a three-column TSV where
+each line has a word, its gold pronunciation, and the predicted pronunciation.
+Assumes that the input files have the same words in the same order.
+"""
+
+import argparse
+import logging
+
+
+def main(args: argparse.Namespace) -> None:
+    with open(args.gold, "r") as gf, open(args.pred, "r") as pf:
+        with open(args.out, "w") as wf:
+            for lineno, (g_line, p_line) in enumerate(zip(gf, pf), 1):
+                g_word, g_pron = g_line.split("\t")
+                p_word, p_pron = p_line.split("\t")
+                # Make sure that gold data and predictions have the
+                # same words.
+                if not g_word == p_word:
+                    logging.warning(
+                        "%s != %s (line %d)", g_word, p_word, lineno
+                    )
+                    continue
+                # Note that we use `strip` to remove the newline.
+                g_pron = g_pron.strip()
+                p_pron = p_pron.strip()
+                line = f"{g_word}\t{g_pron}\t{p_pron}"
+                print(line, file=wf)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "gold", help="TSV with words and correct pronunciations"
+    )
+    parser.add_argument(
+        "pred", help="TSV with words and predicted pronunciations"
+    )
+    parser.add_argument("out", help="file to write to")
+    main(parser.parse_args())