Update test_normalizer.py to include test for perl parity

hplt-project · Oct 26, 2023 · c57e777 · c57e777
1 parent a57726d
commit c57e777
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/sacremoses/test/test_normalizer.py b/sacremoses/test/test_normalizer.py
@@ -70,3 +70,9 @@ def test_normalization_pipeline(self):
         text = "０《１２３》      ４５６％  '' 【７８９】"
         expected = '0"123" 456% " [789]'
         assert moses_norm_unicode.normalize(text) == expected
+
+    def test_moses_normalize_with_perl_parity(self):
+        moses_perl_parity = MosesPunctNormalizer(perl_parity=True)
+        text = 'from the ‘bad bank’, Northern, wala « dox ci jawwu Les « wagonways » étaient construits'
+        expected = '''from the 'bad bank," Northern, wala "dox ci jawwu Les "wagonways" étaient construits'''
+        assert moses_perl_parity.normalize(text) == expected