Lift a couple more hot regexps out of the loop

hplt-project · Sep 14, 2023 · 38d83b5 · 38d83b5
1 parent cf86159
commit 38d83b5
Showing 1 changed file with 12 additions and 8 deletions.
diff --git a/sacremoses/tokenize.py b/sacremoses/tokenize.py
@@ -668,6 +668,10 @@ class MosesDetokenizer(object):
 
     STARTS_WITH_ALPHA = re.compile(r"^[{}]".format(IsAlpha))
 
+    IS_PUNCT = re.compile(r"^[\,\.\?\!\:\;\\\%\}\]\)]+$")
+
+    IS_OPEN_QUOTE = re.compile(r"""^[\'\"„“`]+$""")
+
     def __init__(self, lang="en"):
         super(MosesDetokenizer, self).__init__()
         self.lang = lang
@@ -716,12 +720,12 @@ def tokenize(self, tokens, return_str=True, unescape=True):
                     detokenized_text += prepend_space + token
                 prepend_space = " "
             # If it's a currency symbol.
-            elif re.search(self.IS_CURRENCY_SYMBOL, token):
+            elif self.IS_CURRENCY_SYMBOL.search(token):
                 # Perform right shift on currency and other random punctuation items
                 detokenized_text += prepend_space + token
                 prepend_space = ""
 
-            elif re.search(r"^[\,\.\?\!\:\;\\\%\}\]\)]+$", token):
+            elif self.IS_PUNCT.search(token):
                 # In French, these punctuations are prefixed with a non-breakable space.
                 if self.lang == "fr" and re.search(r"^[\?\!\:\;\\\%]$", token):
                     detokenized_text += " "
@@ -732,7 +736,7 @@ def tokenize(self, tokens, return_str=True, unescape=True):
             elif (
                 self.lang == "en"
                 and i > 0
-                and re.search(self.IS_ENGLISH_CONTRACTION, token)
+                and self.IS_ENGLISH_CONTRACTION.search(token)
             ):
                 # and re.search('[{}]$'.format(self.IsAlnum), tokens[i-1])):
                 # For English, left-shift the contraction.
@@ -755,8 +759,8 @@ def tokenize(self, tokens, return_str=True, unescape=True):
             elif (
                 self.lang in ["fr", "it", "ga"]
                 and i <= len(tokens) - 2
-                and re.search(self.IS_FRENCH_CONRTACTION, token)
-                and re.search(self.STARTS_WITH_ALPHA, tokens[i + 1])
+                and self.IS_FRENCH_CONRTACTION.search(token)
+                and self.STARTS_WITH_ALPHA.search(tokens[i + 1])
             ):  # If the next token is alpha.
                 # For French and Italian, right-shift the contraction.
                 detokenized_text += prepend_space + token
@@ -765,7 +769,7 @@ def tokenize(self, tokens, return_str=True, unescape=True):
             elif (
                 self.lang == "cs"
                 and i <= len(tokens) - 3
-                and re.search(self.IS_FRENCH_CONRTACTION, token)
+                and self.IS_FRENCH_CONRTACTION.search(token)
                 and re.search(r"^[-–]$", tokens[i + 1])
                 and re.search(r"^li$|^mail.*", tokens[i + 2], re.IGNORECASE)
             ):  # In Perl, ($words[$i+2] =~ /^li$|^mail.*/i)
@@ -775,7 +779,7 @@ def tokenize(self, tokens, return_str=True, unescape=True):
                 prepend_space = ""
 
             # Combine punctuation smartly.
-            elif re.search(r"""^[\'\"„“`]+$""", token):
+            elif self.IS_OPEN_QUOTE.search(token):
                 normalized_quo = token
                 if re.search(r"^[„“”]+$", token):
                     normalized_quo = '"'
@@ -811,7 +815,7 @@ def tokenize(self, tokens, return_str=True, unescape=True):
             elif (
                 self.lang == "fi"
                 and re.search(r":$", tokens[i - 1])
-                and re.search(self.FINNISH_REGEX, token)
+                and self.FINNISH_REGEX.search(token)
             ):
                 # Finnish : without intervening space if followed by case suffix
                 # EU:N EU:n EU:ssa EU:sta EU:hun EU:iin ...