[common] Update French number formula.

Author: alexnanchen

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "lib/nltk"]
 	path = lib/nltk
 	url = https://github.com/nltk/nltk.git
+[submodule "lib/num2words"]
+	path = lib/num2words
+	url = https://github.com/savoirfairelinux/num2words.git

INSTALL.md

@@ -8,7 +8,9 @@ This install manual is for Debian based systems.
 
    `git submodule update --init lib/unicodecsv`
 
-2. Install both libraries using the *setup.py* scripts.
+   `git submodule update --init lib/num2words`
+
+2. Install the three libraries using the *setup.py* scripts.
 
    `setup.py install --prefix=/path/to/local/install`
 

lib/num2words

@@ -121,9 +121,10 @@
 (ur"\u215D",u"5/8",u"Number forms: cinq huitieme",u"0"),
 (ur"\u215E",u"7/8",u"Number forms: sept huitieme",u"0")]
 
-PUNCTUATIONEXCLUDE = ['!', '"', '#', "'", '(', ')', '*', '+', ',', '-',
+#Do not exclude single quote
+PUNCTUATIONEXCLUDE = ['!', '"', '#', '(', ')', '*', '+', ',', '-',
                       '.', '/', ':', ';', '<', '=', '>', '?', '[', '\\',
-                      ']', '^', '_', '`', '{', '|', '}', '~', "'"]
+                      ']', '^', '_', '`', '{', '|', '}', '~']
 PUNCTUATIONMAP  = {
     "%": (r"%",r"pourcent", u"Prozent", u"percent", u"per cento"),
     "&": (r"&",r"et", u"und", u"and", u"e"),

scripts/common/AsrtConstants.py

@@ -212,8 +212,9 @@ def _normalizePunctuation(self):
         unicodeList = []
         for i, c in enumerate(self.strText):
             strC = c.encode('utf-8')
+            #For date format, i.e. 21-Jul
             if strC in PUNCTUATIONEXCLUDE:
-                continue
+                unicodeList.append(u" ")
             elif self.languageId != 0 and strC in PUNCTUATIONMAP:
                 unicodeList.append(u" " + PUNCTUATIONMAP[strC][self.languageId] + u" ")
             else:

scripts/common/FormulaLMPreparation.py

@@ -28,16 +28,22 @@
 
 import logging, re
 from FormulaLMPreparation import LMPreparationFormula
+from num2words import num2words
+from roman import fromRoman
 
 class NumberFormula():
     """Various number formats expansion.
     """
     logger                  = logging.getLogger("Asrt.NumberFormula")
 
+    THOUSANDSEPARATOR       = u"'"
+   
+    HASNUMBERREGEX          = re.compile(u"([0-9]|I|V|X|L|C|D|M)+", flags=re.UNICODE)
     CARDINALNUMBERREGEX     = re.compile(u"[0-9]+$", flags=re.UNICODE)
-    ORDINALNUMBERREGEX      = re.compile(u"(1er|1re|[0-9]+e||[0-9]+ème)$", flags=re.UNICODE)
-    DECIMALNUMBERREGEX      = re.compile(u"[0-9]+[,.][0-9]+$", flags=re.UNICODE)
-    ROMANNUMBERREGEX        = re.compile(u"(I|V|X|L|C|D|M)+(er|re|e|eme|ème)$", flags=re.UNICODE)
+    ORDINALNUMBERREGEX      = re.compile(u"(1er|1re|1ère|[0-9]+e|[0-9]+ème|Ier|Ière|[IVXLCDM]+ème)$", flags=re.UNICODE)
+    ORDINALREPLACEREGEX     = re.compile(u"[erèm]", flags=re.UNICODE)
+    DECIMALNUMBERREGEX      = re.compile(u"[0-9,.]+[0-9,.]*$", flags=re.UNICODE)
+    ROMANNUMBERREGEX        = re.compile(u"[IVXLCDM]+(er|re|ère|e|ème)?$", flags=re.UNICODE)
 
     ##################
     #Public interface
@@ -54,26 +60,61 @@ def apply(self, strText):
            param strText: an utf-8 encoded string
            return an utf-8 encoded string 
         """
-        wordsList = re.split(LMPreparationFormula.SPACEREGEX, strText, flags=re.UNICODE)
+        wordsList = re.split(LMPreparationFormula.SPACEPATTERN, strText, flags=re.UNICODE)
 
         newWordsList = []
         for w in wordsList:
-            if self._isCardinalNumber(w):
-                pass
-            elif self._isOrdinalNumber(w):
-                pass
-            elif self._isDecimalNumber(w):
-                pass
-            elif self._isRomanNumber(w):
-                pass
-            else:
+            if not self._hasNumber(w):
+                newWordsList.append(w)
+                continue
+            #Numbers may contain alphanumeric
+            #characters
+            wNorm = self._normalizeNumber(w)
+            try:
+                #Now check number type
+                if self._isCardinalNumber(wNorm):
+                    wNorm = self._cardinal2word(wNorm)
+                elif self._isOrdinalNumber(wNorm):
+                    wNorm = self._ordinal2word(wNorm)
+                elif self._isDecimalNumber(wNorm):
+                    wNorm = self._decimal2word(wNorm)
+                elif self._isRomanNumber(wNorm):
+                    wNorm = self._roman2word(wNorm)
+                else:
+                    self.logger.info("Unknown number format: %s" % w.encode('utf-8'))
+                    wNorm = w
+                newWordsList.append(wNorm)
+
+            except Exception, e:
+                self.logger.warning("Error formatting number (%s): %s" % \
+                    (w.encode('utf-8'), str(e)))
                 newWordsList.append(w)
 
         return u" ".join(newWordsList)
 
     ##################
     #Implementation
     #
+    @staticmethod
+    def _hasNumber(strWord):
+        """Check if 'strWord' contains numbers.
+
+           param strWord: an utf-8 encoded words
+           return True or False
+        """
+        #Use search instead of match
+        return NumberFormula.HASNUMBERREGEX.search(strWord) != None
+
+    @staticmethod
+    def _normalizeNumber(strWord):
+        """Remove tousand separator.
+
+           param strWord: an utf-8 encoded words
+           return an utf-8 encoded string
+        """
+        strWord = strWord.replace(NumberFormula.THOUSANDSEPARATOR, u"")
+        return strWord
+
     @staticmethod
     def _cardinal2word(strNumber):
         """Convert a cardinal number to a written
@@ -82,18 +123,36 @@ def _cardinal2word(strNumber):
            param strNumber: an utf-8 cardinal number
            return a 'written' cardinal number
         """
-        pass
+        strNumber = num2words(int(strNumber), lang='fr')
+        return strNumber.replace(u"-", u" ")
 
     @staticmethod
     def _ordinal2word(strNumber):
         """Convert an ordinal number to a written
            word.
 
+           i.e. 1er --> premier
+
            param strNumber: an utf-8 ordinal number
            return a 'written' ordinal number
         """
-        pass
-
+        if strNumber.encode('utf-8') == u"1ère".encode('utf-8'):
+            return u"première"
+
+        strNewNumber = re.sub(u"[erèm]", "", strNumber)
+        if NumberFormula._isCardinalNumber(strNewNumber):
+            strNewNumber = num2words(int(strNewNumber), ordinal=True, lang='fr')
+        elif NumberFormula._isRomanNumber(strNewNumber):
+            #Roman to cardinal
+            strNewNumber = strNewNumber.encode('utf-8')
+            cardinalNumber = fromRoman(strNewNumber)
+            #Digits to ordinal
+            strNewNumber = num2words(cardinalNumber, ordinal=True, lang='fr')
+        else:
+            strNewNumber = strNumber
+
+        return strNewNumber
+            
     @staticmethod
     def _decimal2word(strNumber):
         """Convert a decimal number to a written
@@ -102,7 +161,17 @@ def _decimal2word(strNumber):
            param strNumber: an utf-8 decimal number
            return a 'written' decimal number
         """
-        pass
+        strNumber = u" virgule ".join(re.split("[,]",strNumber))
+        strNumber = u" point ".join(re.split("[.]",strNumber))
+
+        tokenList = []
+        for w in re.split(LMPreparationFormula.SPACEPATTERN, strNumber):
+            w = w.strip()
+            if NumberFormula._isCardinalNumber(w):
+                w = NumberFormula._cardinal2word(w)
+            tokenList.append(w)
+
+        return u" ".join(tokenList)
 
     @staticmethod
     def _roman2word(strNumber):
@@ -112,8 +181,10 @@ def _roman2word(strNumber):
            param strNumber: an utf-8 roman number
            return a 'written' roman number
         """
-        pass
-
+        strNumber = strNumber.encode('utf-8')
+        cardinalNumber = fromRoman(strNumber)
+        return NumberFormula._cardinal2word(cardinalNumber)
+        
     @staticmethod
     def _isCardinalNumber(strWord):
         """Check if 'strWord' is a cardinal number.

scripts/common/french/FormulaNumber.py

@@ -26,9 +26,82 @@
 
 class FormulaNumberUnitTest(unittest.TestCase):
 
-    ###############
+    testDict = { "cardinal": [(u"10",u"dix"),(u"25",u"vingt cinq")],
+                 "ordinal" : [(u"1er",u"premier"),(u"1ère",u"première"),(u"2ème",u"deuxième"),
+                              (u"Vème",u"cinquième"), (u"Xème",u"dixième")],
+                 "decimal" : [(u"2,5",u"deux virgule cinq"), (u"2.5,3",u"deux point cinq virgule trois")],
+                 "roman"   : [(u"V",u"cinq"), (u"X",u"dix")],
+                 "all"     : [(u"1ab",u"1ab"),(u"ab",u"ab"),
+                              (u"le 25 mars 2015 2.5 Xème",u"le vingt cinq mars deux mille quinze deux point cinq dixième")]
+    }
+
+    #################
+    # Implementation
+    #
+    def evaluateListValues(self, testList, callback):
+        for t, gt in testList:
+            r = callback(t).encode('utf-8')
+            self.assertEquals(gt.encode('utf-8'), r, r)
+
+    #################
     # Unit tests
     #
-    def testIsCardinal(self):
-        pass
+    def test_isCardinal(self):
+        testList = [(u"2",True),(u"123",True), (u"123.",False)]
+
+        for t, gt in testList:
+        	self.assertEquals(NumberFormula._isCardinalNumber(t), gt, t.encode('utf-8'))
+    
+    def test_isOrdinal(self):
+        testList = [(u"1er",True), (u"1re",True), (u"1ère",True), (u"2e",True), (u"2ème",True), ]
+
+        for t, gt in testList:
+        	self.assertEquals(NumberFormula._isOrdinalNumber(t), gt, t.encode('utf-8'))
+ 
+    def test_isDecimal(self):
+        testList = [(u"2.5",True), (u"2,5",True),(u"2,5,3",True), (u"2-5",False)]
+
+        for t, gt in testList:
+        	self.assertEquals(NumberFormula._isDecimalNumber(t), gt, t.encode('utf-8'))
+
+    def test_isRoman(self):
+    	testList = [(u"V",True), (u"Ier",True),(u"XII",True), (u"XIIème",True)]
+
+    	for t, gt in testList:
+        	self.assertEquals(NumberFormula._isRomanNumber(t), gt, t.encode('utf-8'))
+
+    def test_hasNumber(self):
+        testList = [(u"12",True), (u"1ab",True),(u"ab22",True), (u"Xab",True),
+                    (u"xab",False), (u"a1ab",True)]
+
+        for t, gt in testList:
+            self.assertEquals(NumberFormula._hasNumber(t), gt, t.encode('utf-8'))
+
+    def test_normalizeNumber(self):
+        testList = [(u"50'000",u"50000"),(u"550'000'000",u"550000000")]
+        self.evaluateListValues(testList, NumberFormula._normalizeNumber)
+        
+    def test_cardinal2word(self):
+        testList = self.testDict["cardinal"]
+        self.evaluateListValues(testList, NumberFormula._cardinal2word)
+        
+    def test_ordinal2word(self):
+        testList = self.testDict["ordinal"]
+        self.evaluateListValues(testList, NumberFormula._ordinal2word)
+        
+    def test_decimal2word(self):
+        testList = self.testDict["decimal"]
+        self.evaluateListValues(testList, NumberFormula._decimal2word)
+
+    def test_roman2word(self):
+        testList = self.testDict["roman"]
+        self.evaluateListValues(testList, NumberFormula._roman2word)
+
+    def test_apply(self):
+        f = NumberFormula()
+
+        for k in self.testDict.keys():
+            #print "Testing %s " % k
+            testList = self.testDict[k]
+            self.evaluateListValues(testList, f.apply)