extract: Determine python-format flag explicitly

During extraction, Message instances can be created with the
"python-format" flag, indicating that the message string contains Python
percent-formatting placeholders. To avoid setting the flag erroneously
because the string source is not Python code or otherwise is not
expected to contain such placeholders, the extractor interface must be
extended to allow extractor functions to indicate which flags are
valid.

Fixes python-babel#35

Author: Jonathan Ross Rogers

babel/messages/catalog.py

@@ -99,10 +99,7 @@ def __init__(self, id, string=u'', locations=(), flags=(), auto_comments=(),
         self.string = string
         self.locations = list(distinct(locations))
         self.flags = set(flags)
-        if id and self.python_format:
-            self.flags.add('python-format')
-        else:
-            self.flags.discard('python-format')
+
         self.auto_comments = list(distinct(auto_comments))
         self.user_comments = list(distinct(user_comments))
         if isinstance(previous_id, string_types):
@@ -112,6 +109,13 @@ def __init__(self, id, string=u'', locations=(), flags=(), auto_comments=(),
         self.lineno = lineno
         self.context = context
 
+    def determine_python_format(self):
+        """Sets python-format flag if message contains a format string"""
+        if self.id and self.python_format:
+            self.flags.add('python-format')
+        else:
+            self.flags.discard('python-format')
+
     def __repr__(self):
         return '<%s %r (flags: %r)>' % (type(self).__name__, self.id,
                                         list(self.flags))

babel/messages/extract.py

@@ -67,7 +67,7 @@ def extract_from_dir(dirname=None, method_map=DEFAULT_MAPPING,
     """Extract messages from any source files found in the given directory.
 
     This function generates tuples of the form ``(filename, lineno, message,
-    comments, context)``.
+    comments, context, flags)``.
 
     Which extraction method is used per file is determined by the `method_map`
     parameter, which maps extended glob patterns to extraction method names.
@@ -220,7 +220,7 @@ def extract_from_file(method, filename, keywords=DEFAULT_KEYWORDS,
                       comment_tags=(), options=None, strip_comment_tags=False):
     """Extract messages from a specific file.
 
-    This function returns a list of tuples of the form ``(lineno, message, comments, context)``.
+    This function returns a list of tuples of the form ``(lineno, message, comments, context, flags)``.
 
     :param filename: the path to the file to extract messages from
     :param method: a string specifying the extraction method (.e.g. "python")
@@ -246,7 +246,7 @@ def extract(method, fileobj, keywords=DEFAULT_KEYWORDS, comment_tags=(),
     """Extract messages from the given file-like object using the specified
     extraction method.
 
-    This function returns tuples of the form ``(lineno, message, comments, context)``.
+    This function returns tuples of the form ``(lineno, message, comments, context, flags)``.
 
     The implementation dispatches the actual extraction to plugins, based on the
     value of the ``method`` parameter.
@@ -259,7 +259,7 @@ def extract(method, fileobj, keywords=DEFAULT_KEYWORDS, comment_tags=(),
     >>> from babel._compat import BytesIO
     >>> for message in extract('python', BytesIO(source)):
     ...     print(message)
-    (3, u'Hello, world!', [], None)
+    (3, u'Hello, world!', [], None, ())
 
     :param method: an extraction method (a callable), or
                    a string specifying the extraction method (.e.g. "python");
@@ -316,10 +316,17 @@ def extract(method, fileobj, keywords=DEFAULT_KEYWORDS, comment_tags=(),
     if func is None:
         raise ValueError('Unknown extraction method %r' % method)
 
-    results = func(fileobj, keywords.keys(), comment_tags,
-                   options=options or {})
+    for result in func(fileobj, keywords.keys(), comment_tags,
+                       options=options or {}):
+        flags = ()
+        if len(result) == 4:
+            lineno, funcname, messages, comments = result
+        elif len(result) == 5:
+            lineno, funcname, messages, comments, flags = result
+        else:
+            raise ValueError(
+                'Extraction function must yield tuples with 4 or 5 values')
 
-    for lineno, funcname, messages, comments in results:
         if funcname:
             spec = keywords[funcname] or (1,)
         else:
@@ -370,7 +377,7 @@ def extract(method, fileobj, keywords=DEFAULT_KEYWORDS, comment_tags=(),
 
         if strip_comment_tags:
             _strip_comment_tags(comments, comment_tags)
-        yield lineno, messages, comments, context
+        yield lineno, messages, comments, context, flags
 
 
 def extract_nothing(fileobj, keywords, comment_tags, options):
@@ -465,7 +472,7 @@ def extract_python(fileobj, keywords, comment_tags, options):
                     translator_comments = []
 
                 yield (message_lineno, funcname, messages,
-                       [comment[1] for comment in translator_comments])
+                       [comment[1] for comment in translator_comments], ())
 
                 funcname = lineno = message_lineno = None
                 call_stack = -1

babel/messages/frontend.py

@@ -467,13 +467,13 @@ def callback(filename, method, options):
                         callback=callback,
                         strip_comment_tags=self.strip_comments
                     )
-                for filename, lineno, message, comments, context in extracted:
+                for fname, lineno, msg, comments, context, flags in extracted:
                     if os.path.isfile(path):
-                        filepath = filename  # already normalized
+                        filepath = fname  # already normalized
                     else:
-                        filepath = os.path.normpath(os.path.join(path, filename))
+                        filepath = os.path.normpath(os.path.join(path, fname))
 
-                    catalog.add(message, None, [(filepath, lineno)],
+                    catalog.add(msg, None, [(filepath, lineno)], flags=flags,
                                 auto_comments=comments, context=context)
 
             self.log.info('writing PO template file to %s', self.output_file)

babel/messages/pofile.py

@@ -442,9 +442,9 @@ def write_po(fileobj, catalog, width=76, no_location=False, omit_header=False,
     message catalog to the provided file-like object.
 
     >>> catalog = Catalog()
-    >>> catalog.add(u'foo %(name)s', locations=[('main.py', 1)],
+    >>> message = catalog.add(u'foo %(name)s', locations=[('main.py', 1)],
     ...             flags=('fuzzy',))
-    <Message...>
+    >>> message.determine_python_format()
     >>> catalog.add((u'bar', u'baz'), locations=[('main.py', 3)])
     <Message...>
     >>> from babel._compat import BytesIO

tests/messages/test_extract.py

@@ -37,33 +37,34 @@ def test_nested_calls(self):
         messages = list(extract.extract_python(buf,
                                                extract.DEFAULT_KEYWORDS.keys(),
                                                [], {}))
-        self.assertEqual([
-            (1, '_', None, []),
-            (2, 'ungettext', (None, None, None), []),
-            (3, 'ungettext', (u'Babel', None, None), []),
-            (4, 'ungettext', (None, u'Babels', None), []),
-            (5, 'ungettext', (u'bunny', u'bunnies', None), []),
-            (6, 'ungettext', (None, u'bunnies', None), []),
-            (7, '_', None, []),
-            (8, 'gettext', u'Rabbit', []),
-            (9, 'dgettext', (u'wiki', None), []),
-            (10, 'dngettext', (None, u'Page', u'Pages', None), [])],
-            messages)
+        self.assertEqual(
+            [
+                (1, '_', None, [], ()),
+                (2, 'ungettext', (None, None, None), [], ()),
+                (3, 'ungettext', (u'Babel', None, None), [], ()),
+                (4, 'ungettext', (None, u'Babels', None), [], ()),
+                (5, 'ungettext', (u'bunny', u'bunnies', None), [], ()),
+                (6, 'ungettext', (None, u'bunnies', None), [], ()),
+                (7, '_', None, [], ()),
+                (8, 'gettext', u'Rabbit', [], ()),
+                (9, 'dgettext', (u'wiki', None), [], ()),
+                (10, 'dngettext', (None, u'Page', u'Pages', None), [], ())
+            ], messages)
 
     def test_extract_default_encoding_ascii(self):
         buf = BytesIO(b'_("a")')
         messages = list(extract.extract_python(
             buf, list(extract.DEFAULT_KEYWORDS), [], {},
         ))
         # Should work great in both py2 and py3
-        self.assertEqual([(1, '_', 'a', [])], messages)
+        self.assertEqual([(1, '_', 'a', [], ())], messages)
 
     def test_extract_default_encoding_utf8(self):
         buf = BytesIO(u'_("☃")'.encode('UTF-8'))
         messages = list(extract.extract_python(
             buf, list(extract.DEFAULT_KEYWORDS), [], {},
         ))
-        self.assertEqual([(1, '_', u'☃', [])], messages)
+        self.assertEqual([(1, '_', u'☃', [], ())], messages)
 
     def test_nested_comments(self):
         buf = BytesIO(b"""\
@@ -73,7 +74,7 @@ def test_nested_comments(self):
 """)
         messages = list(extract.extract_python(buf, ('ngettext',),
                                                ['TRANSLATORS:'], {}))
-        self.assertEqual([(1, 'ngettext', (u'pylon', u'pylons', None), [])],
+        self.assertEqual([(1, 'ngettext', (u'pylon', u'pylons', None), [], ())],
                          messages)
 
     def test_comments_with_calls_that_spawn_multiple_lines(self):
@@ -98,21 +99,21 @@ def test_comments_with_calls_that_spawn_multiple_lines(self):
 
                                                {'strip_comment_tags': False}))
         self.assertEqual((6, '_', 'Locale deleted.',
-                          [u'NOTE: This Comment SHOULD Be Extracted']),
+                          [u'NOTE: This Comment SHOULD Be Extracted'], ()),
                          messages[1])
         self.assertEqual((10, 'ngettext', (u'Foo deleted.', u'Foos deleted.',
                                            None),
-                          [u'NOTE: This Comment SHOULD Be Extracted']),
+                          [u'NOTE: This Comment SHOULD Be Extracted'], ()),
                          messages[2])
         self.assertEqual((3, 'ngettext',
-                          (u'Catalog deleted.',
-                           u'Catalogs deleted.', None),
-                          [u'NOTE: This Comment SHOULD Be Extracted']),
+                           (u'Catalog deleted.',
+                            u'Catalogs deleted.', None),
+                           [u'NOTE: This Comment SHOULD Be Extracted'], ()),
                          messages[0])
         self.assertEqual((15, 'ngettext', (u'Bar deleted.', u'Bars deleted.',
                                            None),
                           [u'NOTE: This Comment SHOULD Be Extracted',
-                           u'NOTE: And This One Too']),
+                           u'NOTE: And This One Too'], ()),
                          messages[3])
 
     def test_declarations(self):
@@ -129,9 +130,9 @@ class Meta:
         messages = list(extract.extract_python(buf,
                                                extract.DEFAULT_KEYWORDS.keys(),
                                                [], {}))
-        self.assertEqual([(3, '_', u'Page arg 1', []),
-                          (3, '_', u'Page arg 2', []),
-                          (8, '_', u'log entry', [])],
+        self.assertEqual([(3, '_', u'Page arg 1', [], ()),
+                          (3, '_', u'Page arg 2', [], ()),
+                          (8, '_', u'log entry', [], ())],
                          messages)
 
     def test_multiline(self):
@@ -143,8 +144,8 @@ def test_multiline(self):
                  count)
 """)
         messages = list(extract.extract_python(buf, ('ngettext',), [], {}))
-        self.assertEqual([(1, 'ngettext', (u'pylon', u'pylons', None), []),
-                          (3, 'ngettext', (u'elvis', u'elvises', None), [])],
+        self.assertEqual([(1, 'ngettext', (u'pylon', u'pylons', None), [], ()),
+                          (3, 'ngettext', (u'elvis', u'elvises', None), [], ())],
                          messages)
 
     def test_npgettext(self):
@@ -156,8 +157,8 @@ def test_npgettext(self):
                  count)
 """)
         messages = list(extract.extract_python(buf, ('npgettext',), [], {}))
-        self.assertEqual([(1, 'npgettext', (u'Strings', u'pylon', u'pylons', None), []),
-                          (3, 'npgettext', (u'Strings', u'elvis', u'elvises', None), [])],
+        self.assertEqual([(1, 'npgettext', (u'Strings', u'pylon', u'pylons', None), [], ()),
+                          (3, 'npgettext', (u'Strings', u'elvis', u'elvises', None), [], ())],
                          messages)
         buf = BytesIO(b"""\
 msg = npgettext('Strings', 'pylon',  # TRANSLATORS: shouldn't be
@@ -166,7 +167,7 @@ def test_npgettext(self):
 """)
         messages = list(extract.extract_python(buf, ('npgettext',),
                                                ['TRANSLATORS:'], {}))
-        self.assertEqual([(1, 'npgettext', (u'Strings', u'pylon', u'pylons', None), [])],
+        self.assertEqual([(1, 'npgettext', (u'Strings', u'pylon', u'pylons', None), [], ())],
                          messages)
 
     def test_triple_quoted_strings(self):
@@ -178,9 +179,9 @@ def test_triple_quoted_strings(self):
         messages = list(extract.extract_python(buf,
                                                extract.DEFAULT_KEYWORDS.keys(),
                                                [], {}))
-        self.assertEqual([(1, '_', u'pylons', []),
-                          (2, 'ngettext', (u'elvis', u'elvises', None), []),
-                          (3, 'ngettext', (u'elvis', u'elvises', None), [])],
+        self.assertEqual([(1, '_', u'pylons', [], ()),
+                          (2, 'ngettext', (u'elvis', u'elvises', None), [], ()),
+                          (3, 'ngettext', (u'elvis', u'elvises', None), [], ())],
                          messages)
 
     def test_multiline_strings(self):
@@ -196,7 +197,7 @@ def test_multiline_strings(self):
             [(1, '_',
               u'This module provides internationalization and localization\n'
               'support for your Python programs by providing an interface to '
-              'the GNU\ngettext message catalog library.', [])],
+              'the GNU\ngettext message catalog library.', [], ())],
             messages)
 
     def test_concatenated_strings(self):
@@ -456,6 +457,10 @@ def test_nested_messages(self):
         self.assertEqual([], messages[7][3])
 
 
+def extract_bad(fileobj, keywords, comment_tags, options):
+    yield (None,)
+
+
 class ExtractTestCase(unittest.TestCase):
 
     def test_invalid_filter(self):
@@ -474,14 +479,19 @@ def test_invalid_filter(self):
         messages = \
             list(extract.extract('python', buf, extract.DEFAULT_KEYWORDS, [],
                                  {}))
-        self.assertEqual([(5, (u'bunny', u'bunnies'), [], None),
-                          (8, u'Rabbit', [], None),
-                          (10, (u'Page', u'Pages'), [], None)], messages)
+        self.assertEqual([(5, (u'bunny', u'bunnies'), [], None, ()),
+                          (8, u'Rabbit', [], None, ()),
+                          (10, (u'Page', u'Pages'), [], None, ())], messages)
 
     def test_invalid_extract_method(self):
         buf = BytesIO(b'')
         self.assertRaises(ValueError, list, extract.extract('spam', buf))
 
+    def test_bad_extract_function(self):
+        self.assertRaises(
+            ValueError, list,
+            extract.extract('tests.messages.test_extract:extract_bad', ''))
+
     def test_different_signatures(self):
         buf = BytesIO(b"""
 foo = _('foo', 'bar')

tests/messages/test_js_extract.py

@@ -14,9 +14,9 @@ def test_simple_extract():
         list(extract.extract('javascript', buf, extract.DEFAULT_KEYWORDS,
                              [], {}))
 
-    assert messages == [(1, 'simple', [], None),
-                        (2, 'simple', [], None),
-                        (3, ('s', 'p'), [], None)]
+    assert messages == [(1, 'simple', [], None, ()),
+                        (2, 'simple', [], None, ()),
+                        (3, ('s', 'p'), [], None, ())]
 
 
 def test_various_calls():
@@ -36,9 +36,9 @@ def test_various_calls():
         list(extract.extract('javascript', buf, extract.DEFAULT_KEYWORDS, [],
                              {}))
     assert messages == [
-        (5, (u'bunny', u'bunnies'), [], None),
-        (8, u'Rabbit', [], None),
-        (10, (u'Page', u'Pages'), [], None)
+        (5, (u'bunny', u'bunnies'), [], None, ()),
+        (8, u'Rabbit', [], None, ()),
+        (10, (u'Page', u'Pages'), [], None, ())
     ]
 
 
@@ -132,7 +132,7 @@ def test_dotted_keyword_extract():
         extract.extract('javascript', buf, {"com.corporate.i18n.formatMessage": None}, [], {})
     )
 
-    assert messages == [(1, 'Insert coin to continue', [], None)]
+    assert messages == [(1, 'Insert coin to continue', [], None, ())]
 
 
 def test_template_string_standard_usage():
@@ -141,7 +141,7 @@ def test_template_string_standard_usage():
         extract.extract('javascript', buf, {"gettext": None}, [], {})
     )
 
-    assert messages == [(1, 'Very template, wow', [], None)]
+    assert messages == [(1, 'Very template, wow', [], None, ())]
 
 
 def test_template_string_tag_usage():
@@ -150,4 +150,4 @@ def test_template_string_tag_usage():
         extract.extract('javascript', buf, {"i18n": None}, [], {})
     )
 
-    assert messages == [(1, 'Tag template, wow', [], None)]
+    assert messages == [(1, 'Tag template, wow', [], None, ())]