extract: Determine python-format flag explicitly

During extraction, Message instances can be created with the
"python-format" flag, indicating that the message string contains Python
percent-formatting placeholders. To avoid setting the flag erroneously
because the string source is not Python code or otherwise is not
expected to contain such placeholders, the extractor interface must be
extended to allow extractor functions to indicate which flags are
valid.

Fixes python-babel#35

Author: Jonathan Ross Rogers

babel/messages/catalog.py

@@ -99,10 +99,7 @@ def __init__(self, id, string=u'', locations=(), flags=(), auto_comments=(),
         self.string = string #: The message translation
         self.locations = list(distinct(locations))
         self.flags = set(flags)
-        if id and self.python_format:
-            self.flags.add('python-format')
-        else:
-            self.flags.discard('python-format')
+
         self.auto_comments = list(distinct(auto_comments))
         self.user_comments = list(distinct(user_comments))
         if isinstance(previous_id, string_types):
@@ -112,6 +109,13 @@ def __init__(self, id, string=u'', locations=(), flags=(), auto_comments=(),
         self.lineno = lineno
         self.context = context
 
+    def determine_python_format(self):
+        """Sets python-format flag if message contains a format string"""
+        if self.id and self.python_format:
+            self.flags.add('python-format')
+        else:
+            self.flags.discard('python-format')
+
     def __repr__(self):
         return '<%s %r (flags: %r)>' % (type(self).__name__, self.id,
                                         list(self.flags))

babel/messages/extract.py

@@ -65,7 +65,7 @@ def extract_from_dir(dirname=None, method_map=DEFAULT_MAPPING,
     """Extract messages from any source files found in the given directory.
 
     This function generates tuples of the form ``(filename, lineno, message,
-    comments, context)``.
+    comments, context, flags)``.
 
     Which extraction method is used per file is determined by the `method_map`
     parameter, which maps extended glob patterns to extraction method names.
@@ -154,23 +154,23 @@ def extract_from_dir(dirname=None, method_map=DEFAULT_MAPPING,
                             options = odict
                     if callback:
                         callback(filename, method, options)
-                    for lineno, message, comments, context in \
+                    for lineno, message, comments, context, flags in \
                           extract_from_file(method, filepath,
                                             keywords=keywords,
                                             comment_tags=comment_tags,
                                             options=options,
                                             strip_comment_tags=
                                                 strip_comment_tags):
-                        yield filename, lineno, message, comments, context
+                        yield filename, lineno, message, comments, context, flags
                     break
 
 
 def extract_from_file(method, filename, keywords=DEFAULT_KEYWORDS,
                       comment_tags=(), options=None, strip_comment_tags=False):
     """Extract messages from a specific file.
 
-    This function returns a list of tuples of the form ``(lineno, funcname,
-    message)``.
+    This function returns a list of tuples of the form
+    ``(lineno, messages, comments, context, flags)``.
 
     :param filename: the path to the file to extract messages from
     :param method: a string specifying the extraction method (.e.g. "python")
@@ -197,7 +197,8 @@ def extract(method, fileobj, keywords=DEFAULT_KEYWORDS, comment_tags=(),
     """Extract messages from the given file-like object using the specified
     extraction method.
 
-    This function returns tuples of the form ``(lineno, message, comments)``.
+    This generator function yields tuples of the form
+    ``(lineno, messages, comments, context, flags)``.
 
     The implementation dispatches the actual extraction to plugins, based on the
     value of the ``method`` parameter.
@@ -210,7 +211,7 @@ def extract(method, fileobj, keywords=DEFAULT_KEYWORDS, comment_tags=(),
     >>> from StringIO import StringIO
     >>> for message in extract('python', StringIO(source)):
     ...     print message
-    (3, u'Hello, world!', [], None)
+    (3, u'Hello, world!', [], None, ())
 
     :param method: a string specifying the extraction method (.e.g. "python");
                    if this is a simple name, the extraction function will be
@@ -261,10 +262,17 @@ def extract(method, fileobj, keywords=DEFAULT_KEYWORDS, comment_tags=(),
     if func is None:
         raise ValueError('Unknown extraction method %r' % method)
 
-    results = func(fileobj, keywords.keys(), comment_tags,
-                   options=options or {})
+    for result in func(fileobj, keywords.keys(), comment_tags,
+                       options=options or {}):
+        flags = ()
+        if len(result) == 4:
+            lineno, funcname, messages, comments = result
+        elif len(result) == 5:
+            lineno, funcname, messages, comments, flags = result
+        else:
+            raise ValueError(
+                'Extraction function must yield tuples with 4 or 5 values')
 
-    for lineno, funcname, messages, comments in results:
         if funcname:
             spec = keywords[funcname] or (1,)
         else:
@@ -315,7 +323,7 @@ def extract(method, fileobj, keywords=DEFAULT_KEYWORDS, comment_tags=(),
 
         if strip_comment_tags:
             _strip_comment_tags(comments, comment_tags)
-        yield lineno, messages, comments, context
+        yield lineno, messages, comments, context, flags
 
 
 def extract_nothing(fileobj, keywords, comment_tags, options):
@@ -408,7 +416,7 @@ def extract_python(fileobj, keywords, comment_tags, options):
                     translator_comments = []
 
                 yield (message_lineno, funcname, messages,
-                       [comment[1] for comment in translator_comments])
+                       [comment[1] for comment in translator_comments], ())
 
                 funcname = lineno = message_lineno = None
                 call_stack = -1

babel/messages/frontend.py

@@ -302,9 +302,10 @@ def callback(filename, method, options):
                                              callback=callback,
                                              strip_comment_tags=
                                                 self.strip_comments)
-                for filename, lineno, message, comments, context in extracted:
+                for filename, lineno, message, comments, context, flags \
+                        in extracted:
                     filepath = os.path.normpath(os.path.join(dirname, filename))
-                    catalog.add(message, None, [(filepath, lineno)],
+                    catalog.add(message, None, [(filepath, lineno)], flags=flags,
                                 auto_comments=comments, context=context)
 
             log.info('writing PO template file to %s' % self.output_file)
@@ -916,9 +917,9 @@ def callback(filename, method, options):
                                          callback=callback,
                                          strip_comment_tags=
                                             options.strip_comment_tags)
-            for filename, lineno, message, comments, context in extracted:
+            for filename, lineno, message, comments, context, flags in extracted:
                 filepath = os.path.normpath(os.path.join(dirname, filename))
-                catalog.add(message, None, [(filepath, lineno)],
+                catalog.add(message, None, [(filepath, lineno)], flags=flags,
                             auto_comments=comments, context=context)
 
         catalog_charset = catalog.charset

babel/messages/pofile.py

@@ -341,9 +341,9 @@ def write_po(fileobj, catalog, width=76, no_location=False, omit_header=False,
     message catalog to the provided file-like object.
 
     >>> catalog = Catalog()
-    >>> catalog.add(u'foo %(name)s', locations=[('main.py', 1)],
+    >>> message = catalog.add(u'foo %(name)s', locations=[('main.py', 1)],
     ...             flags=('fuzzy',))
-    <Message...>
+    >>> message.determine_python_format()
     >>> catalog.add((u'bar', u'baz'), locations=[('main.py', 3)])
     <Message...>
     >>> from io import BytesIO