[0.29] Add --module-name argument to cython command (GH-4906)

Backport of #4548

It can be useful to specify the module name for the output file
directly, rather than working it out from the enclosing file tree -
particularly for out of tree build systems, like Meson.

See background in
rgommers/scipy#31 (comment)

Cython/Compiler/CmdLine.py

@@ -50,6 +50,9 @@
   --warning-extra, -Wextra       Enable extra warnings
   -X, --directive <name>=<value>[,<name=value,...] Overrides a compiler directive
   -E, --compile-time-env name=value[,<name=value,...] Provides compile time env like DEF would do.
+  --module-name                  Fully qualified module name. If not given, it is deduced from the
+                                 import path if source file is in a package, or equals the
+                                 filename otherwise.
 """
 
 
@@ -190,6 +193,8 @@ def get_param(option):
                 except ValueError as e:
                     sys.stderr.write("Error in compile-time-env: %s\n" % e.args[0])
                     sys.exit(1)
+            elif option == "--module-name":
+                options.module_name = pop_value()
             elif option.startswith('--debug'):
                 option = option[2:].replace('-', '_')
                 from . import DebugFlags
@@ -202,6 +207,7 @@ def get_param(option):
                 sys.stdout.write(usage)
                 sys.exit(0)
             else:
+                sys.stderr.write(usage)
                 sys.stderr.write("Unknown compiler flag: %s\n" % option)
                 sys.exit(1)
         else:
@@ -218,7 +224,16 @@ def get_param(option):
         bad_usage()
     if Options.embed and len(sources) > 1:
         sys.stderr.write(
-            "cython: Only one source file allowed when using -embed\n")
+            "cython: Only one source file allowed when using --embed\n")
         sys.exit(1)
+    if options.module_name:
+        if options.timestamps:
+            sys.stderr.write(
+                "cython: Cannot use --module-name with --timestamps\n")
+            sys.exit(1)
+        if len(sources) > 1:
+            sys.stderr.write(
+                "cython: Only one source file allowed when using --module-name\n")
+            sys.exit(1)
     return options, sources
 

Cython/Compiler/Main.py

@@ -735,6 +735,9 @@ def compile_multiple(sources, options):
     a CompilationResultSet. Performs timestamp checking and/or recursion
     if these are specified in the options.
     """
+    if options.module_name and len(sources) > 1:
+        raise RuntimeError('Full module name can only be set '
+                           'for single source compilation')
     # run_pipeline creates the context
     # context = options.create_context()
     sources = [os.path.abspath(source) for source in sources]
@@ -753,8 +756,9 @@ def compile_multiple(sources, options):
             if (not timestamps) or out_of_date:
                 if verbose:
                     sys.stderr.write("Compiling %s\n" % source)
-
-                result = run_pipeline(source, options, context=context)
+                result = run_pipeline(source, options,
+                                      full_module_name=options.module_name,
+                                      context=context)
                 results.add(source, result)
                 # Compiling multiple sources in one context doesn't quite
                 # work properly yet.
@@ -900,5 +904,6 @@ def main(command_line = 0):
     build_dir=None,
     cache=None,
     create_extension=None,
+    module_name=None,
     np_pythran=False
 )

Cython/Compiler/Tests/TestCmdLine.py

@@ -1,5 +1,6 @@
 
 import sys
+import re
 from unittest import TestCase
 try:
     from StringIO import StringIO
@@ -10,6 +11,18 @@
 from ..CmdLine import parse_command_line
 
 
+def check_global_options(expected_options, white_list=[]):
+    """
+    returns error message of "" if check Ok
+    """
+    no_value = object()
+    for name, orig_value in expected_options.items():
+        if name not in white_list:
+            if getattr(Options, name, no_value) != orig_value:
+                return "error in option " + name
+    return ""
+
+
 class CmdLineParserTest(TestCase):
     def setUp(self):
         backup = {}
@@ -23,6 +36,17 @@ def tearDown(self):
             if getattr(Options, name, no_value) != orig_value:
                 setattr(Options, name, orig_value)
 
+    def check_default_global_options(self, white_list=[]):
+        self.assertEqual(check_global_options(self._options_backup, white_list), "")
+
+    def check_default_options(self, options, white_list=[]):
+        from ..Main import CompilationOptions, default_options
+        default_options = CompilationOptions(default_options)
+        no_value = object()
+        for name in default_options.__dict__.keys():
+            if name not in white_list:
+                self.assertEqual(getattr(options, name, no_value), getattr(default_options, name), msg="error in option " + name)
+
     def test_short_options(self):
         options, sources = parse_command_line([
             '-V', '-l', '-+', '-t', '-v', '-v', '-v', '-p', '-D', '-a', '-3',
@@ -98,21 +122,49 @@ def test_options_with_values(self):
         self.assertTrue(options.gdb_debug)
         self.assertEqual(options.output_dir, '/gdb/outdir')
 
+    def test_module_name(self):
+        options, sources = parse_command_line([
+            'source.pyx'
+        ])
+        self.assertEqual(options.module_name, None)
+        self.check_default_global_options()
+        self.check_default_options(options)
+        options, sources = parse_command_line([
+            '--module-name', 'foo.bar',
+            'source.pyx'
+        ])
+        self.assertEqual(options.module_name, 'foo.bar')
+        self.check_default_global_options()
+        self.check_default_options(options, ['module_name'])
+
     def test_errors(self):
-        def error(*args):
+        def error(args, regex=None):
             old_stderr = sys.stderr
             stderr = sys.stderr = StringIO()
             try:
                 self.assertRaises(SystemExit, parse_command_line, list(args))
             finally:
                 sys.stderr = old_stderr
-            self.assertTrue(stderr.getvalue())
-
-        error('-1')
-        error('-I')
-        error('--version=-a')
-        error('--version=--annotate=true')
-        error('--working')
-        error('--verbose=1')
-        error('--verbose=1')
-        error('--cleanup')
+            msg = stderr.getvalue().strip()
+            self.assertTrue(msg)
+            if regex:
+                self.assertTrue(re.search(regex, msg),
+                                '"%s" does not match search "%s"' %
+                                (msg, regex))
+
+        error(['-1'],
+              'Unknown compiler flag: -1')
+        error(['-I'])
+        error(['--version=-a'])
+        error(['--version=--annotate=true'])
+        error(['--working'])
+        error(['--verbose=1'])
+        error(['--cleanup'])
+        error(['--debug-disposal-code-wrong-name', 'file3.pyx'],
+              "Unknown debug flag: debug_disposal_code_wrong_name")
+        error(['--module-name', 'foo.pyx'])
+        error(['--module-name', 'foo.bar'])
+        error(['--module-name', 'foo.bar', 'foo.pyx', 'bar.pyx'],
+              "Only one source file allowed when using --module-name")
+        error(['--module-name', 'foo.bar', '--timestamps', 'foo.pyx'],
+              "Cannot use --module-name with --timestamps")

tests/compile/module_name_arg.srctree

@@ -0,0 +1,52 @@
+# Test that we can set module name with --module-name arg to cython
+CYTHON a.pyx
+CYTHON --module-name w b.pyx
+CYTHON --module-name my_module.submod.x c.pyx
+PYTHON setup.py build_ext --inplace
+PYTHON checks.py
+
+######## checks.py ########
+
+from importlib import import_module
+
+try:
+    exc = ModuleNotFoundError
+except NameError:
+    exc = ImportError
+
+for module_name, should_import in (
+    ('a', True),
+    ('b', False),
+    ('w', True),
+    ('my_module.submod.x', True),
+    ('c', False),
+    ):
+    try:
+        import_module(module_name)
+    except exc:
+        if should_import:
+            assert False, "Cannot import module " + module_name
+    else:
+        if not should_import:
+            assert False, ("Can import module " + module_name +
+                           " but import should not be possible")
+
+
+######## setup.py ########
+
+from distutils.core import setup
+from distutils.extension import Extension
+
+setup(
+  ext_modules = [
+    Extension("a", ["a.c"]),
+    Extension("w", ["b.c"]),
+    Extension("my_module.submod.x", ["c.c"]),
+  ],
+)
+
+######## a.pyx ########
+######## b.pyx ########
+######## c.pyx ########
+######## my_module/__init__.py ########
+######## my_module/submod/__init__.py ########