Add --module-name argument to cython command (GH-4548)

It can be useful to specify the module name for the output file
directly, rather than working it out from the enclosing file tree -
particularly for out of tree build systems, like Meson.

See background in
https://github.com/rgommers/scipy/issues/31#issuecomment-1002662816

5 files changed, 123 insertions, 15 deletions

diff --git a/Cython/Compiler/CmdLine.py b/Cython/Compiler/CmdLine.py
index ffff6a61c..894eacd0a 100644
--- a/Cython/Compiler/CmdLine.py
+++ b/Cython/Compiler/CmdLine.py
@@ -145,6 +145,11 @@ def create_cython_argparser():
                       dest='compile_time_env', type=str,
                       action=ParseCompileTimeEnvAction,
                       help='Provides compile time env like DEF would do.')
+    parser.add_argument("--module-name",
+                      dest='module_name', type=str, action='store',
+                      help='Fully qualified module name. If not given, is '
+                           'deduced from the import path if source file is in '
+                           'a package, or equals the filename otherwise.')
     parser.add_argument('sources', nargs='*', default=[])
 
     # TODO: add help
@@ -222,5 +227,10 @@ def parse_command_line(args):
     if len(sources) == 0 and not options.show_version:
         parser.error("cython: Need at least one source file\n")
     if Options.embed and len(sources) > 1:
-        parser.error("cython: Only one source file allowed when using -embed\n")
+        parser.error("cython: Only one source file allowed when using --embed\n")
+    if options.module_name:
+        if options.timestamps:
+            parser.error("cython: Cannot use --module-name with --timestamps\n")
+        if len(sources) > 1:
+            parser.error("cython: Only one source file allowed when using --module-name\n")
     return options, sources
diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py
index 764d9af21..0ea5f7748 100644
--- a/Cython/Compiler/Main.py
+++ b/Cython/Compiler/Main.py
@@ -583,6 +583,9 @@ def compile_multiple(sources, options):
     a CompilationResultSet. Performs timestamp checking and/or recursion
     if these are specified in the options.
     """
+    if len(sources) > 1 and options.module_name:
+        raise RuntimeError('Full module name can only be set '
+                           'for single source compilation')
     # run_pipeline creates the context
     # context = Context.from_options(options)
     sources = [os.path.abspath(source) for source in sources]
@@ -601,8 +604,9 @@ def compile_multiple(sources, options):
             if (not timestamps) or out_of_date:
                 if verbose:
                     sys.stderr.write("Compiling %s\n" % source)
-
-                result = run_pipeline(source, options, context=context)
+                result = run_pipeline(source, options,
+                                      full_module_name=options.module_name,
+                                      context=context)
                 results.add(source, result)
                 # Compiling multiple sources in one context doesn't quite
                 # work properly yet.
diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.py
index 97f288905..ea0b95c90 100644
--- a/Cython/Compiler/Options.py
+++ b/Cython/Compiler/Options.py
@@ -757,6 +757,7 @@ default_options = dict(
     formal_grammar=False,
     gdb_debug=False,
     compile_time_env=None,
+    module_name=None,
     common_utility_include_dir=None,
     output_dir=None,
     build_dir=None,
diff --git a/Cython/Compiler/Tests/TestCmdLine.py b/Cython/Compiler/Tests/TestCmdLine.py
index 5953112dc..6c74fe3a2 100644
--- a/Cython/Compiler/Tests/TestCmdLine.py
+++ b/Cython/Compiler/Tests/TestCmdLine.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import re
 from unittest import TestCase
 try:
     from StringIO import StringIO
@@ -495,22 +496,62 @@ class CmdLineParserTest(TestCase):
         self.check_default_global_options()
         self.check_default_options(options, ['compiler_directives'])
 
+    def test_module_name(self):
+        options, sources = parse_command_line([
+            'source.pyx'
+        ])
+        self.assertEqual(options.module_name, None)
+        self.check_default_global_options()
+        self.check_default_options(options)
+        options, sources = parse_command_line([
+            '--module-name', 'foo.bar',
+            'source.pyx'
+        ])
+        self.assertEqual(options.module_name, 'foo.bar')
+        self.check_default_global_options()
+        self.check_default_options(options, ['module_name'])
+
     def test_errors(self):
-        def error(*args):
+        def error(args, regex=None):
             old_stderr = sys.stderr
             stderr = sys.stderr = StringIO()
             try:
                 self.assertRaises(SystemExit, parse_command_line, list(args))
             finally:
                 sys.stderr = old_stderr
-            self.assertTrue(stderr.getvalue())
-
-        error('-1')
-        error('-I')
-        error('--version=-a')
-        error('--version=--annotate=true')
-        error('--working')
-        error('--verbose=1')
-        error('--verbose=1')
-        error('--cleanup')
-        error('--debug-disposal-code-wrong-name', 'file3.pyx')
+            msg = stderr.getvalue()
+            err_msg = 'Message "{}"'.format(msg.strip())
+            self.assertTrue(msg.startswith('usage: '),
+                            '%s does not start with "usage :"' % err_msg)
+            self.assertTrue(': error: ' in msg,
+                            '%s does not contain ": error :"' % err_msg)
+            if regex:
+                self.assertTrue(re.search(regex, msg),
+                                '%s does not match search "%s"' %
+                                (err_msg, regex))
+
+        error(['-1'],
+              'unknown option -1')
+        error(['-I'],
+              'argument -I/--include-dir: expected one argument')
+        error(['--version=-a'],
+              "argument -V/--version: ignored explicit argument '-a'")
+        error(['--version=--annotate=true'],
+              "argument -V/--version: ignored explicit argument "
+              "'--annotate=true'")
+        error(['--working'],
+              "argument -w/--working: expected one argument")
+        error(['--verbose=1'],
+              "argument -v/--verbose: ignored explicit argument '1'")
+        error(['--cleanup'],
+              "argument --cleanup: expected one argument")
+        error(['--debug-disposal-code-wrong-name', 'file3.pyx'],
+              "unknown option --debug-disposal-code-wrong-name")
+        error(['--module-name', 'foo.pyx'],
+              "Need at least one source file")
+        error(['--module-name', 'foo.bar'],
+              "Need at least one source file")
+        error(['--module-name', 'foo.bar', 'foo.pyx', 'bar.pyx'],
+              "Only one source file allowed when using --module-name")
+        error(['--module-name', 'foo.bar', '--timestamps', 'foo.pyx'],
+              "Cannot use --module-name with --timestamps")
diff --git a/tests/compile/module_name_arg.srctree b/tests/compile/module_name_arg.srctree
new file mode 100644
index 000000000..81e75b008
--- /dev/null
+++ b/tests/compile/module_name_arg.srctree
@@ -0,0 +1,52 @@
+# Test that we can set module name with --module-name arg to cython
+CYTHON a.pyx
+CYTHON --module-name w b.pyx
+CYTHON --module-name my_module.submod.x c.pyx
+PYTHON setup.py build_ext --inplace
+PYTHON checks.py
+
+######## checks.py ########
+
+from importlib import import_module
+
+try:
+    exc = ModuleNotFoundError
+except NameError:
+    exc = ImportError
+
+for module_name, should_import in (
+    ('a', True),
+    ('b', False),
+    ('w', True),
+    ('my_module.submod.x', True),
+    ('c', False),
+    ):
+    try:
+        import_module(module_name)
+    except exc:
+        if should_import:
+            assert False, "Cannot import module " + module_name
+    else:
+        if not should_import:
+            assert False, ("Can import module " + module_name +
+                           " but import should not be possible")
+
+
+######## setup.py ########
+
+from distutils.core import setup
+from distutils.extension import Extension
+
+setup(
+  ext_modules = [
+    Extension("a", ["a.c"]),
+    Extension("w", ["b.c"]),
+    Extension("my_module.submod.x", ["c.c"]),
+  ],
+)
+
+######## a.pyx ########
+######## b.pyx ########
+######## c.pyx ########
+######## my_module/__init__.py ########
+######## my_module/submod/__init__.py ########