Issue spotify#838: setup.py: improve packaging of 'bin' scripts with 'entry_points/console_scripts'

Author: Stefaan Lippens

.travis.yml

@@ -27,6 +27,7 @@ cache:
   - $HOME/.pip-cache
 
 install:
+  - pip install .
   - pip install tox
 
 before_script:

MANIFEST.in

@@ -1,4 +1,3 @@
-include bin/luigid
 include README.rst
 include LICENSE
 include examples/*.py

bin/luigi

@@ -1,6 +1,14 @@
 #!/usr/bin/env python
 
-import luigi
+import sys
+import warnings
+import luigi.cmdline
+
+
+def main(argv):
+    warnings.warn("'bin/luigi' has moved to console script 'luigi'", DeprecationWarning)
+    luigi.cmdline.luigi_run(argv)
+
 
 if __name__ == '__main__':
-    luigi.run(use_dynamic_argparse=True)
+    main(sys.argv[1:])

bin/luigid

@@ -1,40 +1,14 @@
 #!/usr/bin/env python
 
-import luigi.server
-import luigi.process
-import os
-import luigi.configuration
-import optparse
-import logging
 import sys
+import warnings
+import luigi.cmdline
 
-def main(argv):
-    parser = optparse.OptionParser()
-    parser.add_option(u'--background', help=u'Run in background mode', action='store_true')
-    parser.add_option(u'--pidfile', help=u'Write pidfile')
-    parser.add_option(u'--logdir', help=u'log directory')
-    parser.add_option(u'--state-path', help=u'Pickled state file')
-    parser.add_option(u'--address', help=u'Listening interface')
-    parser.add_option(u'--port', default=8082, help=u'Listening port')
-        
-    opts, args = parser.parse_args(argv)
 
-    if opts.state_path:
-        config = luigi.configuration.get_config()
-        config.set('scheduler', 'state-path', opts.state_path)
+def main(argv):
+    warnings.warn("'bin/luigid' has moved to console script 'luigid'", DeprecationWarning)
+    luigi.cmdline.luigid(argv)
 
-    if opts.background:
-        # daemonize sets up logging to spooled log files
-        logging.getLogger().setLevel(logging.INFO)
-        luigi.process.daemonize(luigi.server.run, api_port=opts.port,
-                                address=opts.address, pidfile=opts.pidfile,
-                                logdir=opts.logdir)
-    else:
-        if opts.logdir:
-            logging.basicConfig(level=logging.INFO, format=luigi.process.get_log_format(), filename=os.path.join(opts.logdir, "luigi-server.log"))
-        else:
-            logging.basicConfig(level=logging.INFO, format=luigi.process.get_log_format())
-        luigi.server.run(api_port=opts.port, address=opts.address)
 
 if __name__ == '__main__':
     main(sys.argv[1:])

doc/central_scheduler.rst

@@ -24,7 +24,7 @@ To run the server as a daemon run:
 
 ::
 
-    PYTHONPATH=. python bin/luigid --background --pidfile <PATH_TO_PIDFILE> --logdir <PATH_TO_LOGDIR> --state-path <PATH_TO_STATEFILE>
+    luigid --background --pidfile <PATH_TO_PIDFILE> --logdir <PATH_TO_LOGDIR> --state-path <PATH_TO_STATEFILE>
 
 Note that this requires ``python-daemon``.
 By default, the server starts on port ``8082``

doc/example_top_artists.rst

@@ -238,7 +238,7 @@ If you run
 
 ::
 
-    PYTHONPATH=. python bin/luigid
+    luigid
 
 in the background and then run
 

luigi/cmdline.py

@@ -0,0 +1,43 @@
+import os
+import optparse
+import logging
+import sys
+
+import luigi.server
+import luigi.process
+import luigi.configuration
+import luigi.interface
+
+
+def luigi_run(argv=sys.argv[1:]):
+    luigi.interface.run(argv, use_dynamic_argparse=True)
+
+
+def luigid(argv=sys.argv[1:]):
+    parser = optparse.OptionParser()
+    parser.add_option(u'--background', help=u'Run in background mode', action='store_true')
+    parser.add_option(u'--pidfile', help=u'Write pidfile')
+    parser.add_option(u'--logdir', help=u'log directory')
+    parser.add_option(u'--state-path', help=u'Pickled state file')
+    parser.add_option(u'--address', help=u'Listening interface')
+    parser.add_option(u'--port', default=8082, help=u'Listening port')
+
+    opts, args = parser.parse_args(argv)
+
+    if opts.state_path:
+        config = luigi.configuration.get_config()
+        config.set('scheduler', 'state-path', opts.state_path)
+
+    if opts.background:
+        # daemonize sets up logging to spooled log files
+        logging.getLogger().setLevel(logging.INFO)
+        luigi.process.daemonize(luigi.server.run, api_port=opts.port,
+                                address=opts.address, pidfile=opts.pidfile,
+                                logdir=opts.logdir)
+    else:
+        if opts.logdir:
+            logging.basicConfig(level=logging.INFO, format=luigi.process.get_log_format(),
+                                filename=os.path.join(opts.logdir, "luigi-server.log"))
+        else:
+            logging.basicConfig(level=logging.INFO, format=luigi.process.get_log_format())
+        luigi.server.run(api_port=opts.port, address=opts.address)

luigi/server.py

@@ -38,6 +38,7 @@
 
 from luigi.scheduler import CentralPlannerScheduler
 
+
 logger = logging.getLogger("luigi.server")
 
 

bin/deps.py luigi/tools/deps.py

@@ -88,7 +88,7 @@ def find_deps_cli():
     return find_deps(task, upstream)
 
 
-if __name__ == '__main__':
+def main():
     deps = find_deps_cli()
     for d in deps:
         task_name = d
@@ -105,3 +105,7 @@ def find_deps_cli():
             task_output = "to be determined"
         print(u"""   TASK: {0}
                        : {1}""".format(task_name, task_output))
+
+
+if __name__ == '__main__':
+    main()

bin/luigi-grep.py luigi/tools/luigi_grep.py

@@ -7,15 +7,6 @@
 
 from luigi import six
 
-parser = argparse.ArgumentParser(
-    "luigi-grep is used to search for workflows using the luigi scheduler's json api")
-parser.add_argument(
-    "--scheduler-host", default="localhost", help="hostname of the luigi scheduler")
-parser.add_argument(
-    "--scheduler-port", default="8082", help="port of the luigi scheduler")
-parser.add_argument("--prefix", help="prefix of a task query to search for", default=None)
-parser.add_argument("--status", help="search for jobs with the given status", default=None)
-
 
 class LuigiGrep(object):
 
@@ -29,7 +20,7 @@ def graph_url(self):
 
     def _fetch_json(self):
         """Returns the json representation of the dep graph"""
-        print "Fetching from url: " + self.graph_url
+        print("Fetching from url: " + self.graph_url)
         resp = urllib2.urlopen(self.graph_url).read()
         return json.loads(resp)
 
@@ -61,7 +52,17 @@ def status_search(self, status):
             if job_info['status'].lower() == status.lower():
                 yield self._build_results(jobs, job)
 
-if __name__ == '__main__':
+
+def main():
+    parser = argparse.ArgumentParser(
+        "luigi-grep is used to search for workflows using the luigi scheduler's json api")
+    parser.add_argument(
+        "--scheduler-host", default="localhost", help="hostname of the luigi scheduler")
+    parser.add_argument(
+        "--scheduler-port", default="8082", help="port of the luigi scheduler")
+    parser.add_argument("--prefix", help="prefix of a task query to search for", default=None)
+    parser.add_argument("--status", help="search for jobs with the given status", default=None)
+
     args = parser.parse_args()
     grep = LuigiGrep(args.scheduler_host, args.scheduler_port)
 
@@ -77,3 +78,6 @@ def status_search(self, status):
             print("  status={status}".format(status=status))
             for job in jobs:
                 print("    {job}".format(job=job))
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -69,10 +69,14 @@ def get_static_files(path):
     package_data={
         'luigi': luigi_package_data
     },
-    scripts=[
-        'bin/luigid',
-        'bin/luigi'
-    ],
+    entry_points={
+        'console_scripts': [
+            'luigi = luigi.cmdline:luigi_run',
+            'luigid = luigi.cmdline:luigid',
+            'luigi-grep = luigi.tools.luigi_grep:main',
+            'luigi-deps = luigi.tools.deps:main',
+        ]
+    },
     install_requires=install_requires,
     classifiers=[
         'Development Status :: 5 - Production/Stable',

test/cmdline_test.py

@@ -177,6 +177,12 @@ def test_bin_luigi(self):
         self._run_cmdline(args)
         self.assertTrue(t.exists())
 
+    def test_console_script_luigi(self):
+        t = luigi.LocalTarget(is_tmp=True)
+        args = ['luigi', '--module', 'cmdline_test', 'WriteToFile', '--filename', t.path, '--local-scheduler', '--no-lock']
+        self._run_cmdline(args)
+        self.assertTrue(t.exists())
+
     def test_direct_python(self):
         t = luigi.LocalTarget(is_tmp=True)
         args = ['python', 'test/cmdline_test.py', 'WriteToFile', '--filename', t.path, '--local-scheduler', '--no-lock']