Speed up server startup by running asset compilation in parallel

pavelib/assets.py

@@ -10,6 +10,7 @@
 import traceback
 from .utils.envs import Env
 from .utils.cmd import cmd, django_cmd
+from .utils.thread import run_threaded
 
 COFFEE_DIRS = ['lms', 'cms', 'common']
 SASS_LOAD_PATHS = ['./common/static/sass']
@@ -146,8 +147,10 @@ def compile_templated_sass(systems, settings):
     `systems` is a list of systems (e.g. 'lms' or 'studio' or both)
     `settings` is the Django settings module to use.
     """
-    for sys in systems:
-        sh(django_cmd(sys, settings, 'preprocess_assets'))
+    run_threaded([
+        (sh, [django_cmd(sys, settings, 'preprocess_assets')])
+        for sys in systems
+    ])
 
 
 def process_xmodule_assets():
@@ -163,8 +166,10 @@ def collect_assets(systems, settings):
     `systems` is a list of systems (e.g. 'lms' or 'studio' or both)
     `settings` is the Django settings module to use.
     """
-    for sys in systems:
-        sh(django_cmd(sys, settings, "collectstatic --noinput > /dev/null"))
+    run_threaded([
+        (sh, [django_cmd(sys, settings, "collectstatic --noinput > /dev/null")])
+        for sys in systems
+    ])
 
 
 @task
@@ -221,11 +226,13 @@ def update_assets(args):
         help="Watch files for changes",
     )
     args = parser.parse_args(args)
-
+    # this needs to run before other files are processed
     compile_templated_sass(args.system, args.settings)
-    process_xmodule_assets()
-    compile_coffeescript()
-    compile_sass(args.debug)
+    run_threaded([
+        process_xmodule_assets,
+        compile_coffeescript,
+        (compile_sass, (args.debug,)),
+    ])
 
     if args.collect:
         collect_assets(args.system, args.settings)

pavelib/prereqs.py

@@ -7,7 +7,7 @@
 from distutils import sysconfig
 from paver.easy import *
 from .utils.envs import Env
-
+from .utils.thread import run_threaded
 
 PREREQS_MD5_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache')
 NPM_REGISTRY = "http://registry.npmjs.org/"
@@ -114,6 +114,8 @@ def install_prereqs():
     if os.environ.get("NO_PREREQ_INSTALL", False):
         return
 
-    prereq_cache("Ruby prereqs", ["Gemfile"], install_ruby_prereqs)
-    prereq_cache("Node prereqs", ["package.json"], install_node_prereqs)
-    prereq_cache("Python prereqs", PYTHON_REQ_FILES + [sysconfig.get_python_lib()], install_python_prereqs)
+    run_threaded([
+        (prereq_cache, ("Ruby prereqs", ["Gemfile"], install_ruby_prereqs)),
+        (prereq_cache, ("Node prereqs", ["package.json"], install_node_prereqs)),
+        (prereq_cache, ("Python prereqs", PYTHON_REQ_FILES, install_python_prereqs)),
+    ])

pavelib/utils/thread.py

@@ -0,0 +1,41 @@
+"""
+Helper functions for managing threads.
+"""
+import multiprocessing
+
+
+def _get_func_args(func):
+    args = []
+    kwargs = {}
+    if not hasattr(func, '__iter__'):
+        return func, args, kwargs
+    f = func[0]
+    if len(func) > 1:
+        args = func[1]
+        if len(func) > 2:
+            kwargs = func[2]
+    return f, args, kwargs
+
+
+def run_threaded(funcs_to_run):
+    """
+    Run a list of functions in parallel, returning when all have completed.
+    expects a list of (function, args, kwargs)
+    """
+    try:
+        from concurrent.futures import ThreadPoolExecutor
+    except ImportError:
+        from threading import Thread
+        threads = []
+        for func in funcs_to_run:
+            f, args, kwargs = _get_func_args(func)
+            t = Thread(target=f, args=args, kwargs=kwargs)
+            t.start()
+            threads.append(t)
+        for t in threads:
+            t.join()
+    else:
+        with ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor:
+            for func in funcs_to_run:
+                f, args, kwargs = _get_func_args(func)
+                executor.submit(f, *args, **kwargs)

requirements/edx/base.txt

@@ -48,7 +48,7 @@ networkx==1.7
 nltk==2.0.4
 oauthlib==0.5.1
 paramiko==1.9.0
-path.py==3.0.1
+path.py==5.1
 Pillow==1.7.8
 pip>=1.4
 polib==1.0.3

requirements/edx/paver.txt

@@ -1,5 +1,5 @@
 # Requirements to run Paver
-Paver==1.2.1
+Paver==1.2.2
 psutil==1.2.1
 lazy==1.1
-path.py==3.0.1
+path.py==5.1