From 296487878273dadc621edf7fb5c3563f530ff275 Mon Sep 17 00:00:00 2001
From: "Dave St.Germain" <dstgermain@edx.org>
Date: Thu, 27 Mar 2014 14:47:21 -0400
Subject: [PATCH] Attempt to speed up server startup by running asset
 compilation in parallel

---
 pavelib/assets.py          | 23 +++++++++++++--------
 pavelib/prereqs.py         | 10 ++++++----
 pavelib/utils/thread.py    | 41 ++++++++++++++++++++++++++++++++++++++
 requirements/edx/base.txt  |  2 +-
 requirements/edx/paver.txt |  4 ++--
 5 files changed, 65 insertions(+), 15 deletions(-)
 create mode 100644 pavelib/utils/thread.py

diff --git a/pavelib/assets.py b/pavelib/assets.py
index a9acc69980ea..d99fc0b3ed6d 100644
--- a/pavelib/assets.py
+++ b/pavelib/assets.py
@@ -10,6 +10,7 @@
 import traceback
 from .utils.envs import Env
 from .utils.cmd import cmd, django_cmd
+from .utils.thread import run_threaded
 
 COFFEE_DIRS = ['lms', 'cms', 'common']
 SASS_LOAD_PATHS = ['./common/static/sass']
@@ -146,8 +147,10 @@ def compile_templated_sass(systems, settings):
     `systems` is a list of systems (e.g. 'lms' or 'studio' or both)
     `settings` is the Django settings module to use.
     """
-    for sys in systems:
-        sh(django_cmd(sys, settings, 'preprocess_assets'))
+    run_threaded([
+        (sh, [django_cmd(sys, settings, 'preprocess_assets')])
+        for sys in systems
+    ])
 
 
 def process_xmodule_assets():
@@ -163,8 +166,10 @@ def collect_assets(systems, settings):
     `systems` is a list of systems (e.g. 'lms' or 'studio' or both)
     `settings` is the Django settings module to use.
     """
-    for sys in systems:
-        sh(django_cmd(sys, settings, "collectstatic --noinput > /dev/null"))
+    run_threaded([
+        (sh, [django_cmd(sys, settings, "collectstatic --noinput > /dev/null")])
+        for sys in systems
+    ])
 
 
 @task
@@ -221,11 +226,13 @@ def update_assets(args):
         help="Watch files for changes",
     )
     args = parser.parse_args(args)
-
+    # this needs to run before other files are processed
     compile_templated_sass(args.system, args.settings)
-    process_xmodule_assets()
-    compile_coffeescript()
-    compile_sass(args.debug)
+    run_threaded([
+        process_xmodule_assets,
+        compile_coffeescript,
+        (compile_sass, (args.debug,)),
+    ])
 
     if args.collect:
         collect_assets(args.system, args.settings)
diff --git a/pavelib/prereqs.py b/pavelib/prereqs.py
index 220ab9cd5b75..d63064288851 100644
--- a/pavelib/prereqs.py
+++ b/pavelib/prereqs.py
@@ -7,7 +7,7 @@
 from distutils import sysconfig
 from paver.easy import *
 from .utils.envs import Env
-
+from .utils.thread import run_threaded
 
 PREREQS_MD5_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache')
 NPM_REGISTRY = "http://registry.npmjs.org/"
@@ -114,6 +114,8 @@ def install_prereqs():
     if os.environ.get("NO_PREREQ_INSTALL", False):
         return
 
-    prereq_cache("Ruby prereqs", ["Gemfile"], install_ruby_prereqs)
-    prereq_cache("Node prereqs", ["package.json"], install_node_prereqs)
-    prereq_cache("Python prereqs", PYTHON_REQ_FILES + [sysconfig.get_python_lib()], install_python_prereqs)
+    run_threaded([
+        (prereq_cache, ("Ruby prereqs", ["Gemfile"], install_ruby_prereqs)),
+        (prereq_cache, ("Node prereqs", ["package.json"], install_node_prereqs)),
+        (prereq_cache, ("Python prereqs", PYTHON_REQ_FILES, install_python_prereqs)),
+    ])
diff --git a/pavelib/utils/thread.py b/pavelib/utils/thread.py
new file mode 100644
index 000000000000..e54e7e73d546
--- /dev/null
+++ b/pavelib/utils/thread.py
@@ -0,0 +1,41 @@
+"""
+Helper functions for managing threads.
+"""
+import multiprocessing
+
+
+def _get_func_args(func):
+    args = []
+    kwargs = {}
+    if not hasattr(func, '__iter__'):
+        return func, args, kwargs
+    f = func[0]
+    if len(func) > 1:
+        args = func[1]
+        if len(func) > 2:
+            kwargs = func[2]
+    return f, args, kwargs
+
+
+def run_threaded(funcs_to_run):
+    """
+    Run a list of functions in parallel, returning when all have completed.
+    expects a list of (function, args, kwargs)
+    """
+    try:
+        from concurrent.futures import ThreadPoolExecutor
+    except ImportError:
+        from threading import Thread
+        threads = []
+        for func in funcs_to_run:
+            f, args, kwargs = _get_func_args(func)
+            t = Thread(target=f, args=args, kwargs=kwargs)
+            t.start()
+            threads.append(t)
+        for t in threads:
+            t.join()
+    else:
+        with ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor:
+            for func in funcs_to_run:
+                f, args, kwargs = _get_func_args(func)
+                executor.submit(f, *args, **kwargs)
diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
index 80b3184fab7e..028e285542bc 100644
--- a/requirements/edx/base.txt
+++ b/requirements/edx/base.txt
@@ -48,7 +48,7 @@ networkx==1.7
 nltk==2.0.4
 oauthlib==0.5.1
 paramiko==1.9.0
-path.py==3.0.1
+path.py==5.1
 Pillow==1.7.8
 pip>=1.4
 polib==1.0.3
diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt
index 01ee9edeb039..84980fc0882c 100644
--- a/requirements/edx/paver.txt
+++ b/requirements/edx/paver.txt
@@ -1,5 +1,5 @@
 # Requirements to run Paver
-Paver==1.2.1
+Paver==1.2.2
 psutil==1.2.1
 lazy==1.1
-path.py==3.0.1
+path.py==5.1