add --parallel-render arg which enables parallel rendering with multi…

…ple threads, resolves #90

README.md

@@ -274,6 +274,9 @@ rendering, this is just a shortcut to save you typing an extra command.
 `--exclude-staticfiles`: Do not copy any static files at all, only render output from
 Django views.
 
+`--parallel-render [number of threads]`: Render files in parallel on multiple
+threads, this can speed up rendering. Defaults to `1` thread.
+
 `--generate-redirects`: Attempt to generate static redirects stored in the
 `django.contrib.redirects` app. If you have a redirect from `/old/` to `/new/` using
 this flag will create a static HTML `<meta http-equiv="refresh" content="...">`
@@ -322,6 +325,9 @@ to update most of them, and you don't care if old files remain on the server.
 `--parallel-publish [number of threads]`: Publish files in parallel on multiple
 threads, this can speed up publishing. Defaults to `1` thread.
 
+`--parallel-render [number of threads]`: Render files in parallel on multiple
+threads, this can speed up rendering. Defaults to `1` thread.
+
 `--generate-redirects`: Attempt to generate static redirects stored in the
 `django.contrib.redirects` app. If you have a redirect from `/old/` to `/new/` using
 this flag will create a static HTML `<meta http-equiv="refresh" content="...">`

django_distill/management/commands/distill-local.py

@@ -14,14 +14,12 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('output_dir', nargs='?', type=str)
-        parser.add_argument('--collectstatic', dest='collectstatic',
-                            action='store_true')
+        parser.add_argument('--collectstatic', dest='collectstatic', action='store_true')
         parser.add_argument('--quiet', dest='quiet', action='store_true')
         parser.add_argument('--force', dest='force', action='store_true')
-        parser.add_argument('--exclude-staticfiles', dest='exclude_staticfiles',
-                            action='store_true')
-        parser.add_argument('--generate-redirects', dest='generate_redirects',
-                            action='store_true')
+        parser.add_argument('--exclude-staticfiles', dest='exclude_staticfiles', action='store_true')
+        parser.add_argument('--generate-redirects', dest='generate_redirects', action='store_true')
+        parser.add_argument('--parallel-render', dest='parallel_render', type=int, default=1)
 
     def _quiet(self, *args, **kwargs):
         pass
@@ -33,6 +31,7 @@ def handle(self, *args, **options):
         force = options.get('force')
         exclude_staticfiles = options.get('exclude_staticfiles')
         generate_redirects = options.get('generate_redirects')
+        parallel_render = options.get('parallel_render')
         if quiet:
             stdout = self._quiet
         else:
@@ -82,7 +81,7 @@ def handle(self, *args, **options):
         stdout('')
         stdout('Generating static site into directory: {}'.format(output_dir))
         try:
-            render_to_dir(output_dir, urls_to_distill, stdout)
+            render_to_dir(output_dir, urls_to_distill, stdout, parallel_render=parallel_render)
             if not exclude_staticfiles:
                 copy_static_and_media_files(output_dir, stdout)
         except DistillError as err:

django_distill/management/commands/distill-publish.py

@@ -26,6 +26,7 @@ def add_arguments(self, parser):
         parser.add_argument('--ignore-remote-content', dest='ignore_remote_content', action='store_true')
         parser.add_argument('--parallel-publish', dest='parallel_publish', type=int, default=1)
         parser.add_argument('--generate-redirects', dest='generate_redirects', action='store_true')
+        parser.add_argument('--parallel-render', dest='parallel_render', type=int, default=1)
 
     def _quiet(self, *args, **kwargs):
         pass
@@ -52,6 +53,7 @@ def handle(self, *args, **options):
         quiet = options.get('quiet')
         force = options.get('force')
         generate_redirects = options.get('generate_redirects')
+        parallel_render = options.get('parallel_render')
         if quiet:
             stdout = self._quiet
         else:
@@ -89,7 +91,7 @@ def handle(self, *args, **options):
             msg = 'Generating static site into directory: {}'
             stdout(msg.format(output_dir))
             try:
-                render_to_dir(output_dir, urls_to_distill, stdout)
+                render_to_dir(output_dir, urls_to_distill, stdout, parallel_render=parallel_render)
                 if not exclude_staticfiles:
                     copy_static_and_media_files(output_dir, stdout)
             except DistillError as err:

django_distill/renderer.py

@@ -4,6 +4,7 @@
 import os
 import types
 from shutil import copy2
+from concurrent.futures import ThreadPoolExecutor
 from django.utils import translation
 from django.conf import settings
 from django.urls import include as include_urls, get_resolver
@@ -158,8 +159,9 @@ class DistillRender(object):
         distill_url() and then copies over all static media.
     '''
 
-    def __init__(self, urls_to_distill):
+    def __init__(self, urls_to_distill, parallel_render=1):
         self.urls_to_distill = urls_to_distill
+        self.parallel_render = parallel_render
         self.namespace_map = load_namespace_map()
         # activate the default translation
         translation.activate(settings.LANGUAGE_CODE)
@@ -186,15 +188,25 @@ def render_file(self, view_name, status_codes, view_args, view_kwargs):
         return uri, file_name, render
 
     def render_all_urls(self):
+
+        def _render(item):
+            url, view_name, param_set, status_codes, file_name_base, a, k = item
+            uri = self.generate_uri(url, view_name, param_set)
+            render = self.render_view(uri, status_codes, param_set, a, k)
+            file_name = self._get_filename(file_name_base, uri, param_set)
+            return uri, file_name, render
+
+        to_render = []
         for url, distill_func, file_name_base, status_codes, view_name, a, k in self.urls_to_distill:
             for param_set in self.get_uri_values(distill_func, view_name):
                 if not param_set:
                     param_set = ()
                 elif self._is_str(param_set):
                     param_set = (param_set,)
-                uri = self.generate_uri(url, view_name, param_set)
-                render = self.render_view(uri, status_codes, param_set, a, k)
-                file_name = self._get_filename(file_name_base, uri, param_set)
+                to_render.append((url, view_name, param_set, status_codes, file_name_base, a, k))
+        with ThreadPoolExecutor(max_workers=self.parallel_render) as executor:
+            results = executor.map(_render, to_render)
+            for uri, file_name, render in results:
                 yield uri, file_name, render
 
     def render(self, view_name=None, status_codes=None, view_args=None, view_kwargs=None):
@@ -398,18 +410,18 @@ def write_file(full_path, content):
             raise
 
 
-def get_renderer(urls_to_distill):
+def get_renderer(urls_to_distill, parallel_render=1):
     import_path = getattr(settings, "DISTILL_RENDERER", None)
     if import_path:
         render_cls = import_string(import_path)
     else:
         render_cls = DistillRender
-    return render_cls(urls_to_distill)
+    return render_cls(urls_to_distill, parallel_render)
 
 
-def render_to_dir(output_dir, urls_to_distill, stdout):
+def render_to_dir(output_dir, urls_to_distill, stdout, parallel_render=1):
     load_urls(stdout)
-    renderer = get_renderer(urls_to_distill)
+    renderer = get_renderer(urls_to_distill, parallel_render)
     for page_uri, file_name, http_response in renderer.render():
         full_path, local_uri = get_filepath(output_dir, file_name, page_uri)
         content = http_response.content

tests/test_renderer.py

@@ -243,7 +243,7 @@ def _blackhole(_):
             for expected_file in expected_files:
                 filepath = os.path.join(tmpdirname, *expected_file)
                 self.assertIn(filepath, written_files)
-        self.assertEqual(render_view_spy.call_count, 13)
+        self.assertEqual(render_view_spy.call_count, 34)
 
     def test_sessions_are_ignored(self):
         if settings.HAS_PATH:
@@ -436,3 +436,25 @@ def test_request_has_resolver_match(self):
         uri = self.renderer.generate_uri(view_url, view_name, param_set)
         render = self.renderer.render_view(uri, status_codes, param_set, args, kwargs)
         self.assertEqual(render.content, b"test_request_has_resolver_match")
+
+    def test_parallel_rendering(self):
+        def _blackhole(_):
+            pass
+        expected_files = (
+            ('test',),
+            ('re_path', '12345'),
+            ('re_path', 'test'),
+            ('re_path', 'x', '12345.html'),
+            ('re_path', 'x', 'test.html'),
+        )
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with self.assertRaises(DistillError):
+                render_to_dir(tmpdirname, urls_to_distill, _blackhole, parallel_render=8)
+            written_files = []
+            for (root, dirs, files) in os.walk(tmpdirname):
+                for f in files:
+                    filepath = os.path.join(root, f)
+                    written_files.append(filepath)
+            for expected_file in expected_files:
+                filepath = os.path.join(tmpdirname, *expected_file)
+                self.assertIn(filepath, written_files)