From d6d8c9bfa7d855378b09db3c2867e2efc7a42f19 Mon Sep 17 00:00:00 2001 From: meeb Date: Sat, 25 May 2024 16:07:13 +1000 Subject: [PATCH] add --parallel-render arg which enables parallel rendering with multiple threads, resolves #90 --- README.md | 6 ++++ .../management/commands/distill-local.py | 13 ++++----- .../management/commands/distill-publish.py | 4 ++- django_distill/renderer.py | 28 +++++++++++++------ tests/test_renderer.py | 24 +++++++++++++++- 5 files changed, 58 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 3ae3bd5..d049645 100644 --- a/README.md +++ b/README.md @@ -274,6 +274,9 @@ rendering, this is just a shortcut to save you typing an extra command. `--exclude-staticfiles`: Do not copy any static files at all, only render output from Django views. +`--parallel-render [number of threads]`: Render files in parallel on multiple +threads, this can speed up rendering. Defaults to `1` thread. + `--generate-redirects`: Attempt to generate static redirects stored in the `django.contrib.redirects` app. If you have a redirect from `/old/` to `/new/` using this flag will create a static HTML `` @@ -322,6 +325,9 @@ to update most of them, and you don't care if old files remain on the server. `--parallel-publish [number of threads]`: Publish files in parallel on multiple threads, this can speed up publishing. Defaults to `1` thread. +`--parallel-render [number of threads]`: Render files in parallel on multiple +threads, this can speed up rendering. Defaults to `1` thread. + `--generate-redirects`: Attempt to generate static redirects stored in the `django.contrib.redirects` app. If you have a redirect from `/old/` to `/new/` using this flag will create a static HTML `` diff --git a/django_distill/management/commands/distill-local.py b/django_distill/management/commands/distill-local.py index 06d1cb2..22b6b26 100644 --- a/django_distill/management/commands/distill-local.py +++ b/django_distill/management/commands/distill-local.py @@ -14,14 +14,12 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('output_dir', nargs='?', type=str) - parser.add_argument('--collectstatic', dest='collectstatic', - action='store_true') + parser.add_argument('--collectstatic', dest='collectstatic', action='store_true') parser.add_argument('--quiet', dest='quiet', action='store_true') parser.add_argument('--force', dest='force', action='store_true') - parser.add_argument('--exclude-staticfiles', dest='exclude_staticfiles', - action='store_true') - parser.add_argument('--generate-redirects', dest='generate_redirects', - action='store_true') + parser.add_argument('--exclude-staticfiles', dest='exclude_staticfiles', action='store_true') + parser.add_argument('--generate-redirects', dest='generate_redirects', action='store_true') + parser.add_argument('--parallel-render', dest='parallel_render', type=int, default=1) def _quiet(self, *args, **kwargs): pass @@ -33,6 +31,7 @@ def handle(self, *args, **options): force = options.get('force') exclude_staticfiles = options.get('exclude_staticfiles') generate_redirects = options.get('generate_redirects') + parallel_render = options.get('parallel_render') if quiet: stdout = self._quiet else: @@ -82,7 +81,7 @@ def handle(self, *args, **options): stdout('') stdout('Generating static site into directory: {}'.format(output_dir)) try: - render_to_dir(output_dir, urls_to_distill, stdout) + render_to_dir(output_dir, urls_to_distill, stdout, parallel_render=parallel_render) if not exclude_staticfiles: copy_static_and_media_files(output_dir, stdout) except DistillError as err: diff --git a/django_distill/management/commands/distill-publish.py b/django_distill/management/commands/distill-publish.py index f0349bb..0766dcc 100644 --- a/django_distill/management/commands/distill-publish.py +++ b/django_distill/management/commands/distill-publish.py @@ -26,6 +26,7 @@ def add_arguments(self, parser): parser.add_argument('--ignore-remote-content', dest='ignore_remote_content', action='store_true') parser.add_argument('--parallel-publish', dest='parallel_publish', type=int, default=1) parser.add_argument('--generate-redirects', dest='generate_redirects', action='store_true') + parser.add_argument('--parallel-render', dest='parallel_render', type=int, default=1) def _quiet(self, *args, **kwargs): pass @@ -52,6 +53,7 @@ def handle(self, *args, **options): quiet = options.get('quiet') force = options.get('force') generate_redirects = options.get('generate_redirects') + parallel_render = options.get('parallel_render') if quiet: stdout = self._quiet else: @@ -89,7 +91,7 @@ def handle(self, *args, **options): msg = 'Generating static site into directory: {}' stdout(msg.format(output_dir)) try: - render_to_dir(output_dir, urls_to_distill, stdout) + render_to_dir(output_dir, urls_to_distill, stdout, parallel_render=parallel_render) if not exclude_staticfiles: copy_static_and_media_files(output_dir, stdout) except DistillError as err: diff --git a/django_distill/renderer.py b/django_distill/renderer.py index c9853bc..1a38257 100644 --- a/django_distill/renderer.py +++ b/django_distill/renderer.py @@ -4,6 +4,7 @@ import os import types from shutil import copy2 +from concurrent.futures import ThreadPoolExecutor from django.utils import translation from django.conf import settings from django.urls import include as include_urls, get_resolver @@ -158,8 +159,9 @@ class DistillRender(object): distill_url() and then copies over all static media. ''' - def __init__(self, urls_to_distill): + def __init__(self, urls_to_distill, parallel_render=1): self.urls_to_distill = urls_to_distill + self.parallel_render = parallel_render self.namespace_map = load_namespace_map() # activate the default translation translation.activate(settings.LANGUAGE_CODE) @@ -186,15 +188,25 @@ def render_file(self, view_name, status_codes, view_args, view_kwargs): return uri, file_name, render def render_all_urls(self): + + def _render(item): + url, view_name, param_set, status_codes, file_name_base, a, k = item + uri = self.generate_uri(url, view_name, param_set) + render = self.render_view(uri, status_codes, param_set, a, k) + file_name = self._get_filename(file_name_base, uri, param_set) + return uri, file_name, render + + to_render = [] for url, distill_func, file_name_base, status_codes, view_name, a, k in self.urls_to_distill: for param_set in self.get_uri_values(distill_func, view_name): if not param_set: param_set = () elif self._is_str(param_set): param_set = (param_set,) - uri = self.generate_uri(url, view_name, param_set) - render = self.render_view(uri, status_codes, param_set, a, k) - file_name = self._get_filename(file_name_base, uri, param_set) + to_render.append((url, view_name, param_set, status_codes, file_name_base, a, k)) + with ThreadPoolExecutor(max_workers=self.parallel_render) as executor: + results = executor.map(_render, to_render) + for uri, file_name, render in results: yield uri, file_name, render def render(self, view_name=None, status_codes=None, view_args=None, view_kwargs=None): @@ -398,18 +410,18 @@ def write_file(full_path, content): raise -def get_renderer(urls_to_distill): +def get_renderer(urls_to_distill, parallel_render=1): import_path = getattr(settings, "DISTILL_RENDERER", None) if import_path: render_cls = import_string(import_path) else: render_cls = DistillRender - return render_cls(urls_to_distill) + return render_cls(urls_to_distill, parallel_render) -def render_to_dir(output_dir, urls_to_distill, stdout): +def render_to_dir(output_dir, urls_to_distill, stdout, parallel_render=1): load_urls(stdout) - renderer = get_renderer(urls_to_distill) + renderer = get_renderer(urls_to_distill, parallel_render) for page_uri, file_name, http_response in renderer.render(): full_path, local_uri = get_filepath(output_dir, file_name, page_uri) content = http_response.content diff --git a/tests/test_renderer.py b/tests/test_renderer.py index 1294d08..e29cb5e 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -243,7 +243,7 @@ def _blackhole(_): for expected_file in expected_files: filepath = os.path.join(tmpdirname, *expected_file) self.assertIn(filepath, written_files) - self.assertEqual(render_view_spy.call_count, 13) + self.assertEqual(render_view_spy.call_count, 34) def test_sessions_are_ignored(self): if settings.HAS_PATH: @@ -436,3 +436,25 @@ def test_request_has_resolver_match(self): uri = self.renderer.generate_uri(view_url, view_name, param_set) render = self.renderer.render_view(uri, status_codes, param_set, args, kwargs) self.assertEqual(render.content, b"test_request_has_resolver_match") + + def test_parallel_rendering(self): + def _blackhole(_): + pass + expected_files = ( + ('test',), + ('re_path', '12345'), + ('re_path', 'test'), + ('re_path', 'x', '12345.html'), + ('re_path', 'x', 'test.html'), + ) + with tempfile.TemporaryDirectory() as tmpdirname: + with self.assertRaises(DistillError): + render_to_dir(tmpdirname, urls_to_distill, _blackhole, parallel_render=8) + written_files = [] + for (root, dirs, files) in os.walk(tmpdirname): + for f in files: + filepath = os.path.join(root, f) + written_files.append(filepath) + for expected_file in expected_files: + filepath = os.path.join(tmpdirname, *expected_file) + self.assertIn(filepath, written_files)