From 42a031cf836d9c5f8d715c2f44ca3f62f337dd2c Mon Sep 17 00:00:00 2001 From: Raphael Krupinski Date: Sun, 26 Feb 2023 20:54:48 +0100 Subject: [PATCH 1/2] refactor(Worker): extract the rendering code to a separate class --- copier/main.py | 135 ++++++------------------------------------- copier/renderer.py | 139 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 118 deletions(-) create mode 100644 copier/renderer.py diff --git a/copier/main.py b/copier/main.py index 1b133b139..da4010e1e 100644 --- a/copier/main.py +++ b/copier/main.py @@ -27,6 +27,7 @@ from questionary import unsafe_prompt from .errors import CopierAnswersInterrupt, ExtensionNotFoundError, UserMessageError +from .renderer import Renderer from .subproject import Subproject from .template import Task, Template from .tools import Style, TemporaryDirectory, printf @@ -466,113 +467,6 @@ def match_skip(self) -> Callable[[Path], bool]: ) ) - def _render_file(self, src_abspath: Path) -> None: - """Render one file. - - Args: - src_abspath: - The absolute path to the file that will be rendered. - """ - # TODO Get from main.render_file() - assert src_abspath.is_absolute() - src_relpath = src_abspath.relative_to(self.template.local_abspath).as_posix() - src_renderpath = src_abspath.relative_to(self.template_copy_root) - dst_relpath = self._render_path(src_renderpath) - if dst_relpath is None: - return - if src_abspath.name.endswith(self.template.templates_suffix): - try: - tpl = self.jinja_env.get_template(src_relpath) - except UnicodeDecodeError: - if self.template.templates_suffix: - # suffix is not empty, re-raise - raise - # suffix is empty, fallback to copy - new_content = src_abspath.read_bytes() - else: - new_content = tpl.render(**self._render_context()).encode() - else: - new_content = src_abspath.read_bytes() - dst_abspath = Path(self.subproject.local_abspath, dst_relpath) - src_mode = src_abspath.stat().st_mode - if not self._render_allowed(dst_relpath, expected_contents=new_content): - return - if not self.pretend: - dst_abspath.parent.mkdir(parents=True, exist_ok=True) - dst_abspath.write_bytes(new_content) - dst_abspath.chmod(src_mode) - - def _render_folder(self, src_abspath: Path) -> None: - """Recursively render a folder. - - Args: - src_path: - Folder to be rendered. It must be an absolute path within - the template. - """ - assert src_abspath.is_absolute() - src_relpath = src_abspath.relative_to(self.template_copy_root) - dst_relpath = self._render_path(src_relpath) - if dst_relpath is None: - return - if not self._render_allowed(dst_relpath, is_dir=True): - return - dst_abspath = Path(self.subproject.local_abspath, dst_relpath) - if not self.pretend: - dst_abspath.mkdir(parents=True, exist_ok=True) - for file in src_abspath.iterdir(): - if file.is_dir(): - self._render_folder(file) - else: - self._render_file(file) - - def _render_path(self, relpath: Path) -> Optional[Path]: - """Render one relative path. - - Args: - relpath: - The relative path to be rendered. Obviously, it can be templated. - """ - is_template = relpath.name.endswith(self.template.templates_suffix) - templated_sibling = ( - self.template.local_abspath / f"{relpath}{self.template.templates_suffix}" - ) - # With an empty suffix, the templated sibling always exists. - if templated_sibling.exists() and self.template.templates_suffix: - return None - if self.template.templates_suffix and is_template: - relpath = relpath.with_suffix("") - rendered_parts = [] - for part in relpath.parts: - # Skip folder if any part is rendered as an empty string - part = self._render_string(part) - if not part: - return None - # {{ _copier_conf.answers_file }} becomes the full path; in that case, - # restore part to be just the end leaf - if str(self.answers_relpath) == part: - part = Path(part).name - rendered_parts.append(part) - result = Path(*rendered_parts) - if not is_template: - templated_sibling = ( - self.template.local_abspath - / f"{result}{self.template.templates_suffix}" - ) - if templated_sibling.exists(): - return None - return result - - def _render_string(self, string: str) -> str: - """Render one templated string. - - Args: - string: - The template source string. - """ - tpl = self.jinja_env.from_string(string) - return tpl.render(**self._render_context()) - @cached_property def subproject(self) -> Subproject: """Get related subproject.""" @@ -591,15 +485,6 @@ def template(self) -> Template: url = str(self.subproject.template.url) return Template(url=url, ref=self.vcs_ref, use_prereleases=self.use_prereleases) - @cached_property - def template_copy_root(self) -> Path: - """Absolute path from where to start copying. - - It points to the cloned template local abspath + the rendered subdir, if any. - """ - subdir = self._render_string(self.template.subdirectory) or "" - return self.template.local_abspath / subdir - # Main operations def run_auto(self) -> None: """Copy or update automatically. @@ -624,7 +509,6 @@ def run_copy(self) -> None: See [generating a project][generating-a-project]. """ was_existing = self.subproject.local_abspath.exists() - src_abspath = self.template_copy_root try: if not self.quiet: # TODO Unify printing tools @@ -632,7 +516,7 @@ def run_copy(self) -> None: f"\nCopying from template version {self.template.version}", file=sys.stderr, ) - self._render_folder(src_abspath) + self._default_renderer.render() if not self.quiet: # TODO Unify printing tools print("") # padding space @@ -810,6 +694,21 @@ def _git_initialize_repo(self): git("config", "--unset", "user.name") git("config", "--unset", "user.email") + @cached_property + def _default_renderer(self) -> Renderer: + return Renderer( + template=self.template, + subproject=self.subproject, + _render_allowed=self._render_allowed, + pretend=self.pretend, + jinja_env=self.jinja_env, + answers_relpath=self.answers_relpath, + _render_context=self._render_context(), + ) + + def _render_string(self, string: str) -> str: + return self._default_renderer.render_string(string) + def run_copy( src_path: str, diff --git a/copier/renderer.py b/copier/renderer.py new file mode 100644 index 000000000..cc968e7bd --- /dev/null +++ b/copier/renderer.py @@ -0,0 +1,139 @@ +from dataclasses import dataclass +from functools import cached_property +from pathlib import Path +from typing import Any, Callable, Mapping, Optional + +from jinja2 import Environment + +from .subproject import Subproject +from .template import Template + + +@dataclass +class Renderer: + template: Template + subproject: Subproject + _render_allowed: Callable + pretend: bool + jinja_env: Environment + answers_relpath: Path + _render_context: Mapping[str, Any] + + def render(self): + self._render_folder(self.template.local_abspath) + + def _render_folder(self, src_abspath: Path) -> None: + """Recursively render a folder. + + Args: + src_abspath: + Folder to be rendered. It must be an absolute path within + the template. + """ + assert src_abspath.is_absolute() + src_relpath = src_abspath.relative_to(self.template_copy_root) + dst_relpath = self._render_path(src_relpath) + if dst_relpath is None: + return + if not self._render_allowed(dst_relpath, is_dir=True): + return + dst_abspath = Path(self.subproject.local_abspath, dst_relpath) + if not self.pretend: + dst_abspath.mkdir(parents=True, exist_ok=True) + for file in src_abspath.iterdir(): + if file.is_dir(): + self._render_folder(file) + else: + self._render_file(file) + + def _render_path(self, relpath: Path) -> Optional[Path]: + """Render one relative path. + + Args: + relpath: + The relative path to be rendered. Obviously, it can be templated. + """ + is_template = relpath.name.endswith(self.template.templates_suffix) + templated_sibling = ( + self.template.local_abspath / f"{relpath}{self.template.templates_suffix}" + ) + # With an empty suffix, the templated sibling always exists. + if templated_sibling.exists() and self.template.templates_suffix: + return None + if self.template.templates_suffix and is_template: + relpath = relpath.with_suffix("") + rendered_parts = [] + for part in relpath.parts: + # Skip folder if any part is rendered as an empty string + part = self.render_string(part) + if not part: + return None + # {{ _copier_conf.answers_file }} becomes the full path; in that case, + # restore part to be just the end leaf + if str(self.answers_relpath) == part: + part = Path(part).name + rendered_parts.append(part) + result = Path(*rendered_parts) + if not is_template: + templated_sibling = ( + self.template.local_abspath + / f"{result}{self.template.templates_suffix}" + ) + if templated_sibling.exists(): + return None + return result + + def _render_file(self, src_abspath: Path) -> None: + """Render one file. + + Args: + src_abspath: + The absolute path to the file that will be rendered. + """ + # TODO Get from main.render_file() + assert src_abspath.is_absolute() + src_relpath = src_abspath.relative_to(self.template.local_abspath).as_posix() + src_renderpath = src_abspath.relative_to(self.template_copy_root) + dst_relpath = self._render_path(src_renderpath) + if dst_relpath is None: + return + if src_abspath.name.endswith(self.template.templates_suffix): + try: + tpl = self.jinja_env.get_template(src_relpath) + except UnicodeDecodeError: + if self.template.templates_suffix: + # suffix is not empty, re-raise + raise + # suffix is empty, fallback to copy + new_content = src_abspath.read_bytes() + else: + new_content = tpl.render(**self._render_context).encode() + else: + new_content = src_abspath.read_bytes() + dst_abspath = Path(self.subproject.local_abspath, dst_relpath) + src_mode = src_abspath.stat().st_mode + if not self._render_allowed(dst_relpath, expected_contents=new_content): + return + if not self.pretend: + dst_abspath.parent.mkdir(parents=True, exist_ok=True) + dst_abspath.write_bytes(new_content) + dst_abspath.chmod(src_mode) + + def render_string(self, string: str) -> str: + """Render one templated string. + + Args: + string: + The template source string. + """ + tpl = self.jinja_env.from_string(string) + return tpl.render(**self._render_context) + + @cached_property + def template_copy_root(self) -> Path: + """Absolute path from where to start copying. + + It points to the cloned template local abspath + the rendered subdir, if any. + """ + subdir = self.render_string(self.template.subdirectory) or "" + return self.template.local_abspath / subdir From 066a76a95cec156a9aeb92276afd09c2b563c9a7 Mon Sep 17 00:00:00 2001 From: Raphael Krupinski Date: Sat, 14 Jan 2023 12:18:35 +0100 Subject: [PATCH 2/2] feat: project mode --- copier/main.py | 35 ++++++++++- copier/renderer.py | 45 ++++++++++++-- tests/test_project_mode.py | 120 +++++++++++++++++++++++++++++++++++++ 3 files changed, 192 insertions(+), 8 deletions(-) create mode 100644 tests/test_project_mode.py diff --git a/copier/main.py b/copier/main.py index da4010e1e..e6b0fe8b8 100644 --- a/copier/main.py +++ b/copier/main.py @@ -1,5 +1,6 @@ """Main functions and classes, used to generate or update projects.""" +import collections.abc import os import platform import subprocess @@ -11,10 +12,12 @@ from itertools import chain from pathlib import Path from shutil import rmtree -from typing import Callable, Iterable, List, Mapping, Optional, Sequence +from typing import Callable, Iterable, List, Mapping, Optional, Sequence, Type, cast from unicodedata import normalize +from jinja2 import Environment from jinja2.loaders import FileSystemLoader +from jinja2.nativetypes import NativeEnvironment from jinja2.sandbox import SandboxedEnvironment from pathspec import PathSpec from plumbum import ProcessExecutionError, colors @@ -422,6 +425,13 @@ def all_exclusions(self) -> StrSeq: @cached_property def jinja_env(self) -> SandboxedEnvironment: + return cast(SandboxedEnvironment, self.abstract_jinja_env(SandboxedEnvironment)) + + @cached_property + def native_jinja_env(self) -> NativeEnvironment: + return cast(NativeEnvironment, self.abstract_jinja_env(NativeEnvironment)) + + def abstract_jinja_env(self, env_cls: Type) -> Environment: """Return a pre-configured Jinja environment. Respects template settings. @@ -437,7 +447,7 @@ def jinja_env(self) -> SandboxedEnvironment: # Of course we still have the post-copy tasks to worry about, but at least # they are more visible to the final user. try: - env = SandboxedEnvironment( + env = env_cls( loader=loader, extensions=extensions, **self.template.envops ) except ModuleNotFoundError as error: @@ -516,7 +526,7 @@ def run_copy(self) -> None: f"\nCopying from template version {self.template.version}", file=sys.stderr, ) - self._default_renderer.render() + self._render_with_items() if not self.quiet: # TODO Unify printing tools print("") # padding space @@ -529,6 +539,13 @@ def run_copy(self) -> None: # TODO Unify printing tools print("") # padding space + def _render_with_items(self): + self._default_renderer.render() + + for items_key in self.project_mode_keys: + for project_mode_item in self.answers.combined[items_key]: + self._default_renderer.with_item(items_key, project_mode_item).render() + def run_update(self) -> None: """Update a subproject that was already generated. @@ -704,11 +721,23 @@ def _default_renderer(self) -> Renderer: jinja_env=self.jinja_env, answers_relpath=self.answers_relpath, _render_context=self._render_context(), + _items_keys=self.project_mode_keys, ) def _render_string(self, string: str) -> str: return self._default_renderer.render_string(string) + @cached_property + def project_mode_keys(self) -> Iterable[str]: + raw_items = self.template.config_data.get("items") + if not raw_items: + return () + return ( + [raw_items] + if isinstance(raw_items, str) + else cast(collections.abc.Iterable, raw_items) + ) + def run_copy( src_path: str, diff --git a/copier/renderer.py b/copier/renderer.py index cc968e7bd..a5aa76953 100644 --- a/copier/renderer.py +++ b/copier/renderer.py @@ -1,7 +1,8 @@ -from dataclasses import dataclass +from collections import ChainMap +from dataclasses import dataclass, field from functools import cached_property from pathlib import Path -from typing import Any, Callable, Mapping, Optional +from typing import Any, Callable, Iterable, Mapping, Optional from jinja2 import Environment @@ -18,9 +19,21 @@ class Renderer: jinja_env: Environment answers_relpath: Path _render_context: Mapping[str, Any] + _items_key: Optional[str] = None + _items_keys: Iterable[str] = field(default_factory=list) def render(self): - self._render_folder(self.template.local_abspath) + dst_abspath = Path(self.subproject.local_abspath) + if not self.pretend: + dst_abspath.mkdir(parents=True, exist_ok=True) + + for file in self.template_copy_root.iterdir(): + if file.name in self._items_keys: + continue + if file.is_dir(): + self._render_folder(file) + else: + self._render_file(file) def _render_folder(self, src_abspath: Path) -> None: """Recursively render a folder. @@ -135,5 +148,27 @@ def template_copy_root(self) -> Path: It points to the cloned template local abspath + the rendered subdir, if any. """ - subdir = self.render_string(self.template.subdirectory) or "" - return self.template.local_abspath / subdir + parts = [ + self.template.local_abspath, + ] + subdir = self.render_string(self.template.subdirectory) + if subdir: + parts.append(subdir) + + if self._items_key: + parts.append(self._items_key) + + return Path(*parts) + + def with_item(self, items_key: str, items_value: Any) -> "Renderer": + return Renderer( + self.template, + self.subproject, + self._render_allowed, + self.pretend, + self.jinja_env, + self.answers_relpath, + ChainMap({"_item": items_value}, self._render_context), + items_key, + [], + ) diff --git a/tests/test_project_mode.py b/tests/test_project_mode.py new file mode 100644 index 000000000..092895fc7 --- /dev/null +++ b/tests/test_project_mode.py @@ -0,0 +1,120 @@ +import warnings + +import copier +from tests.helpers import build_file_tree + + +def test_render_items(tmp_path_factory): + src, dst = map(tmp_path_factory.mktemp, ("src", "dst")) + build_file_tree( + { + src + / "copier.yml": """ + _items: items + """, + src + / "items" + / "{{_item}}.txt.jinja": "This is {{ _item }}. Hello {{hello}}", + } + ) + # No warnings, because template is explicit + with warnings.catch_warnings(): + warnings.simplefilter("error") + copier.run_auto( + str(src), + dst, + data={ + "items": ["one", "two", "three"], + "hello": "world", + }, + defaults=True, + overwrite=True, + ) + + one_rendered = (dst / "one.txt").read_text() + one_expected = "This is one. Hello world" + assert one_rendered == one_expected + + one_rendered = (dst / "two.txt").read_text() + one_expected = "This is two. Hello world" + assert one_rendered == one_expected + + one_rendered = (dst / "three.txt").read_text() + one_expected = "This is three. Hello world" + assert one_rendered == one_expected + + +def test_render_items_conditional(tmp_path_factory): + src, dst = map(tmp_path_factory.mktemp, ("src", "dst")) + build_file_tree( + { + src + / "copier.yml": """ + _items: a + """, + src + / "a" + / "{{_item.value}}.txt.jinja": "This is {{ _item.value }}. Hello {{hello}}", + } + ) + # No warnings, because template is explicit + with warnings.catch_warnings(): + warnings.simplefilter("error") + + data = { + "a": [ + {"value": "one"}, + ], + "b": [ + {"value": "two"}, + ], + "hello": "world", + } + copier.run_auto(str(src), dst, data=data, defaults=True, overwrite=True) + + one_rendered = (dst / "one.txt").read_text() + one_expected = "This is one. Hello world" + assert one_rendered == one_expected + + assert not (dst / "two.txt").exists() + + +def test_render_items_list(tmp_path_factory): + src, dst = map(tmp_path_factory.mktemp, ("src", "dst")) + build_file_tree( + { + src + / "copier.yml": """ + _items: + - apples + - oranges + """, + src + / "apples" + / "{{_item.value}}.txt.jinja": "This is {{ _item.value }}. Hello {{hello}}", + src + / "oranges" + / "{{_item.value}}.txt.jinja": "This is {{ _item.value }}. Hello {{hello}}", + } + ) + # No warnings, because template is explicit + with warnings.catch_warnings(): + warnings.simplefilter("error") + + data = { + "apples": [ + {"value": "apple_one"}, + {"value": "apple_two"}, + ], + "oranges": [ + {"value": "orange_one"}, + {"value": "orange_two"}, + ], + "hello": "world", + } + copier.run_auto(str(src), dst, data=data, defaults=True, overwrite=True) + + assert (dst / "apple_one.txt").read_text() == "This is apple_one. Hello world" + assert (dst / "apple_two.txt").read_text() == "This is apple_two. Hello world" + assert (dst / "orange_one.txt").read_text() == "This is orange_one. Hello world" + assert (dst / "orange_one.txt").read_text() == "This is orange_one. Hello world"