From 249fae33b10d551e26a75d9955869e51420b8af8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9=20Kijewski?=
 <kijewski@library.vetmed.fu-berlin.de>
Date: Fri, 7 Jan 2022 19:37:37 +0100
Subject: [PATCH 1/2] Add whitespace stripping in the input

In HTML sources indentation can aid readability, but it only bloats the
output. This PR adds the argument "strip" to templates, which lets you
automatically strip the template input:

* "none": Don't strip any spaces in the input
* "tail": Remove a single single newline at the end of the input. This
  is the default
* "trim-lines": Remove all whitespaces at the front and back of all
  lines, and remove empty lines
* "eager": Like "trim", but also replace runs of whitespaces with a
  single space.

The automatic stripping does not try to understand the input at all. It
does not know what `xml:space="preserve"`, `<pre>` or `<textarea>`
means.
---
 askama_derive/src/lib.rs                   |   8 +-
 askama_shared/src/generator.rs             |   6 +-
 askama_shared/src/input.rs                 |  11 +-
 askama_shared/src/lib.rs                   |  19 ++--
 askama_shared/src/strip.rs                 | 117 +++++++++++++++++++++
 testing/templates/whitespace_trimming.html |  10 ++
 testing/tests/whitespace.rs                |  88 ++++++++++++++++
 7 files changed, 242 insertions(+), 17 deletions(-)
 create mode 100644 askama_shared/src/strip.rs
 create mode 100644 testing/templates/whitespace_trimming.html

diff --git a/askama_derive/src/lib.rs b/askama_derive/src/lib.rs
index 4d81433a5..17adcade0 100644
--- a/askama_derive/src/lib.rs
+++ b/askama_derive/src/lib.rs
@@ -37,8 +37,8 @@ fn build_template(ast: &syn::DeriveInput) -> Result<String, CompileError> {
     let config = Config::new(&config_toml)?;
     let input = TemplateInput::new(ast, &config)?;
     let source: String = match input.source {
-        Source::Source(ref s) => s.clone(),
-        Source::Path(_) => get_template_source(&input.path)?,
+        Source::Source(ref s) => input.strip.apply(s),
+        Source::Path(_) => get_template_source(&input.path, input.strip)?,
     };
 
     let mut sources = HashMap::new();
@@ -96,12 +96,12 @@ fn find_used_templates(
                         .into());
                     }
                     dependency_graph.push(dependency_path);
-                    let source = get_template_source(&extends)?;
+                    let source = get_template_source(&extends, input.strip)?;
                     check.push((extends, source));
                 }
                 Node::Import(_, import, _) => {
                     let import = input.config.find_template(import, Some(&path))?;
-                    let source = get_template_source(&import)?;
+                    let source = get_template_source(&import, input.strip)?;
                     check.push((import, source));
                 }
                 _ => {}
diff --git a/askama_shared/src/generator.rs b/askama_shared/src/generator.rs
index 63a0154ad..e13b29a4e 100644
--- a/askama_shared/src/generator.rs
+++ b/askama_shared/src/generator.rs
@@ -1,11 +1,9 @@
-use super::{get_template_source, CompileError, Integrations};
-use crate::filters;
 use crate::heritage::{Context, Heritage};
 use crate::input::{Source, TemplateInput};
 use crate::parser::{parse, Cond, CondTest, Expr, Loop, Node, Target, When, Ws};
+use crate::{filters, get_template_source, CompileError, Integrations};
 
 use proc_macro2::Span;
-
 use quote::{quote, ToTokens};
 
 use std::collections::HashMap;
@@ -765,7 +763,7 @@ impl<'a, S: std::hash::BuildHasher> Generator<'a, S> {
             .input
             .config
             .find_template(path, Some(&self.input.path))?;
-        let src = get_template_source(&path)?;
+        let src = get_template_source(&path, self.input.strip)?;
         let nodes = parse(&src, self.input.syntax)?;
 
         // Make sure the compiler understands that the generated code depends on the template file.
diff --git a/askama_shared/src/input.rs b/askama_shared/src/input.rs
index f7eac2336..0300c2c3a 100644
--- a/askama_shared/src/input.rs
+++ b/askama_shared/src/input.rs
@@ -1,4 +1,4 @@
-use crate::{CompileError, Config, Syntax};
+use crate::{CompileError, Config, Strip, Syntax};
 
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
@@ -17,6 +17,7 @@ pub struct TemplateInput<'a> {
     pub mime_type: String,
     pub parent: Option<&'a syn::Type>,
     pub path: PathBuf,
+    pub strip: Strip,
 }
 
 impl TemplateInput<'_> {
@@ -62,6 +63,7 @@ impl TemplateInput<'_> {
         let mut escaping = None;
         let mut ext = None;
         let mut syntax = None;
+        let mut strip = config.strip;
         for item in template_args {
             let pair = match item {
                 syn::NestedMeta::Meta(syn::Meta::NameValue(ref pair)) => pair,
@@ -120,6 +122,12 @@ impl TemplateInput<'_> {
                 } else {
                     return Err("syntax value must be string literal".into());
                 }
+            } else if ident == "strip" {
+                if let syn::Lit::Str(ref s) = pair.lit {
+                    strip = s.value().parse()?;
+                } else {
+                    return Err("syntax strip must be string literal".into());
+                }
             } else {
                 return Err(format!("unsupported attribute key {:?} found", ident).into());
             }
@@ -205,6 +213,7 @@ impl TemplateInput<'_> {
             mime_type,
             parent,
             path,
+            strip,
         })
     }
 
diff --git a/askama_shared/src/lib.rs b/askama_shared/src/lib.rs
index b2bebd0c5..23678c040 100644
--- a/askama_shared/src/lib.rs
+++ b/askama_shared/src/lib.rs
@@ -13,6 +13,7 @@ use std::{env, fmt, fs};
 use serde::Deserialize;
 
 pub use crate::input::extension_to_mime_type;
+pub use crate::strip::Strip;
 pub use askama_escape::MarkupDisplay;
 
 mod error;
@@ -27,6 +28,7 @@ pub mod heritage;
 pub mod input;
 #[doc(hidden)]
 pub mod parser;
+mod strip;
 
 #[derive(Debug)]
 pub struct Config<'a> {
@@ -34,6 +36,7 @@ pub struct Config<'a> {
     pub syntaxes: BTreeMap<String, Syntax<'a>>,
     pub default_syntax: &'a str,
     pub escapers: Vec<(HashSet<String>, String)>,
+    pub strip: Strip,
 }
 
 impl Config<'_> {
@@ -102,6 +105,7 @@ impl Config<'_> {
             syntaxes,
             default_syntax,
             escapers,
+            strip: raw.strip.unwrap_or_default(),
         })
     }
 
@@ -200,6 +204,7 @@ struct RawConfig<'d> {
     general: Option<General<'d>>,
     syntax: Option<Vec<RawSyntax<'d>>>,
     escaper: Option<Vec<RawEscaper<'d>>>,
+    strip: Option<Strip>,
 }
 
 impl RawConfig<'_> {
@@ -257,19 +262,17 @@ where
 }
 
 #[allow(clippy::match_wild_err_arm)]
-pub fn get_template_source(tpl_path: &Path) -> std::result::Result<String, CompileError> {
+pub fn get_template_source(
+    tpl_path: &Path,
+    strip: Strip,
+) -> std::result::Result<String, CompileError> {
     match fs::read_to_string(tpl_path) {
         Err(_) => Err(format!(
             "unable to open template file '{}'",
             tpl_path.to_str().unwrap()
         )
         .into()),
-        Ok(mut source) => {
-            if source.ends_with('\n') {
-                let _ = source.pop();
-            }
-            Ok(source)
-        }
+        Ok(source) => Ok(strip.apply(source)),
     }
 }
 
@@ -328,7 +331,7 @@ mod tests {
         let path = Config::new("")
             .and_then(|config| config.find_template("b.html", None))
             .unwrap();
-        assert_eq!(get_template_source(&path).unwrap(), "bar");
+        assert_eq!(get_template_source(&path, Strip::Tail).unwrap(), "bar");
     }
 
     #[test]
diff --git a/askama_shared/src/strip.rs b/askama_shared/src/strip.rs
new file mode 100644
index 000000000..c8409911b
--- /dev/null
+++ b/askama_shared/src/strip.rs
@@ -0,0 +1,117 @@
+use crate::CompileError;
+
+use std::borrow::Cow;
+use std::convert::{TryFrom, TryInto};
+use std::str::FromStr;
+
+/// Whitespace handling of the input source.
+///
+/// The automatic stripping does not try to understand the input at all. It
+/// does not know what `xml:space="preserve"`, `<pre>` or `<textarea>`
+/// means.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Strip {
+    /// "none": Don't strip any spaces in the input
+    None,
+    /// "tail": Remove a single single newline at the end of the input. This is the default
+    Tail,
+    /// "trim-lines": Remove all whitespaces at the front and back of all lines, and remove empty
+    /// lines
+    TrimLines,
+    /// "eager": Like "trim", but also replace runs of whitespaces with a single space.
+    Eager,
+}
+
+impl Default for Strip {
+    fn default() -> Self {
+        Strip::Tail
+    }
+}
+
+impl TryFrom<&str> for Strip {
+    type Error = String;
+
+    fn try_from(s: &str) -> Result<Self, Self::Error> {
+        match s {
+            "none" => Ok(Strip::None),
+            "tail" => Ok(Strip::Tail),
+            "trim-lines" => Ok(Strip::TrimLines),
+            "eager" => Ok(Strip::Eager),
+            v => return Err(format!("invalid value for strip: {:?}", v)),
+        }
+    }
+}
+
+impl FromStr for Strip {
+    type Err = CompileError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        s.try_into().map_err(Into::into)
+    }
+}
+
+#[cfg(feature = "serde")]
+const _: () = {
+    use std::fmt;
+
+    struct StripVisitor;
+
+    impl<'de> serde::de::Visitor<'de> for StripVisitor {
+        type Value = Strip;
+
+        fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            write!(f, r#"the string "none", "tail", "trim-lines", or "eager""#)
+        }
+
+        fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>
+        where
+            E: serde::de::Error,
+        {
+            s.try_into().map_err(E::custom)
+        }
+    }
+
+    impl<'de> serde::Deserialize<'de> for Strip {
+        fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+        where
+            D: serde::Deserializer<'de>,
+        {
+            deserializer.deserialize_str(StripVisitor)
+        }
+    }
+};
+
+impl Strip {
+    pub fn apply<'a, S: Into<Cow<'a, str>>>(self, src: S) -> String {
+        let src = src.into();
+        match self {
+            Strip::None => src.into_owned(),
+            Strip::Tail => {
+                let mut s = src.into_owned();
+                if s.ends_with('\n') {
+                    s.pop();
+                }
+                s
+            }
+            Strip::TrimLines | Strip::Eager => {
+                let mut stripped = String::with_capacity(src.len());
+                for line in src.lines().map(|s| s.trim()).filter(|&s| !s.is_empty()) {
+                    if !stripped.is_empty() {
+                        stripped.push('\n');
+                    }
+                    if self == Strip::Eager {
+                        for (index, word) in line.split_ascii_whitespace().enumerate() {
+                            if index > 0 {
+                                stripped.push(' ');
+                            }
+                            stripped.push_str(word);
+                        }
+                    } else {
+                        stripped.push_str(line);
+                    }
+                }
+                stripped
+            }
+        }
+    }
+}
diff --git a/testing/templates/whitespace_trimming.html b/testing/templates/whitespace_trimming.html
new file mode 100644
index 000000000..167c0ea4a
--- /dev/null
+++ b/testing/templates/whitespace_trimming.html
@@ -0,0 +1,10 @@
+
+<!DOCTYPE html>
+
+<html>
+ <body>
+  <p>
+   .  .   .
+  </p>
+ </body>
+</html>
diff --git a/testing/tests/whitespace.rs b/testing/tests/whitespace.rs
index ca72b23c6..cbc2ce623 100644
--- a/testing/tests/whitespace.rs
+++ b/testing/tests/whitespace.rs
@@ -39,3 +39,91 @@ fn test_extra_whitespace() {
     template.nested_1.nested_2.hash.insert("key", "value");
     assert_eq!(template.render().unwrap(), "\n0\n0\n0\n0\n\n\n\n0\n0\n0\n0\n0\n\na0\na1\nvalue\n\n\n\n\n\n[\n  \"a0\",\n  \"a1\",\n  \"a2\",\n  \"a3\"\n]\n[\n  \"a0\",\n  \"a1\",\n  \"a2\",\n  \"a3\"\n][\n  \"a0\",\n  \"a1\",\n  \"a2\",\n  \"a3\"\n]\n[\n  \"a1\"\n][\n  \"a1\"\n]\n[\n  \"a1\",\n  \"a2\"\n][\n  \"a1\",\n  \"a2\"\n]\n[\n  \"a1\"\n][\n  \"a1\"\n]1-1-1\n3333 3\n2222 2\n0000 0\n3333 3\n\ntruefalse\nfalsefalsefalse\n\n\n\n\n\n\n\n\n\n\n\n\n\n");
 }
+
+#[derive(askama::Template)]
+#[template(source = " a \t b   c\n\nd e\n  f\n\n\n", ext = "txt", strip = "none")]
+struct StripNone;
+
+#[test]
+fn test_strip_none() {
+    assert_eq!(StripNone.render().unwrap(), " a \t b   c\n\nd e\n  f");
+}
+
+#[derive(askama::Template)]
+#[template(source = " a \t b   c\n\nd e\n  f\n\n\n", ext = "txt", strip = "tail")]
+struct StripTail;
+
+#[test]
+fn test_strip_tail() {
+    assert_eq!(StripTail.render().unwrap(), " a \t b   c\n\nd e\n  f");
+}
+
+#[derive(askama::Template)]
+#[template(
+    source = " a \t b   c\n\nd e\n  f\n\n\n",
+    ext = "txt",
+    strip = "trim-lines"
+)]
+struct StripTrimLines;
+
+#[test]
+fn test_strip_trim_lines() {
+    assert_eq!(StripTrimLines.render().unwrap(), "a \t b   c\nd e\nf");
+}
+
+#[derive(askama::Template)]
+#[template(source = " a \t b   c\n\nd e\n  f\n\n\n", ext = "txt", strip = "eager")]
+struct StripEager;
+
+#[test]
+fn test_strip_eager() {
+    assert_eq!(StripEager.render().unwrap(), "a b c\nd e\nf");
+}
+
+#[derive(askama::Template)]
+#[template(path = "whitespace_trimming.html", strip = "none")]
+struct StripNone2;
+
+#[test]
+fn test_strip_none2() {
+    assert_eq!(
+        StripNone2.render().unwrap(),
+        "\n<!DOCTYPE html>\n\n<html>\n <body>\n  <p>\n   .  .   .\n  </p>\n </body>\n</html>"
+    );
+}
+
+#[derive(askama::Template)]
+#[template(path = "whitespace_trimming.html", strip = "tail")]
+struct StripTail2;
+
+#[test]
+fn test_strip_tail2() {
+    assert_eq!(
+        StripTail2.render().unwrap(),
+        "\n<!DOCTYPE html>\n\n<html>\n <body>\n  <p>\n   .  .   .\n  </p>\n </body>\n</html>"
+    );
+}
+
+#[derive(askama::Template)]
+#[template(path = "whitespace_trimming.html", strip = "trim-lines")]
+struct StripTrimLines2;
+
+#[test]
+fn test_strip_trim_lines2() {
+    assert_eq!(
+        StripTrimLines2.render().unwrap(),
+        "<!DOCTYPE html>\n<html>\n<body>\n<p>\n.  .   .\n</p>\n</body>\n</html>"
+    );
+}
+
+#[derive(askama::Template)]
+#[template(path = "whitespace_trimming.html", strip = "eager")]
+struct StripEager2;
+
+#[test]
+fn test_strip_eager2() {
+    assert_eq!(
+        StripEager2.render().unwrap(),
+        "<!DOCTYPE html>\n<html>\n<body>\n<p>\n. . .\n</p>\n</body>\n</html>"
+    );
+}

From c6e66a93592a093a8583c3a32df682ed208a6402 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9=20Kijewski?=
 <kijewski@library.vetmed.fu-berlin.de>
Date: Fri, 4 Feb 2022 22:51:01 +0100
Subject: [PATCH 2/2] Add another strip=eager test

---
 testing/tests/whitespace.rs | 120 ++++++++++++++++++++++++++++++++++++
 1 file changed, 120 insertions(+)

diff --git a/testing/tests/whitespace.rs b/testing/tests/whitespace.rs
index cbc2ce623..d9d634559 100644
--- a/testing/tests/whitespace.rs
+++ b/testing/tests/whitespace.rs
@@ -127,3 +127,123 @@ fn test_strip_eager2() {
         "<!DOCTYPE html>\n<html>\n<body>\n<p>\n. . .\n</p>\n</body>\n</html>"
     );
 }
+
+#[test]
+fn test_strip_class() {
+    #[derive(askama::Template)]
+    #[template(
+        source = r#"<li{% if self.0 %}class="active"{% endif %}>"#,
+        ext = "txt",
+        strip = "eager"
+    )]
+    struct StripClass00(bool);
+    assert_eq!(StripClass00(false).render().unwrap(), "<li>");
+    assert_eq!(
+        StripClass00(true).render().unwrap(),
+        r#"<liclass="active">"#
+    );
+
+    #[derive(askama::Template)]
+    #[template(
+        source = r#"<li{% if self.0 %} class="active"{% endif %}>"#,
+        ext = "txt",
+        strip = "eager"
+    )]
+    struct StripClass01(bool);
+    assert_eq!(StripClass01(false).render().unwrap(), "<li>");
+    assert_eq!(
+        StripClass01(true).render().unwrap(),
+        r#"<li class="active">"#
+    );
+
+    #[derive(askama::Template)]
+    #[template(
+        source = r#"<li{% if self.0 %}  class="active"{% endif %}>"#,
+        ext = "txt",
+        strip = "eager"
+    )]
+    struct StripClass02(bool);
+    assert_eq!(StripClass02(false).render().unwrap(), "<li>");
+    assert_eq!(
+        StripClass02(true).render().unwrap(),
+        r#"<li class="active">"#
+    );
+
+    #[derive(askama::Template)]
+    #[template(
+        source = r#"<li {% if self.0 %}class="active"{% endif %}>"#,
+        ext = "txt",
+        strip = "eager"
+    )]
+    struct StripClass10(bool);
+    assert_eq!(StripClass10(false).render().unwrap(), "<li >");
+    assert_eq!(
+        StripClass10(true).render().unwrap(),
+        r#"<li class="active">"#
+    );
+
+    #[derive(askama::Template)]
+    #[template(
+        source = r#"<li {% if self.0 %} class="active"{% endif %}>"#,
+        ext = "txt",
+        strip = "eager"
+    )]
+    struct StripClass11(bool);
+    assert_eq!(StripClass11(false).render().unwrap(), "<li >");
+    assert_eq!(
+        StripClass11(true).render().unwrap(),
+        r#"<li  class="active">"#
+    );
+
+    #[derive(askama::Template)]
+    #[template(
+        source = r#"<li {% if self.0 %}  class="active"{% endif %}>"#,
+        ext = "txt",
+        strip = "eager"
+    )]
+    struct StripClass12(bool);
+    assert_eq!(StripClass12(false).render().unwrap(), "<li >");
+    assert_eq!(
+        StripClass12(true).render().unwrap(),
+        r#"<li  class="active">"#
+    );
+
+    #[derive(askama::Template)]
+    #[template(
+        source = r#"<li  {% if self.0 %}class="active"{% endif %}>"#,
+        ext = "txt",
+        strip = "eager"
+    )]
+    struct StripClass20(bool);
+    assert_eq!(StripClass20(false).render().unwrap(), "<li >");
+    assert_eq!(
+        StripClass20(true).render().unwrap(),
+        r#"<li class="active">"#
+    );
+
+    #[derive(askama::Template)]
+    #[template(
+        source = r#"<li  {% if self.0 %} class="active"{% endif %}>"#,
+        ext = "txt",
+        strip = "eager"
+    )]
+    struct StripClass21(bool);
+    assert_eq!(StripClass21(false).render().unwrap(), "<li >");
+    assert_eq!(
+        StripClass21(true).render().unwrap(),
+        r#"<li  class="active">"#
+    );
+
+    #[derive(askama::Template)]
+    #[template(
+        source = r#"<li  {% if self.0 %}  class="active"{% endif %}>"#,
+        ext = "txt",
+        strip = "eager"
+    )]
+    struct StripClass22(bool);
+    assert_eq!(StripClass22(false).render().unwrap(), "<li >");
+    assert_eq!(
+        StripClass22(true).render().unwrap(),
+        r#"<li  class="active">"#
+    );
+}