Merge branch 'next' of github.com:getzola/zola

* 'next' of github.com:getzola/zola:
  Apply orientation transformation based on EXIF data (getzola#1912)
  Stop printing invalid files in dirs
  fix windows EOL in markdown files (getzola#1911)
  Properly escape anchor names (getzola#1908)
  Allow new lines before anchor (getzola#1905)
  Fix incorrect documentation for syntax directory (getzola#1901)

components/imageproc/Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 serde = { version = "1", features = ["derive"] }
+kamadak-exif = "0.5.4"
 
 errors = { path = "../errors" }
 utils = { path = "../utils" }

components/imageproc/src/lib.rs

@@ -10,6 +10,7 @@ use image::error::ImageResult;
 use image::io::Reader as ImgReader;
 use image::{imageops::FilterType, EncodableLayout};
 use image::{ImageFormat, ImageOutputFormat};
+use libs::image::DynamicImage;
 use libs::{image, once_cell, rayon, regex, svg_metadata, webp};
 use once_cell::sync::Lazy;
 use rayon::prelude::*;
@@ -319,6 +320,8 @@ impl ImageOp {
             None => img,
         };
 
+        let img = fix_orientation(&img, &self.input_path).unwrap_or(img);
+
         let mut f = File::create(target_path)?;
 
         match self.format {
@@ -343,6 +346,30 @@ impl ImageOp {
     }
 }
 
+/// Apply image rotation based on EXIF data
+/// Returns `None` if no transformation is needed
+pub fn fix_orientation(img: &DynamicImage, path: &Path) -> Option<DynamicImage> {
+    let file = std::fs::File::open(path).ok()?;
+    let mut buf_reader = std::io::BufReader::new(&file);
+    let exif_reader = exif::Reader::new();
+    let exif = exif_reader.read_from_container(&mut buf_reader).ok()?;
+    let orientation = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY)?.value.get_uint(0)?;
+    match orientation {
+        // Values are taken from the page 30 of
+        // https://www.cipa.jp/std/documents/e/DC-008-2012_E.pdf
+        // For more details check http://sylvana.net/jpegcrop/exif_orientation.html
+        1 => None,
+        2 => Some(img.fliph()),
+        3 => Some(img.rotate180()),
+        4 => Some(img.flipv()),
+        5 => Some(img.fliph().rotate270()),
+        6 => Some(img.rotate90()),
+        7 => Some(img.fliph().rotate90()),
+        8 => Some(img.rotate270()),
+        _ => None,
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct EnqueueResponse {
     /// The final URL for that asset

components/imageproc/tests/resize_image.rs

@@ -2,7 +2,8 @@ use std::env;
 use std::path::{PathBuf, MAIN_SEPARATOR as SLASH};
 
 use config::Config;
-use imageproc::{assert_processed_path_matches, ImageMetaResponse, Processor};
+use imageproc::{assert_processed_path_matches, fix_orientation, ImageMetaResponse, Processor};
+use libs::image::{self, DynamicImage, GenericImageView, Pixel};
 use libs::once_cell::sync::Lazy;
 
 static CONFIG: &str = r#"
@@ -153,4 +154,75 @@ fn read_image_metadata_webp() {
     );
 }
 
+#[test]
+fn fix_orientation_test() {
+    fn load_img_and_fix_orientation(img_name: &str) -> DynamicImage {
+        let path = TEST_IMGS.join(img_name);
+        let img = image::open(&path).unwrap();
+        fix_orientation(&img, &path).unwrap_or(img)
+    }
+
+    let img = image::open(TEST_IMGS.join("exif_1.jpg")).unwrap();
+    assert!(check_img(img));
+    assert!(check_img(load_img_and_fix_orientation("exif_0.jpg")));
+    assert!(check_img(load_img_and_fix_orientation("exif_1.jpg")));
+    assert!(check_img(load_img_and_fix_orientation("exif_2.jpg")));
+    assert!(check_img(load_img_and_fix_orientation("exif_3.jpg")));
+    assert!(check_img(load_img_and_fix_orientation("exif_4.jpg")));
+    assert!(check_img(load_img_and_fix_orientation("exif_5.jpg")));
+    assert!(check_img(load_img_and_fix_orientation("exif_6.jpg")));
+    assert!(check_img(load_img_and_fix_orientation("exif_7.jpg")));
+    assert!(check_img(load_img_and_fix_orientation("exif_8.jpg")));
+}
+
+#[test]
+fn resize_image_applies_exif_rotation() {
+    // No exif metadata
+    assert!(resize_and_check("exif_0.jpg"));
+    // 1: Horizontal (normal)
+    assert!(resize_and_check("exif_1.jpg"));
+    // 2: Mirror horizontal
+    assert!(resize_and_check("exif_2.jpg"));
+    // 3: Rotate 180
+    assert!(resize_and_check("exif_3.jpg"));
+    // 4: Mirror vertical
+    assert!(resize_and_check("exif_4.jpg"));
+    // 5: Mirror horizontal and rotate 270 CW
+    assert!(resize_and_check("exif_5.jpg"));
+    // 6: Rotate 90 CW
+    assert!(resize_and_check("exif_6.jpg"));
+    // 7: Mirror horizontal and rotate 90 CW
+    assert!(resize_and_check("exif_7.jpg"));
+    // 8: Rotate 270 CW
+    assert!(resize_and_check("exif_8.jpg"));
+}
+
+fn resize_and_check(source_img: &str) -> bool {
+    let source_path = TEST_IMGS.join(source_img);
+    let tmpdir = tempfile::tempdir().unwrap().into_path();
+    let config = Config::parse(CONFIG).unwrap();
+    let mut proc = Processor::new(tmpdir.clone(), &config);
+
+    let resp = proc
+        .enqueue(source_img.into(), source_path, "scale", Some(16), Some(16), "jpg", None)
+        .unwrap();
+
+    proc.do_process().unwrap();
+    let processed_path = PathBuf::from(&resp.static_path);
+    let img = image::open(&tmpdir.join(processed_path)).unwrap();
+    check_img(img)
+}
+
+// Checks that an image has the correct orientation
+fn check_img(img: DynamicImage) -> bool {
+    // top left is red
+    img.get_pixel(0, 0)[0] > 250 // because of the jpeg compression some colors are a bit less than 255
+    // top right is green
+        && img.get_pixel(15, 0)[1] > 250
+    // bottom left is blue
+        && img.get_pixel(0, 15)[2] > 250
+    // bottom right is white
+        && img.get_pixel(15, 15).channels() == [255, 255, 255, 255]
+}
+
 // TODO: Test that hash remains the same if physical path is changed

components/markdown/src/markdown.rs

@@ -320,13 +320,12 @@ pub fn markdown_to_html(
             };
         }
 
+        let mut accumulated_block = String::new();
         for (event, mut range) in Parser::new_ext(content, opts).into_offset_iter() {
             match event {
                 Event::Text(text) => {
-                    if let Some(ref mut code_block) = code_block {
-                        let html;
+                    if let Some(ref mut _code_block) = code_block {
                         if contains_shortcode(text.as_ref()) {
-                            let mut accumulated_block = String::new();
                             // mark the start of the code block events
                             let stack_start = events.len();
                             render_shortcodes!(true, text, range);
@@ -344,13 +343,12 @@ pub fn markdown_to_html(
                                     }
                                 }
                             }
-                            html = code_block.highlight(&accumulated_block);
+
                             // remove all the original events from shortcode rendering
                             events.truncate(stack_start);
                         } else {
-                            html = code_block.highlight(&text);
+                            accumulated_block += &text;
                         }
-                        events.push(Event::Html(html.into()));
                     } else {
                         let text = if context.config.markdown.render_emoji {
                             EMOJI_REPLACER.replace_all(&text).to_string().into()
@@ -376,6 +374,12 @@ pub fn markdown_to_html(
                     events.push(Event::Html(begin.into()));
                 }
                 Event::End(Tag::CodeBlock(_)) => {
+                    if let Some(ref mut code_block) = code_block {
+                        let html = code_block.highlight(&accumulated_block);
+                        events.push(Event::Html(html.into()));
+                        accumulated_block.clear();
+                    }
+
                     // reset highlight and close the code block
                     code_block = None;
                     events.push(Event::Html("</code></pre>\n".into()));

components/markdown/tests/codeblocks.rs

@@ -239,6 +239,16 @@ bar
     insta::assert_snapshot!(body);
 }
 
+#[test]
+fn can_add_line_numbers_windows_eol() {
+    let body = render_codeblock(
+        "```linenos\r\nfoo\r\nbar\r\n```\r\n",
+        true,
+    );
+    insta::assert_snapshot!(body);
+} 
+
+
 #[test]
 fn can_add_line_numbers_with_lineno_start() {
     let body = render_codeblock(

components/markdown/tests/snapshots/codeblocks__can_add_line_numbers_windows_eol.snap

@@ -0,0 +1,9 @@
+---
+source: components/markdown/tests/codeblocks.rs
+assertion_line: 248
+expression: body
+---
+<pre data-linenos style="background-color:#2b303b;color:#c0c5ce;"><code><table><tbody><tr><td>1</td><td><span>foo
+</span></td></tr><tr><td>2</td><td><span>bar
+</span></td></tr></tbody></table></code></pre>
+

components/site/src/lib.rs

@@ -218,7 +218,7 @@ impl Site {
             // is it a section or not?
             if path.is_dir() {
                 // if we are processing a section we have to collect
-                // index files for all languages and process them simultaniously
+                // index files for all languages and process them simultaneously
                 // before any of the pages
                 let index_files = WalkDir::new(&path)
                     .follow_links(true)
@@ -228,15 +228,12 @@ impl Site {
                         Err(_) => None,
                         Ok(f) => {
                             let path_str = f.path().file_name().unwrap().to_str().unwrap();
+                            // https://github.com/getzola/zola/issues/1244
                             if f.path().is_file()
                                 && allowed_index_filenames.iter().any(|s| s == path_str)
                             {
                                 Some(f)
                             } else {
-                                // https://github.com/getzola/zola/issues/1244
-                                if path_str.starts_with("_index.") {
-                                    println!("Expected a section filename, got `{}`. Allowed values: `{:?}`", path_str, &allowed_index_filenames);
-                                }
                                 None
                             }
                         }

components/utils/src/anchors.rs

@@ -1,12 +1,14 @@
 use libs::regex::Regex;
+use libs::regex::escape;
 
 pub fn has_anchor_id(content: &str, anchor: &str) -> bool {
     let checks = anchor_id_checks(anchor);
     checks.is_match(content)
 }
 
 fn anchor_id_checks(anchor: &str) -> Regex {
-    Regex::new(&format!(r#" (?i)(id|name) *= *("|')*{}("|'| |>)+"#, anchor)).unwrap()
+    Regex::new(&format!(r#"\s(?i)(id|name) *= *("|')*{}("|'| |>)+"#,
+                        escape(anchor))).unwrap()
 }
 
 #[cfg(test)]
@@ -39,5 +41,16 @@ mod tests {
         // Case variants
         assert!(m(r#"<a ID="fred">"#));
         assert!(m(r#"<a iD="fred">"#));
+
+        // Newline variants
+        assert!(m(r#"<a
+id="fred">"#));
+
+        // Escaped Anchors
+        assert!(check("fred?george", r#"<a id="fred?george">"#));
+        assert!(check("fred.george", r#"<a id="fred.george">"#));
+
+        // Non matchers
+        assert!(!m(r#"<a notid="fred">"#));
     }
 }

docs/content/documentation/getting-started/configuration.md

@@ -93,8 +93,8 @@ build_search_index = false
 # When set to "true", all code blocks are highlighted.
 highlight_code = false
 
-# A list of directories used to search for additional `.sublime-syntax` files.
-extra_syntaxes = []
+# A list of directories used to search for additional `.sublime-syntax` and `.tmTheme` files.
+extra_syntaxes_and_themes = []
 
 # The theme to use for code highlighting.
 # See below for list of allowed values.