diff --git a/Cargo.lock b/Cargo.lock index dae52756e0..58c7fa4606 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1203,6 +1203,7 @@ version = "0.1.0" dependencies = [ "config", "errors", + "kamadak-exif", "libs", "serde", "tempfile", @@ -1340,6 +1341,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kamadak-exif" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70494964492bf8e491eb3951c5d70c9627eb7100ede6cc56d748b9a3f302cfb6" +dependencies = [ + "mutate_once", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -1843,6 +1853,12 @@ dependencies = [ "similar", ] +[[package]] +name = "mutate_once" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16cf681a23b4d0a43fc35024c176437f9dcd818db34e0f42ab456a0ee5ad497b" + [[package]] name = "nanorand" version = "0.7.0" diff --git a/components/imageproc/Cargo.toml b/components/imageproc/Cargo.toml index 29c810d801..7f05f975b1 100644 --- a/components/imageproc/Cargo.toml +++ b/components/imageproc/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] serde = { version = "1", features = ["derive"] } +kamadak-exif = "0.5.4" errors = { path = "../errors" } utils = { path = "../utils" } diff --git a/components/imageproc/src/lib.rs b/components/imageproc/src/lib.rs index 179f9b2561..09bf21f56f 100644 --- a/components/imageproc/src/lib.rs +++ b/components/imageproc/src/lib.rs @@ -10,6 +10,7 @@ use image::error::ImageResult; use image::io::Reader as ImgReader; use image::{imageops::FilterType, EncodableLayout}; use image::{ImageFormat, ImageOutputFormat}; +use libs::image::DynamicImage; use libs::{image, once_cell, rayon, regex, svg_metadata, webp}; use once_cell::sync::Lazy; use rayon::prelude::*; @@ -319,6 +320,8 @@ impl ImageOp { None => img, }; + let img = fix_orientation(&img, &self.input_path).unwrap_or(img); + let mut f = File::create(target_path)?; match self.format { @@ -343,6 +346,30 @@ impl ImageOp { } } +/// Apply image rotation based on EXIF data +/// Returns `None` if no transformation is needed +pub fn fix_orientation(img: &DynamicImage, path: &Path) -> Option { + let file = std::fs::File::open(path).ok()?; + let mut buf_reader = std::io::BufReader::new(&file); + let exif_reader = exif::Reader::new(); + let exif = exif_reader.read_from_container(&mut buf_reader).ok()?; + let orientation = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY)?.value.get_uint(0)?; + match orientation { + // Values are taken from the page 30 of + // https://www.cipa.jp/std/documents/e/DC-008-2012_E.pdf + // For more details check http://sylvana.net/jpegcrop/exif_orientation.html + 1 => None, + 2 => Some(img.fliph()), + 3 => Some(img.rotate180()), + 4 => Some(img.flipv()), + 5 => Some(img.fliph().rotate270()), + 6 => Some(img.rotate90()), + 7 => Some(img.fliph().rotate90()), + 8 => Some(img.rotate270()), + _ => None, + } +} + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct EnqueueResponse { /// The final URL for that asset diff --git a/components/imageproc/tests/resize_image.rs b/components/imageproc/tests/resize_image.rs index 3f5b7f8551..507de5e242 100644 --- a/components/imageproc/tests/resize_image.rs +++ b/components/imageproc/tests/resize_image.rs @@ -2,7 +2,8 @@ use std::env; use std::path::{PathBuf, MAIN_SEPARATOR as SLASH}; use config::Config; -use imageproc::{assert_processed_path_matches, ImageMetaResponse, Processor}; +use imageproc::{assert_processed_path_matches, fix_orientation, ImageMetaResponse, Processor}; +use libs::image::{self, DynamicImage, GenericImageView, Pixel}; use libs::once_cell::sync::Lazy; static CONFIG: &str = r#" @@ -153,4 +154,75 @@ fn read_image_metadata_webp() { ); } +#[test] +fn fix_orientation_test() { + fn load_img_and_fix_orientation(img_name: &str) -> DynamicImage { + let path = TEST_IMGS.join(img_name); + let img = image::open(&path).unwrap(); + fix_orientation(&img, &path).unwrap_or(img) + } + + let img = image::open(TEST_IMGS.join("exif_1.jpg")).unwrap(); + assert!(check_img(img)); + assert!(check_img(load_img_and_fix_orientation("exif_0.jpg"))); + assert!(check_img(load_img_and_fix_orientation("exif_1.jpg"))); + assert!(check_img(load_img_and_fix_orientation("exif_2.jpg"))); + assert!(check_img(load_img_and_fix_orientation("exif_3.jpg"))); + assert!(check_img(load_img_and_fix_orientation("exif_4.jpg"))); + assert!(check_img(load_img_and_fix_orientation("exif_5.jpg"))); + assert!(check_img(load_img_and_fix_orientation("exif_6.jpg"))); + assert!(check_img(load_img_and_fix_orientation("exif_7.jpg"))); + assert!(check_img(load_img_and_fix_orientation("exif_8.jpg"))); +} + +#[test] +fn resize_image_applies_exif_rotation() { + // No exif metadata + assert!(resize_and_check("exif_0.jpg")); + // 1: Horizontal (normal) + assert!(resize_and_check("exif_1.jpg")); + // 2: Mirror horizontal + assert!(resize_and_check("exif_2.jpg")); + // 3: Rotate 180 + assert!(resize_and_check("exif_3.jpg")); + // 4: Mirror vertical + assert!(resize_and_check("exif_4.jpg")); + // 5: Mirror horizontal and rotate 270 CW + assert!(resize_and_check("exif_5.jpg")); + // 6: Rotate 90 CW + assert!(resize_and_check("exif_6.jpg")); + // 7: Mirror horizontal and rotate 90 CW + assert!(resize_and_check("exif_7.jpg")); + // 8: Rotate 270 CW + assert!(resize_and_check("exif_8.jpg")); +} + +fn resize_and_check(source_img: &str) -> bool { + let source_path = TEST_IMGS.join(source_img); + let tmpdir = tempfile::tempdir().unwrap().into_path(); + let config = Config::parse(CONFIG).unwrap(); + let mut proc = Processor::new(tmpdir.clone(), &config); + + let resp = proc + .enqueue(source_img.into(), source_path, "scale", Some(16), Some(16), "jpg", None) + .unwrap(); + + proc.do_process().unwrap(); + let processed_path = PathBuf::from(&resp.static_path); + let img = image::open(&tmpdir.join(processed_path)).unwrap(); + check_img(img) +} + +// Checks that an image has the correct orientation +fn check_img(img: DynamicImage) -> bool { + // top left is red + img.get_pixel(0, 0)[0] > 250 // because of the jpeg compression some colors are a bit less than 255 + // top right is green + && img.get_pixel(15, 0)[1] > 250 + // bottom left is blue + && img.get_pixel(0, 15)[2] > 250 + // bottom right is white + && img.get_pixel(15, 15).channels() == [255, 255, 255, 255] +} + // TODO: Test that hash remains the same if physical path is changed diff --git a/components/imageproc/tests/test_imgs/exif_0.jpg b/components/imageproc/tests/test_imgs/exif_0.jpg new file mode 100644 index 0000000000..36beb6863c Binary files /dev/null and b/components/imageproc/tests/test_imgs/exif_0.jpg differ diff --git a/components/imageproc/tests/test_imgs/exif_1.jpg b/components/imageproc/tests/test_imgs/exif_1.jpg new file mode 100644 index 0000000000..c0bb69bf4d Binary files /dev/null and b/components/imageproc/tests/test_imgs/exif_1.jpg differ diff --git a/components/imageproc/tests/test_imgs/exif_2.jpg b/components/imageproc/tests/test_imgs/exif_2.jpg new file mode 100644 index 0000000000..e6f3ef37f4 Binary files /dev/null and b/components/imageproc/tests/test_imgs/exif_2.jpg differ diff --git a/components/imageproc/tests/test_imgs/exif_3.jpg b/components/imageproc/tests/test_imgs/exif_3.jpg new file mode 100644 index 0000000000..e5012999fa Binary files /dev/null and b/components/imageproc/tests/test_imgs/exif_3.jpg differ diff --git a/components/imageproc/tests/test_imgs/exif_4.jpg b/components/imageproc/tests/test_imgs/exif_4.jpg new file mode 100644 index 0000000000..807020eb05 Binary files /dev/null and b/components/imageproc/tests/test_imgs/exif_4.jpg differ diff --git a/components/imageproc/tests/test_imgs/exif_5.jpg b/components/imageproc/tests/test_imgs/exif_5.jpg new file mode 100644 index 0000000000..eb1495ed47 Binary files /dev/null and b/components/imageproc/tests/test_imgs/exif_5.jpg differ diff --git a/components/imageproc/tests/test_imgs/exif_6.jpg b/components/imageproc/tests/test_imgs/exif_6.jpg new file mode 100644 index 0000000000..64a4ec5f49 Binary files /dev/null and b/components/imageproc/tests/test_imgs/exif_6.jpg differ diff --git a/components/imageproc/tests/test_imgs/exif_7.jpg b/components/imageproc/tests/test_imgs/exif_7.jpg new file mode 100644 index 0000000000..a2acb70484 Binary files /dev/null and b/components/imageproc/tests/test_imgs/exif_7.jpg differ diff --git a/components/imageproc/tests/test_imgs/exif_8.jpg b/components/imageproc/tests/test_imgs/exif_8.jpg new file mode 100644 index 0000000000..9bebefa1a3 Binary files /dev/null and b/components/imageproc/tests/test_imgs/exif_8.jpg differ diff --git a/components/markdown/src/markdown.rs b/components/markdown/src/markdown.rs index 289685868b..4972ce041d 100644 --- a/components/markdown/src/markdown.rs +++ b/components/markdown/src/markdown.rs @@ -320,13 +320,12 @@ pub fn markdown_to_html( }; } + let mut accumulated_block = String::new(); for (event, mut range) in Parser::new_ext(content, opts).into_offset_iter() { match event { Event::Text(text) => { - if let Some(ref mut code_block) = code_block { - let html; + if let Some(ref mut _code_block) = code_block { if contains_shortcode(text.as_ref()) { - let mut accumulated_block = String::new(); // mark the start of the code block events let stack_start = events.len(); render_shortcodes!(true, text, range); @@ -344,13 +343,12 @@ pub fn markdown_to_html( } } } - html = code_block.highlight(&accumulated_block); + // remove all the original events from shortcode rendering events.truncate(stack_start); } else { - html = code_block.highlight(&text); + accumulated_block += &text; } - events.push(Event::Html(html.into())); } else { let text = if context.config.markdown.render_emoji { EMOJI_REPLACER.replace_all(&text).to_string().into() @@ -376,6 +374,12 @@ pub fn markdown_to_html( events.push(Event::Html(begin.into())); } Event::End(Tag::CodeBlock(_)) => { + if let Some(ref mut code_block) = code_block { + let html = code_block.highlight(&accumulated_block); + events.push(Event::Html(html.into())); + accumulated_block.clear(); + } + // reset highlight and close the code block code_block = None; events.push(Event::Html("\n".into())); diff --git a/components/markdown/tests/codeblocks.rs b/components/markdown/tests/codeblocks.rs index 6aa9dc7424..d12e0f32bc 100644 --- a/components/markdown/tests/codeblocks.rs +++ b/components/markdown/tests/codeblocks.rs @@ -239,6 +239,16 @@ bar insta::assert_snapshot!(body); } +#[test] +fn can_add_line_numbers_windows_eol() { + let body = render_codeblock( + "```linenos\r\nfoo\r\nbar\r\n```\r\n", + true, + ); + insta::assert_snapshot!(body); +} + + #[test] fn can_add_line_numbers_with_lineno_start() { let body = render_codeblock( diff --git a/components/markdown/tests/snapshots/codeblocks__can_add_line_numbers_windows_eol.snap b/components/markdown/tests/snapshots/codeblocks__can_add_line_numbers_windows_eol.snap new file mode 100644 index 0000000000..1e79755a7e --- /dev/null +++ b/components/markdown/tests/snapshots/codeblocks__can_add_line_numbers_windows_eol.snap @@ -0,0 +1,9 @@ +--- +source: components/markdown/tests/codeblocks.rs +assertion_line: 248 +expression: body +--- +
1foo
+
2bar
+
+ diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index 92c073b387..4b3b94ba2a 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -218,7 +218,7 @@ impl Site { // is it a section or not? if path.is_dir() { // if we are processing a section we have to collect - // index files for all languages and process them simultaniously + // index files for all languages and process them simultaneously // before any of the pages let index_files = WalkDir::new(&path) .follow_links(true) @@ -228,15 +228,12 @@ impl Site { Err(_) => None, Ok(f) => { let path_str = f.path().file_name().unwrap().to_str().unwrap(); + // https://github.com/getzola/zola/issues/1244 if f.path().is_file() && allowed_index_filenames.iter().any(|s| s == path_str) { Some(f) } else { - // https://github.com/getzola/zola/issues/1244 - if path_str.starts_with("_index.") { - println!("Expected a section filename, got `{}`. Allowed values: `{:?}`", path_str, &allowed_index_filenames); - } None } } diff --git a/components/utils/src/anchors.rs b/components/utils/src/anchors.rs index 642e58937f..fe86e96082 100644 --- a/components/utils/src/anchors.rs +++ b/components/utils/src/anchors.rs @@ -1,4 +1,5 @@ use libs::regex::Regex; +use libs::regex::escape; pub fn has_anchor_id(content: &str, anchor: &str) -> bool { let checks = anchor_id_checks(anchor); @@ -6,7 +7,8 @@ pub fn has_anchor_id(content: &str, anchor: &str) -> bool { } fn anchor_id_checks(anchor: &str) -> Regex { - Regex::new(&format!(r#" (?i)(id|name) *= *("|')*{}("|'| |>)+"#, anchor)).unwrap() + Regex::new(&format!(r#"\s(?i)(id|name) *= *("|')*{}("|'| |>)+"#, + escape(anchor))).unwrap() } #[cfg(test)] @@ -39,5 +41,16 @@ mod tests { // Case variants assert!(m(r#""#)); assert!(m(r#""#)); + + // Newline variants + assert!(m(r#""#)); + + // Escaped Anchors + assert!(check("fred?george", r#""#)); + assert!(check("fred.george", r#""#)); + + // Non matchers + assert!(!m(r#""#)); } } diff --git a/docs/content/documentation/getting-started/configuration.md b/docs/content/documentation/getting-started/configuration.md index 4ddaac2822..f8f2d473fc 100644 --- a/docs/content/documentation/getting-started/configuration.md +++ b/docs/content/documentation/getting-started/configuration.md @@ -93,8 +93,8 @@ build_search_index = false # When set to "true", all code blocks are highlighted. highlight_code = false -# A list of directories used to search for additional `.sublime-syntax` files. -extra_syntaxes = [] +# A list of directories used to search for additional `.sublime-syntax` and `.tmTheme` files. +extra_syntaxes_and_themes = [] # The theme to use for code highlighting. # See below for list of allowed values.