Support custom syntax highlighting themes

Related to getzola#419

Introduces once_cell dependency to store extra SyntaxSet, ThemeSet as statics.
Option<SyntaxSet> is no longer stored in the Config Markdown struct.

Gruvbox tmTheme added to test_site, it is taken from
https://github.com/Colorsublime/Colorsublime-Themes (MIT licensed)

components/config/Cargo.toml

@@ -13,6 +13,7 @@ chrono = "0.4"
 globset = "0.4"
 lazy_static = "1"
 syntect = "4.1"
+once_cell = "1.7.2"
 
 errors = { path = "../errors" }
 utils = { path = "../utils" }

components/config/src/config/markup.rs

@@ -1,5 +1,4 @@
 use serde_derive::{Deserialize, Serialize};
-use syntect::parsing::SyntaxSet;
 
 pub const DEFAULT_HIGHLIGHT_THEME: &str = "base16-ocean-dark";
 
@@ -25,9 +24,8 @@ pub struct Markdown {
 
     /// A list of directories to search for additional `.sublime-syntax` files in.
     pub extra_syntaxes: Vec<String>,
-    /// The compiled extra syntaxes into a syntax set
-    #[serde(skip_serializing, skip_deserializing)] // not a typo, 2 are need
-    pub extra_syntax_set: Option<SyntaxSet>,
+    /// A list of directories to search for additional `.tmTheme` files in.
+    pub extra_highlight_themes: Vec<String>,
 }
 
 impl Markdown {
@@ -74,7 +72,7 @@ impl Default for Markdown {
             external_links_no_referrer: false,
             smart_punctuation: false,
             extra_syntaxes: vec![],
-            extra_syntax_set: None,
+            extra_highlight_themes: vec![],
         }
     }
 }

components/config/src/config/mod.rs

@@ -10,10 +10,13 @@ use std::path::{Path, PathBuf};
 
 use globset::{Glob, GlobSet, GlobSetBuilder};
 use serde_derive::{Deserialize, Serialize};
-use syntect::parsing::SyntaxSetBuilder;
+use syntect::highlighting::ThemeSet;
+use syntect::parsing::{SyntaxSet, SyntaxSetBuilder};
 use toml::Value as Toml;
 
-use crate::highlighting::THEME_SET;
+use crate::highlighting::{
+    BUILTIN_HIGHLIGHT_THEME_SET, EXTRA_HIGHLIGHT_THEME_SET, EXTRA_SYNTAX_SET,
+};
 use crate::theme::Theme;
 use errors::{bail, Error, Result};
 use utils::fs::read_file;
@@ -124,11 +127,6 @@ impl Config {
             bail!("A base URL is required in config.toml with key `base_url`");
         }
 
-        let highlight_theme = config.highlight_theme();
-        if !THEME_SET.themes.contains_key(highlight_theme) {
-            bail!("Highlight theme {} defined in config does not exist.", highlight_theme);
-        }
-
         if config.languages.iter().any(|l| l.code == config.default_language) {
             bail!("Default language `{}` should not appear both in `config.default_language` and `config.languages`", config.default_language)
         }
@@ -170,9 +168,55 @@ impl Config {
     /// Parses a config file from the given path
     pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Config> {
         let path = path.as_ref();
-        let content = read_file(path)
-            .map_err(|e| errors::Error::chain("Failed to load config", e))?;
-        Config::parse(&content)
+        let content =
+            read_file(path).map_err(|e| errors::Error::chain("Failed to load config", e))?;
+
+        let config = Config::parse(&content)?;
+        let config_dir = path.parent().unwrap();
+        config.init_extra_syntaxes_and_highlight_themes(config_dir)?;
+
+        Ok(config)
+    }
+
+    // Initialise static once cells: EXTRA_SYNTAX_SET and EXTRA_HIGHLIGHT_THEME_SET
+    // They can only be initialised once, when building a new site the existing values are reused
+    fn init_extra_syntaxes_and_highlight_themes(&self, path: &Path) -> Result<()> {
+        if let Some(extra_syntax_set) = self.load_extra_syntaxes(path)? {
+            if EXTRA_SYNTAX_SET.get().is_none() {
+                EXTRA_SYNTAX_SET.set(extra_syntax_set).unwrap();
+            }
+        }
+        if let Some(extra_highlight_theme_set) = self.load_extra_highlight_themes(path)? {
+            if EXTRA_HIGHLIGHT_THEME_SET.get().is_none() {
+                EXTRA_HIGHLIGHT_THEME_SET.set(extra_highlight_theme_set).unwrap();
+            }
+        }
+
+        // validate that the chosen highlight_theme exists in the loaded highlight theme sets
+        if !BUILTIN_HIGHLIGHT_THEME_SET.themes.contains_key(self.highlight_theme()) {
+            if let Some(extra) = EXTRA_HIGHLIGHT_THEME_SET.get() {
+                if !extra.themes.contains_key(self.highlight_theme()) {
+                    bail!(
+                        "Highlight theme {} not found in the extra theme set",
+                        self.highlight_theme()
+                    )
+                }
+            } else {
+                bail!("Highlight theme {} not available.\n\
+                You can load custom themes by configuring `extra_highlight_themes` with a list of folders containing .tmTheme files", self.highlight_theme())
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Gets the configured highlight theme from the BUILTIN_HIGHLIGHT_THEME_SET or the EXTRA_HIGHLIGHT_THEME_SET
+    pub fn get_highlight_theme(&self) -> &'static syntect::highlighting::Theme {
+        if let Some(theme) = &BUILTIN_HIGHLIGHT_THEME_SET.themes.get(self.highlight_theme()) {
+            theme
+        } else {
+            &EXTRA_HIGHLIGHT_THEME_SET.get().unwrap().themes[self.highlight_theme()]
+        }
     }
 
     /// Temporary, while we have the settings in 2 places
@@ -214,19 +258,35 @@ impl Config {
 
     /// Attempt to load any extra syntax found in the extra syntaxes of the config
     /// TODO: move to markup.rs in 0.14
-    pub fn load_extra_syntaxes(&mut self, base_path: &Path) -> Result<()> {
+    pub fn load_extra_syntaxes(&self, base_path: &Path) -> Result<Option<SyntaxSet>> {
         let extra_syntaxes = self.extra_syntaxes();
         if extra_syntaxes.is_empty() {
-            return Ok(());
+            return Ok(None);
         }
 
         let mut ss = SyntaxSetBuilder::new();
         for dir in &extra_syntaxes {
             ss.add_from_folder(base_path.join(dir), true)?;
         }
-        self.markdown.extra_syntax_set = Some(ss.build());
 
-        Ok(())
+        Ok(Some(ss.build()))
+    }
+
+    /// Attempt to load any theme sets found in the extra highlighting themes of the config
+    /// TODO: move to markup.rs in 0.14
+    pub fn load_extra_highlight_themes(&self, base_path: &Path) -> Result<Option<ThemeSet>> {
+        let extra_highlight_themes = self.markdown.extra_highlight_themes.clone();
+        if extra_highlight_themes.is_empty() {
+            return Ok(None);
+        }
+
+        let mut ts = ThemeSet::new();
+        for dir in &extra_highlight_themes {
+            ts.add_from_folder(base_path.join(dir))?;
+        }
+        let extra_theme_set = Some(ts);
+
+        Ok(extra_theme_set)
     }
 
     /// Makes a url, taking into account that the base url might have a trailing slash

components/config/src/highlighting.rs

@@ -1,4 +1,5 @@
 use lazy_static::lazy_static;
+use once_cell::sync::OnceCell;
 use syntect::dumps::from_binary;
 use syntect::easy::HighlightLines;
 use syntect::highlighting::ThemeSet;
@@ -7,49 +8,60 @@ use syntect::parsing::SyntaxSet;
 use crate::config::Config;
 
 lazy_static! {
-    pub static ref SYNTAX_SET: SyntaxSet = {
+    pub static ref BUILTIN_SYNTAX_SET: SyntaxSet = {
         let ss: SyntaxSet =
             from_binary(include_bytes!("../../../sublime/syntaxes/newlines.packdump"));
         ss
     };
-    pub static ref THEME_SET: ThemeSet =
+    pub static ref BUILTIN_HIGHLIGHT_THEME_SET: ThemeSet =
         from_binary(include_bytes!("../../../sublime/themes/all.themedump"));
 }
 
-pub enum HighlightSource {
-    Theme,
+pub static EXTRA_SYNTAX_SET: OnceCell<SyntaxSet> = OnceCell::new();
+pub static EXTRA_HIGHLIGHT_THEME_SET: OnceCell<ThemeSet> = OnceCell::new();
+
+pub enum SyntaxSource {
+    BuiltIn,
     Extra,
     Plain,
     NotFound,
 }
 
+impl SyntaxSource {
+    pub fn syntax_set(&self) -> &'static SyntaxSet {
+        match self {
+            SyntaxSource::Extra => EXTRA_SYNTAX_SET.get().unwrap(),
+            _ => &BUILTIN_SYNTAX_SET,
+        }
+    }
+}
+
 /// Returns the highlighter and whether it was found in the extra or not
 pub fn get_highlighter(
     language: Option<&str>,
     config: &Config,
-) -> (HighlightLines<'static>, HighlightSource) {
-    let theme = &THEME_SET.themes[config.highlight_theme()];
+) -> (HighlightLines<'static>, SyntaxSource) {
+    let theme = config.get_highlight_theme();
 
-    if let Some(ref lang) = language {
-        if let Some(ref extra_syntaxes) = config.markdown.extra_syntax_set {
-            if let Some(syntax) = extra_syntaxes.find_syntax_by_token(lang) {
-                return (HighlightLines::new(syntax, theme), HighlightSource::Extra);
-            }
-        }
-        // The JS syntax hangs a lot... the TS syntax is probably better anyway.
-        // https://github.com/getzola/zola/issues/1241
-        // https://github.com/getzola/zola/issues/1211
-        // https://github.com/getzola/zola/issues/1174
-        let hacked_lang = if *lang == "js" || *lang == "javascript" { "ts" } else { lang };
-        if let Some(syntax) = SYNTAX_SET.find_syntax_by_token(hacked_lang) {
-            (HighlightLines::new(syntax, theme), HighlightSource::Theme)
-        } else {
-            (
-                HighlightLines::new(SYNTAX_SET.find_syntax_plain_text(), theme),
-                HighlightSource::NotFound,
-            )
-        }
+    let mut source = SyntaxSource::Plain;
+    if let Some(lang) = language {
+        let syntax = EXTRA_SYNTAX_SET
+            .get()
+            .and_then(|extra| {
+                source = SyntaxSource::Extra;
+                extra.find_syntax_by_token(lang)
+            })
+            .or_else(|| {
+                let hacked_lang = if lang == "js" || lang == "javascript" { "ts" } else { lang };
+                source = SyntaxSource::BuiltIn;
+                BUILTIN_SYNTAX_SET.find_syntax_by_token(hacked_lang)
+            })
+            .unwrap_or_else(|| {
+                source = SyntaxSource::NotFound;
+                BUILTIN_SYNTAX_SET.find_syntax_plain_text()
+            });
+        (HighlightLines::new(syntax, theme), source)
     } else {
-        (HighlightLines::new(SYNTAX_SET.find_syntax_plain_text(), theme), HighlightSource::Plain)
+        (HighlightLines::new(BUILTIN_SYNTAX_SET.find_syntax_plain_text(), theme), source)
     }
 }

components/imageproc/src/lib.rs

@@ -1,11 +1,11 @@
-use std::{collections::hash_map::DefaultHasher, io::Write};
 use std::collections::hash_map::Entry as HEntry;
 use std::collections::HashMap;
 use std::fs::{self, File};
 use std::hash::{Hash, Hasher};
 use std::path::{Path, PathBuf};
+use std::{collections::hash_map::DefaultHasher, io::Write};
 
-use image::{EncodableLayout, imageops::FilterType};
+use image::{imageops::FilterType, EncodableLayout};
 use image::{GenericImageView, ImageOutputFormat};
 use lazy_static::lazy_static;
 use rayon::prelude::*;
@@ -159,7 +159,7 @@ impl Format {
                 None => Err(format!("Unsupported image file: {}", source).into()),
             },
             "jpeg" | "jpg" => Ok(Jpeg(jpg_quality)),
-            "png" => Ok(Png),            
+            "png" => Ok(Png),
             "webp" => Ok(WebP(quality)),
             _ => Err(format!("Invalid image format: {}", format).into()),
         }
@@ -189,7 +189,7 @@ impl Format {
         match *self {
             Png => "png",
             Jpeg(_) => "jpg",
-            WebP(_) => "webp"
+            WebP(_) => "webp",
         }
     }
 }
@@ -201,9 +201,9 @@ impl Hash for Format {
 
         let q = match *self {
             Png => 0,
-            Jpeg(q) => q,                        
+            Jpeg(q) => q,
             WebP(None) => 0,
-            WebP(Some(q)) => q
+            WebP(Some(q)) => q,
         };
 
         hasher.write_u8(q);
@@ -316,12 +316,8 @@ impl ImageOp {
             Format::WebP(q) => {
                 let encoder = webp::Encoder::from_image(&img);
                 let memory = match q {
-                    Some(q) => {
-                        encoder.encode(q as f32 / 100.)
-                    }
-                    None => {
-                        encoder.encode_lossless()
-                    }
+                    Some(q) => encoder.encode(q as f32 / 100.),
+                    None => encoder.encode_lossless(),
                 };
                 let mut bytes = memory.as_bytes();
                 f.write_all(&mut bytes)?;

components/rendering/src/markdown.rs

@@ -5,7 +5,6 @@ use syntect::html::{start_highlighted_html_snippet, IncludeBackground};
 
 use crate::context::RenderContext;
 use crate::table_of_contents::{make_table_of_contents, Heading};
-use config::highlighting::THEME_SET;
 use errors::{Error, Result};
 use front_matter::InsertAnchor;
 use utils::site::resolve_internal_link;
@@ -227,7 +226,7 @@ pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<Render
                             return Event::Html("<pre><code>".into());
                         }
 
-                        let theme = &THEME_SET.themes[context.config.highlight_theme()];
+                        let theme = context.config.get_highlight_theme();
                         match kind {
                             cmark::CodeBlockKind::Indented => (),
                             cmark::CodeBlockKind::Fenced(fence_info) => {