Support custom syntax highlighting themes

Related to getzola#419

Introduces once_cell dependency to store SyntaxSets and ThemeSets as statics.
Option<SyntaxSet> is no longer stored in the Config Markdown struct.

Gruvbox tmTheme added to test_site, it is taken from
https://github.com/Colorsublime/Colorsublime-Themes (MIT licensed)

Author: drmason13

Cargo.lock

@@ -12,6 +12,7 @@ serde_derive = "1"
 chrono = "0.4"
 globset = "0.4"
 lazy_static = "1"
+once_cell = "1.8.0"
 # TODO: go back to version 4/5 once https://github.com/trishume/syntect/pull/337 is merged
 syntect = { git = "https://github.com/Keats/syntect.git", branch = "scopestack" }
 unic-langid = "0.9"

components/config/Cargo.toml

@@ -1,7 +1,10 @@
 use std::path::Path;
 
 use serde_derive::{Deserialize, Serialize};
-use syntect::parsing::{SyntaxSet, SyntaxSetBuilder};
+use syntect::{
+    highlighting::ThemeSet,
+    parsing::{SyntaxSet, SyntaxSetBuilder},
+};
 
 use errors::Result;
 
@@ -43,28 +46,32 @@ pub struct Markdown {
     pub external_links_no_referrer: bool,
     /// Whether smart punctuation is enabled (changing quotes, dashes, dots etc in their typographic form)
     pub smart_punctuation: bool,
-
-    /// A list of directories to search for additional `.sublime-syntax` files in.
-    pub extra_syntaxes: Vec<String>,
-    /// The compiled extra syntaxes into a syntax set
-    #[serde(skip_serializing, skip_deserializing)] // not a typo, 2 are need
-    pub extra_syntax_set: Option<SyntaxSet>,
+    /// A list of directories to search for additional `.sublime-syntax` and `.tmTheme` files in.
+    pub extra_syntaxes_and_themes: Vec<String>,
 }
 
 impl Markdown {
-    /// Attempt to load any extra syntax found in the extra syntaxes of the config
-    pub fn load_extra_syntaxes(&mut self, base_path: &Path) -> Result<()> {
-        if self.extra_syntaxes.is_empty() {
-            return Ok(());
+    /// Attempt to load any extra syntaxes and themes found in the extra_syntaxes_and_themes folders
+    pub fn load_extra_syntaxes_and_themes(
+        &self,
+        base_path: &Path,
+    ) -> Result<(Option<SyntaxSet>, Option<ThemeSet>)> {
+        if self.extra_syntaxes_and_themes.is_empty() {
+            return Ok((None, None));
         }
 
         let mut ss = SyntaxSetBuilder::new();
-        for dir in &self.extra_syntaxes {
+        let mut ts = ThemeSet::new();
+        for dir in &self.extra_syntaxes_and_themes {
             ss.add_from_folder(base_path.join(dir), true)?;
+            ts.add_from_folder(base_path.join(dir))?;
         }
-        self.extra_syntax_set = Some(ss.build());
+        let ss = ss.build();
 
-        Ok(())
+        Ok((
+            if ss.syntaxes().is_empty() { None } else { Some(ss) },
+            if ts.themes.is_empty() { None } else { Some(ts) },
+        ))
     }
 
     pub fn has_external_link_tweaks(&self) -> bool {
@@ -110,8 +117,7 @@ impl Default for Markdown {
             external_links_no_follow: false,
             external_links_no_referrer: false,
             smart_punctuation: false,
-            extra_syntaxes: Vec::new(),
-            extra_syntax_set: None,
+            extra_syntaxes_and_themes: vec![],
         }
     }
 }

components/config/src/config/markup.rs

@@ -12,7 +12,10 @@ use globset::{Glob, GlobSet, GlobSetBuilder};
 use serde_derive::{Deserialize, Serialize};
 use toml::Value as Toml;
 
-use crate::highlighting::THEME_SET;
+use crate::highlighting::{
+    get_highlight_theme_by_name, BUILTIN_HIGHLIGHT_THEME_SET, EXTRA_HIGHLIGHT_THEME_SET,
+    EXTRA_SYNTAX_SET,
+};
 use crate::theme::Theme;
 use errors::{bail, Error, Result};
 use utils::fs::read_file;
@@ -117,15 +120,6 @@ impl Config {
             bail!("A base URL is required in config.toml with key `base_url`");
         }
 
-        if config.markdown.highlight_theme != "css" {
-            if !THEME_SET.themes.contains_key(&config.markdown.highlight_theme) {
-                bail!(
-                    "Highlight theme {} defined in config does not exist.",
-                    config.markdown.highlight_theme
-                );
-            }
-        }
-
         languages::validate_code(&config.default_language)?;
         for code in config.languages.keys() {
             languages::validate_code(&code)?;
@@ -165,7 +159,59 @@ impl Config {
         let path = path.as_ref();
         let content =
             read_file(path).map_err(|e| errors::Error::chain("Failed to load config", e))?;
-        Config::parse(&content)
+
+        let config = Config::parse(&content)?;
+        let config_dir = path.parent().unwrap();
+
+        if config.markdown.highlight_theme != "css" {
+            config.init_extra_syntaxes_and_highlight_themes(config_dir)?;
+        }
+
+        Ok(config)
+    }
+
+    // Initialise static once cells: EXTRA_SYNTAX_SET and EXTRA_HIGHLIGHT_THEME_SET
+    // They can only be initialised once, when building a new site the existing values are reused
+    fn init_extra_syntaxes_and_highlight_themes(&self, path: &Path) -> Result<()> {
+        let (loaded_extra_syntaxes, loaded_extra_highlight_themes) =
+            self.markdown.load_extra_syntaxes_and_themes(path)?;
+
+        if let Some(extra_syntax_set) = loaded_extra_syntaxes {
+            if EXTRA_SYNTAX_SET.get().is_none() {
+                EXTRA_SYNTAX_SET.set(extra_syntax_set).unwrap();
+            }
+        }
+        if let Some(extra_highlight_theme_set) = loaded_extra_highlight_themes {
+            if EXTRA_HIGHLIGHT_THEME_SET.get().is_none() {
+                EXTRA_HIGHLIGHT_THEME_SET.set(extra_highlight_theme_set).unwrap();
+            }
+        }
+
+        // validate that the chosen highlight_theme exists in the loaded highlight theme sets
+        if !BUILTIN_HIGHLIGHT_THEME_SET.themes.contains_key(&self.markdown.highlight_theme) {
+            if let Some(extra) = EXTRA_HIGHLIGHT_THEME_SET.get() {
+                if !extra.themes.contains_key(&self.markdown.highlight_theme) {
+                    bail!(
+                        "Highlight theme {} not found in the extra theme set",
+                        self.markdown.highlight_theme
+                    )
+                }
+            } else {
+                bail!("Highlight theme {} not available.\n\
+                You can load custom themes by configuring `extra_syntaxes_and_themes` to include a list of folders containing '.tmTheme' files", self.markdown.highlight_theme)
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Gets the configured highlight theme from the BUILTIN_HIGHLIGHT_THEME_SET or the EXTRA_HIGHLIGHT_THEME_SET
+    pub fn get_highlight_theme(&self) -> Option<&'static syntect::highlighting::Theme> {
+        if self.markdown.highlight_theme == "css" {
+            None
+        } else {
+            Some(get_highlight_theme_by_name(&self.markdown.highlight_theme))
+        }
     }
 
     /// Makes a url, taking into account that the base url might have a trailing slash

components/config/src/config/mod.rs

@@ -1,4 +1,5 @@
 use lazy_static::lazy_static;
+use once_cell::sync::OnceCell;
 use syntect::dumps::from_binary;
 use syntect::highlighting::{Theme, ThemeSet};
 use syntect::parsing::{SyntaxReference, SyntaxSet};
@@ -7,17 +8,28 @@ use crate::config::Config;
 use syntect::html::{css_for_theme_with_class_style, ClassStyle};
 
 lazy_static! {
-    pub static ref SYNTAX_SET: SyntaxSet = {
+    pub static ref BUILTIN_SYNTAX_SET: SyntaxSet = {
         let ss: SyntaxSet =
             from_binary(include_bytes!("../../../sublime/syntaxes/newlines.packdump"));
         ss
     };
-    pub static ref THEME_SET: ThemeSet =
+    pub static ref BUILTIN_HIGHLIGHT_THEME_SET: ThemeSet =
         from_binary(include_bytes!("../../../sublime/themes/all.themedump"));
 }
 
 pub const CLASS_STYLE: ClassStyle = ClassStyle::SpacedPrefixed { prefix: "z-" };
 
+pub static EXTRA_SYNTAX_SET: OnceCell<SyntaxSet> = OnceCell::new();
+pub static EXTRA_HIGHLIGHT_THEME_SET: OnceCell<ThemeSet> = OnceCell::new();
+
+/// Gets an arbitrary theme from the BUILTIN_HIGHLIGHT_THEME_SET or the EXTRA_HIGHLIGHT_THEME_SET
+pub fn get_highlight_theme_by_name(theme_name: &str) -> &'static syntect::highlighting::Theme {
+    &EXTRA_HIGHLIGHT_THEME_SET
+        .get()
+        .and_then(|ts| ts.themes.get(theme_name))
+        .unwrap_or_else(|| &BUILTIN_HIGHLIGHT_THEME_SET.themes[theme_name])
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum HighlightSource {
     /// One of the built-in Zola syntaxes
@@ -30,66 +42,61 @@ pub enum HighlightSource {
     NotFound,
 }
 
-pub struct SyntaxAndTheme<'config> {
-    pub syntax: &'config SyntaxReference,
-    pub syntax_set: &'config SyntaxSet,
+impl HighlightSource {
+    pub fn syntax_set(&self) -> &'static SyntaxSet {
+        match self {
+            HighlightSource::Extra => EXTRA_SYNTAX_SET.get().unwrap(),
+            _ => &BUILTIN_SYNTAX_SET,
+        }
+    }
+}
+
+pub struct SyntaxAndTheme {
+    pub syntax: &'static SyntaxReference,
+    // pub syntax_set: &'static SyntaxSet,
     /// None if highlighting via CSS
-    pub theme: Option<&'config Theme>,
+    pub theme: Option<&'static Theme>,
     pub source: HighlightSource,
 }
 
-pub fn resolve_syntax_and_theme<'config>(
-    language: Option<&'_ str>,
-    config: &'config Config,
-) -> SyntaxAndTheme<'config> {
-    let theme = if config.markdown.highlight_theme != "css" {
-        Some(&THEME_SET.themes[&config.markdown.highlight_theme])
-    } else {
-        None
-    };
+impl SyntaxAndTheme {
+    pub fn syntax_set(&self) -> &'static SyntaxSet {
+        self.source.syntax_set()
+    }
+}
 
-    if let Some(ref lang) = language {
-        if let Some(ref extra_syntaxes) = config.markdown.extra_syntax_set {
-            if let Some(syntax) = extra_syntaxes.find_syntax_by_token(lang) {
-                return SyntaxAndTheme {
-                    syntax,
-                    syntax_set: extra_syntaxes,
-                    theme,
-                    source: HighlightSource::Extra,
-                };
-            }
-        }
-        // The JS syntax hangs a lot... the TS syntax is probably better anyway.
-        // https://github.com/getzola/zola/issues/1241
-        // https://github.com/getzola/zola/issues/1211
-        // https://github.com/getzola/zola/issues/1174
-        let hacked_lang = if *lang == "js" || *lang == "javascript" { "ts" } else { lang };
-        if let Some(syntax) = SYNTAX_SET.find_syntax_by_token(hacked_lang) {
-            SyntaxAndTheme {
-                syntax,
-                syntax_set: &SYNTAX_SET as &SyntaxSet,
-                theme,
-                source: HighlightSource::BuiltIn,
-            }
-        } else {
-            SyntaxAndTheme {
-                syntax: SYNTAX_SET.find_syntax_plain_text(),
-                syntax_set: &SYNTAX_SET as &SyntaxSet,
-                theme,
-                source: HighlightSource::NotFound,
-            }
-        }
+pub fn resolve_syntax_and_theme(language: Option<&str>, config: &Config) -> SyntaxAndTheme {
+    let theme = config.get_highlight_theme();
+
+    let mut source = HighlightSource::Plain;
+    if let Some(lang) = language {
+        let syntax = EXTRA_SYNTAX_SET
+            .get()
+            .and_then(|extra| {
+                source = HighlightSource::Extra;
+                extra.find_syntax_by_token(lang)
+            })
+            .or_else(|| {
+                // The JS syntax hangs a lot... the TS syntax is probably better anyway.
+                // https://github.com/getzola/zola/issues/1241
+                // https://github.com/getzola/zola/issues/1211
+                // https://github.com/getzola/zola/issues/1174
+                let hacked_lang = if lang == "js" || lang == "javascript" { "ts" } else { lang };
+                source = HighlightSource::BuiltIn;
+                BUILTIN_SYNTAX_SET.find_syntax_by_token(hacked_lang)
+            })
+            .unwrap_or_else(|| {
+                source = HighlightSource::NotFound;
+                BUILTIN_SYNTAX_SET.find_syntax_plain_text()
+            });
+
+        SyntaxAndTheme { syntax, theme, source }
     } else {
-        SyntaxAndTheme {
-            syntax: SYNTAX_SET.find_syntax_plain_text(),
-            syntax_set: &SYNTAX_SET as &SyntaxSet,
-            theme,
-            source: HighlightSource::Plain,
-        }
+        SyntaxAndTheme { syntax: BUILTIN_SYNTAX_SET.find_syntax_plain_text(), theme, source }
     }
 }
 
 pub fn export_theme_css(theme_name: &str) -> String {
-    let theme = &THEME_SET.themes[theme_name];
+    let theme = get_highlight_theme_by_name(theme_name);
     css_for_theme_with_class_style(theme, CLASS_STYLE)
 }

components/config/src/highlighting.rs

@@ -140,7 +140,7 @@ impl ResizeOp {
             }
             Fit(w, h) => {
                 if orig_w <= w && orig_h <= h {
-                    return res;  // ie. no-op
+                    return res; // ie. no-op
                 }
 
                 let orig_w_h = orig_w as u64 * h as u64;

components/imageproc/src/lib.rs

@@ -86,7 +86,7 @@ fn bench_render_content_with_highlighting(b: &mut test::Bencher) {
     tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap();
     let permalinks_ctx = HashMap::new();
     let config = Config::default();
-    let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
+    let context = RenderContext::new(&tera, &config, "", "", &permalinks_ctx, InsertAnchor::None);
     b.iter(|| render_content(CONTENT, &context).unwrap());
 }
 
@@ -97,7 +97,7 @@ fn bench_render_content_without_highlighting(b: &mut test::Bencher) {
     let permalinks_ctx = HashMap::new();
     let mut config = Config::default();
     config.markdown.highlight_code = false;
-    let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
+    let context = RenderContext::new(&tera, &config, "", "", &permalinks_ctx, InsertAnchor::None);
     b.iter(|| render_content(CONTENT, &context).unwrap());
 }
 
@@ -108,7 +108,7 @@ fn bench_render_content_no_shortcode(b: &mut test::Bencher) {
     let mut config = Config::default();
     config.markdown.highlight_code = false;
     let permalinks_ctx = HashMap::new();
-    let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
+    let context = RenderContext::new(&tera, &config, "", "", &permalinks_ctx, InsertAnchor::None);
 
     b.iter(|| render_content(&content2, &context).unwrap());
 }
@@ -119,7 +119,7 @@ fn bench_render_shortcodes_one_present(b: &mut test::Bencher) {
     tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap();
     let config = Config::default();
     let permalinks_ctx = HashMap::new();
-    let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
+    let context = RenderContext::new(&tera, &config, "", "", &permalinks_ctx, InsertAnchor::None);
 
     b.iter(|| render_shortcodes(CONTENT, &context));
 }
@@ -132,7 +132,7 @@ fn bench_render_content_no_shortcode_with_emoji(b: &mut test::Bencher) {
     config.markdown.highlight_code = false;
     config.markdown.render_emoji = true;
     let permalinks_ctx = HashMap::new();
-    let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
+    let context = RenderContext::new(&tera, &config, "", "", &permalinks_ctx, InsertAnchor::None);
 
     b.iter(|| render_content(&content2, &context).unwrap());
 }