feat(templ): template before html conversion!

also fix dt link injection

Author: fiji-flo

Diff for: Cargo.lock

@@ -1,3 +1,5 @@
+use std::borrow::Cow;
+
 use rari_types::fm_types::PageType;
 use rari_types::globals::{base_url, content_branch, git_history, popularities};
 use rari_types::locale::Locale;
@@ -129,17 +131,13 @@ pub fn make_toc(sections: &[BuildSection], with_h3: bool) -> Vec<TocEntry> {
 }
 
 pub fn build_content<T: PageLike>(doc: &T) -> Result<PageContent, DocError> {
-    let html = render_md_to_html(
-        doc.content(),
-        doc.locale(),
-        Some(&doc.full_path().display()),
-    )?;
     let ks_rendered_doc = if let Some(rari_env) = &doc.rari_env() {
-        render(rari_env, &html)?
+        Cow::Owned(render(rari_env, doc.content())?)
     } else {
-        html
+        Cow::Borrowed(doc.content())
     };
-    let post_processed_html = post_process_html(&ks_rendered_doc, doc, false)?;
+    let html = render_md_to_html(&ks_rendered_doc, doc.locale())?;
+    let post_processed_html = post_process_html(&html, doc, false)?;
     let fragment = Html::parse_fragment(&post_processed_html);
     let (sections, summary) = split_sections(&fragment).expect("DOOM");
     let toc = make_toc(&sections, matches!(doc.page_type(), PageType::Curriculum));

Diff for: crates/rari-doc/src/docs/build.rs

@@ -1,24 +1,20 @@
 use std::collections::HashMap;
-use std::fmt::Display;
 use std::fs::read_to_string;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use rari_md::m2h;
 use rari_types::fm_types::{FeatureStatus, PageType};
-use rari_types::globals::deny_warnings;
 use rari_types::locale::Locale;
 use rari_types::RariEnv;
 use serde::{Deserialize, Serialize};
 use serde_yaml::Value;
-use tracing::warn;
 use validator::Validate;
 
 use super::page::{Page, PageCategory, PageLike, PageReader};
 use crate::cached_readers::{page_from_static_files, CACHED_PAGE_FILES};
 use crate::error::DocError;
 use crate::resolve::{build_url, url_to_path_buf};
-use crate::templ::parser::{decode_ks, encode_ks};
 use crate::utils::{locale_and_typ_from_path, root_for_locale, split_fm, t_or_vec};
 
 /*
@@ -238,26 +234,8 @@ fn read_doc(path: impl Into<PathBuf>) -> Result<Doc, DocError> {
     })
 }
 
-pub fn render_md_to_html(
-    input: &str,
-    locale: Locale,
-    path: Option<&impl Display>,
-) -> Result<String, DocError> {
-    let (encoded, before) = encode_ks(input)?;
-    let encoded_html = m2h(&encoded, locale)?;
-    let (html, after) = decode_ks(&encoded_html)?;
-    if before != after {
-        if deny_warnings() {
-            return Err(DocError::InvalidTempl(
-                path.map(|s| s.to_string()).unwrap_or_default(),
-            ));
-        }
-        warn!(
-            "invalid templ: {}",
-            path.map(|s| s.to_string()).unwrap_or_default()
-        );
-    }
-
+pub fn render_md_to_html(input: &str, locale: Locale) -> Result<String, DocError> {
+    let html = m2h(input, locale)?;
     Ok(html)
 }
 

Diff for: crates/rari-doc/src/docs/doc.rs

@@ -87,7 +87,8 @@ pub fn post_process_html<T: PageLike>(
 
             Ok(())
         }),
-        element!("dt:first-child:not(a)", |el| {
+        element!("dt[data-add-link]", |el| {
+            el.remove_attribute("data-add-link");
             if let Some(id) = el.get_attribute("id") {
                 el.prepend(&format!("<a href=\"#{id}\">"), ContentType::Html);
                 el.append("</a>", ContentType::Html);

Diff for: crates/rari-doc/src/html/rewriter.rs

@@ -1,7 +1,3 @@
-use std::fmt::Write;
-
-use base64::engine::general_purpose::STANDARD;
-use base64::Engine;
 use pest::iterators::Pair;
 use pest::Parser;
 use rari_types::{Arg, Quotes};
@@ -115,92 +111,6 @@ pub(crate) fn parse(input: &str) -> Result<Vec<Token>, DocError> {
     Ok(tokens)
 }
 
-fn encode_macro(s: &str, out: &mut String) -> Result<(), DocError> {
-    Ok(write!(
-        out,
-        "!::::{}::::!",
-        STANDARD.encode(&s[2..(s.len() - 2)])
-    )?)
-}
-
-fn decode_macro(s: &str, out: &mut String) -> Result<(), DocError> {
-    Ok(write!(
-        out,
-        "{{{{{}}}}}",
-        std::str::from_utf8(&STANDARD.decode(s)?)?
-    )?)
-}
-
-pub(crate) fn encode_ks(input: &str) -> Result<(String, i32), DocError> {
-    let tokens = parse(input)?;
-    let mut encoded = String::with_capacity(input.len());
-    let mut num_macros = 0;
-    for token in tokens {
-        match token {
-            Token::Text(t) => {
-                encoded.push_str(&input[t.start..t.end]);
-            }
-            Token::Macro(t) => {
-                num_macros += 1;
-                encode_macro(&input[t.start..t.end], &mut encoded)?
-            }
-        }
-    }
-    Ok((encoded, num_macros))
-}
-
-fn _strip_escape_residues(s: &str) -> &str {
-    let s = s.strip_prefix("&gt;").or(s.strip_prefix('>')).unwrap_or(s);
-    let s = s
-        .strip_suffix("!&lt;")
-        .or(s.strip_suffix("!<"))
-        .unwrap_or(s);
-    s
-}
-
-pub(crate) fn decode_ks(input: &str) -> Result<(String, i32), DocError> {
-    let mut decoded = String::with_capacity(input.len());
-    let mut num_macros = 0;
-    // We're splitting only by `!-- ks___` because e.g. ks in a <pre> will be escaped.
-    if !input.contains("!::::") {
-        return Ok((input.to_string(), 0));
-    }
-    let mut frags = vec![];
-    for frag in input.split("!::::") {
-        let has_ks = frag.contains("::::!");
-        for (i, sub_frag) in frag.splitn(2, "::::!").enumerate() {
-            if i == 0 && has_ks {
-                num_macros += 1;
-                frags.push(sub_frag);
-                //decode_macro(sub_frag, &mut decoded)?;
-            } else {
-                //decoded.push_str(strip_escape_residues(sub_frag))
-                frags.push(sub_frag)
-            }
-        }
-    }
-    for i in 0..frags.len() {
-        if i % 2 == 1
-            && i < frags.len() + 1
-            && frags[i - 1].ends_with("<p>")
-            && frags[i + 1].starts_with("</p>")
-        {
-            frags[i - 1] = frags[i - 1].strip_suffix("<p>").unwrap();
-            frags[i + 1] = frags[i + 1].strip_prefix("</p>").unwrap();
-        }
-    }
-
-    for (i, frag) in frags.iter().enumerate() {
-        if i % 2 == 1 {
-            decode_macro(frag, &mut decoded)?;
-        } else {
-            decoded.push_str(frag)
-        }
-    }
-
-    Ok((decoded, num_macros))
-}
-
 #[cfg(test)]
 mod test {
     use super::*;
@@ -236,31 +146,4 @@ mod test {
         let p = parse(r#"foo {{foo(0.1)}} bar"#);
         println!("{:#?}", p);
     }
-
-    #[test]
-    fn ks_escape() -> Result<(), DocError> {
-        let ks = r#"Foo {{jsxref("Array",,1,true) }}bar {{ foo }}"#;
-        let enc = encode_ks(ks)?;
-        let dec = decode_ks(&enc.0)?;
-        assert_eq!(ks, dec.0);
-        Ok(())
-    }
-
-    #[test]
-    fn ks_escape_2() -> Result<(), DocError> {
-        let ks = r#"<{{jsxref("Array",,1,true) }}>bar {{ foo }}"#;
-        let enc = encode_ks(ks)?;
-        let dec = decode_ks(&enc.0)?;
-        assert_eq!(ks, dec.0);
-        Ok(())
-    }
-
-    #[test]
-    fn ks_escape_3() -> Result<(), DocError> {
-        let ks = r#"{{foo}}{{foo-bar}}"#;
-        let enc = encode_ks(ks)?;
-        let dec = decode_ks(&enc.0)?;
-        assert_eq!(ks, dec.0);
-        Ok(())
-    }
 }

Diff for: crates/rari-doc/src/templ/parser.rs

@@ -13,3 +13,4 @@ once_cell = "1"
 thiserror = "1"
 comrak = { version = "0.24", default-features = false, features = ["syntect"] }
 rari-types = { path = "../rari-types" }
+itertools = "0.13"

Diff for: crates/rari-md/Cargo.toml

@@ -97,6 +97,7 @@ pub(crate) fn is_callout<'a>(block_quote: &'a AstNode<'a>, locale: Locale) -> Op
                     }
 
                     if text.starts_with(NoteCard::Warning.prefix_for_locale(locale)) {
+                        grand_child.detach();
                         return Some(NoteCard::Warning);
                     }
                     if text.starts_with(NoteCard::Note.prefix_for_locale(locale)) {

Diff for: crates/rari-md/src/bq.rs

@@ -16,6 +16,7 @@ use comrak::nodes::{
     TableAlignment,
 };
 use comrak::{ComrakOptions, ComrakPlugins, Options};
+use itertools::Itertools;
 use once_cell::sync::Lazy;
 use rari_types::locale::Locale;
 
@@ -472,6 +473,28 @@ impl<'o> HtmlFormatter<'o> {
         }
     }
 
+    fn next_is_link<'a>(node: &'a AstNode<'a>) -> bool {
+        if let Some(child) = node.children().next() {
+            if matches!(child.data.borrow().value, NodeValue::Link(_)) {
+                return true;
+            }
+            if let NodeValue::HtmlInline(res) = &child.data.borrow().value {
+                return res.starts_with("<a ");
+            }
+            if matches!(child.data.borrow().value, NodeValue::Paragraph) {
+                if let Some(child) = child.children().next() {
+                    if matches!(child.data.borrow().value, NodeValue::Link(_)) {
+                        return true;
+                    }
+                    if let NodeValue::HtmlInline(res) = &child.data.borrow().value {
+                        return res.starts_with("<a ");
+                    }
+                }
+            }
+        }
+        false
+    }
+
     fn format_node<'a>(
         &mut self,
         node: &'a AstNode<'a>,
@@ -568,6 +591,9 @@ impl<'o> HtmlFormatter<'o> {
                     let mut id = String::from_utf8(text_content).unwrap();
                     id = self.anchorizer.anchorize(id);
                     write!(self.output, "<dt id=\"{}\"", id)?;
+                    if !Self::next_is_link(node) {
+                        write!(self.output, " data-add-link")?;
+                    }
                     self.render_sourcepos(node)?;
                     self.output.write_all(b">")?;
                 } else {
@@ -682,10 +708,13 @@ impl<'o> HtmlFormatter<'o> {
                                 pre_attributes.extend(code_attributes);
                                 let with_code = if let Some(cls) = pre_attributes.get_mut("class") {
                                     if !ncb.info.is_empty() {
-                                        *cls = format!(
-                                            "brush: {} notranslate",
-                                            ncb.info.strip_suffix("-nolint").unwrap_or(&ncb.info)
-                                        );
+                                        let langs = ncb
+                                            .info
+                                            .split_ascii_whitespace()
+                                            .map(|s| s.strip_suffix("-nolint").unwrap_or(s))
+                                            .join(" ");
+
+                                        *cls = format!("brush: {langs} notranslate",);
                                         &ncb.info != "plain"
                                     } else {
                                         *cls = "notranslate".to_string();

Diff for: crates/rari-md/src/html.rs

@@ -35,7 +35,7 @@ where
 pub fn m2h(input: &str, locale: Locale) -> Result<String, MarkdownError> {
     let arena = Arena::new();
     let mut options = ComrakOptions::default();
-    options.extension.tagfilter = true;
+    options.extension.tagfilter = false;
     options.render.unsafe_ = true;
     options.extension.table = true;
     options.extension.autolink = true;