fix(diff): fix subpages and banners

Also add support for --sidebars in diff-test.

Author: fiji-flo

Diff for: Cargo.lock

@@ -80,6 +80,7 @@ reqwest = { version = "0.12", default-features = false, features = [
     "gzip",
 ] }
 indoc = "2"
+base64 = "0.22"
 
 
 [dependencies]

Diff for: Cargo.toml

@@ -14,11 +14,13 @@ itertools.workspace = true
 regex.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+base64.workspace = true
 
 jsonpath_lib = "0.3.0"
 prettydiff = "0.7"
 html-minifier = "5"
 ansi-to-html = "0.2"
 similar = "2"
 quick-xml = "0.36"
+sha2 = "0.10"
 clap = { version = "4.5.1", features = ["derive"] }

Diff for: crates/diff-test/Cargo.toml

@@ -1,14 +1,15 @@
 use std::cmp::max;
-use std::collections::{BTreeMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fs;
 use std::fs::File;
 use std::io::Write;
 use std::path::Path;
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering::Relaxed;
-use std::sync::LazyLock;
+use std::sync::{Arc, LazyLock, RwLock};
 
 use anyhow::{anyhow, Error};
+use base64::prelude::{Engine as _, BASE64_STANDARD_NO_PAD};
 use clap::{Args, Parser, Subcommand};
 use ignore::types::TypesBuilder;
 use ignore::WalkBuilder;
@@ -18,6 +19,7 @@ use prettydiff::{diff_lines, diff_words};
 use rayon::prelude::*;
 use regex::Regex;
 use serde_json::Value;
+use sha2::{Digest, Sha256};
 use xml::fmt_html;
 
 mod xml;
@@ -147,6 +149,8 @@ struct BuildArgs {
     value: bool,
     #[arg(short, long)]
     verbose: bool,
+    #[arg(long)]
+    sidebars: bool,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -175,7 +179,7 @@ const IGNORE: &[&str] = &[
     "doc.popularity",
     "doc.source.github_url",
     "doc.source.last_commit_url",
-    "doc.sidebarHTML",
+    //"doc.sidebarHTML",
     "doc.sidebarMacro",
     "doc.hasMathML",
     "doc.other_translations",
@@ -187,12 +191,16 @@ static WS_DIFF: LazyLock<Regex> =
 static DATA_FLAW_SRC: LazyLock<Regex> =
     LazyLock::new(|| Regex::new(r#" data-flaw-src="[^"]+""#).unwrap());
 
+static DIFF_MAP: LazyLock<Arc<RwLock<HashMap<String, String>>>> =
+    LazyLock::new(|| Arc::new(RwLock::new(HashMap::new())));
+
 fn full_diff(
     lhs: &Value,
     rhs: &Value,
     path: &[PathIndex],
     diff: &mut BTreeMap<String, String>,
     fast: bool,
+    sidebars: bool,
 ) {
     if path.len() == 1 {
         if let PathIndex::Object(s) = &path[0] {
@@ -203,7 +211,7 @@ fn full_diff(
     }
     if lhs != rhs {
         let key = make_key(path);
-        if IGNORE.iter().any(|i| key.starts_with(i)) {
+        if IGNORE.iter().any(|i| key.starts_with(i)) || key == "doc.sidebarHTML" && !sidebars {
             return;
         }
         match (lhs, rhs) {
@@ -218,6 +226,7 @@ fn full_diff(
                         &path,
                         diff,
                         fast,
+                        sidebars,
                     );
                 }
             }
@@ -233,6 +242,7 @@ fn full_diff(
                         &path,
                         diff,
                         fast,
+                        sidebars,
                     );
                 }
             }
@@ -259,6 +269,15 @@ fn full_diff(
                     rhs = fmt_html(&html_minifier::minify(rhs_t).unwrap());
                 }
                 if lhs != rhs {
+                    let mut diff_hash = Sha256::new();
+                    diff_hash.write_all(lhs.as_bytes()).unwrap();
+                    diff_hash.write_all(rhs.as_bytes()).unwrap();
+                    let diff_hash = BASE64_STANDARD_NO_PAD.encode(&diff_hash.finalize()[..]);
+                    if let Some(hash) = (*DIFF_MAP.read().unwrap()).get(&diff_hash) {
+                        diff.insert(key, format!("See {hash}"));
+                        return;
+                    }
+                    (*DIFF_MAP.write().unwrap()).insert(diff_hash, "somewhere else".into());
                     diff.insert(
                         key,
                         ansi_to_html::convert(&if fast {
@@ -309,7 +328,7 @@ fn main() -> Result<(), anyhow::Error> {
                         let left = v;
                         let right = b.get(k).unwrap_or(&Value::Null);
                         let mut diff = BTreeMap::new();
-                        full_diff(left, right, &[], &mut diff, arg.fast);
+                        full_diff(left, right, &[], &mut diff, arg.fast, arg.sidebars);
                         if !diff.is_empty() {
                             return Some(format!(
                                 r#"<li><span>{k}</span><div class="r"><pre><code>{}</code></pre></div></li>"#,

Diff for: crates/diff-test/src/main.rs

@@ -8,6 +8,7 @@ use rari_types::fm_types::{FeatureStatus, PageType};
 use rari_types::globals::{cache_content, deny_warnings};
 use rari_types::locale::Locale;
 
+use super::l10n::l10n_json_data;
 use super::titles::api_page_title;
 use crate::error::DocError;
 use crate::pages::page::{Page, PageLike, PageReader};
@@ -95,6 +96,14 @@ pub fn add_inline_badges(out: &mut String, page: &Page, locale: Locale) -> Resul
     Ok(())
 }
 
+pub fn write_parent_li(out: &mut String, page: &Page, locale: Locale) -> Result<(), DocError> {
+    let title = l10n_json_data("Template", "overview", locale)?;
+    write!(out, "<li><a href=\"{}\">{}</a>", page.url(), title)?;
+    add_inline_badges(out, page, locale)?;
+    write!(out, "</li>")?;
+    Ok(())
+}
+
 pub fn list_sub_pages_reverse_internal(
     out: &mut String,
     url: &str,
@@ -120,10 +129,14 @@ pub fn list_sub_pages_internal(
     depth: Option<usize>,
     sorter: Option<SubPagesSorter>,
     page_types: &[PageType],
+    include_parent: bool,
 ) -> Result<(), DocError> {
     let sub_pages = get_sub_pages(url, Some(1), sorter.unwrap_or_default())?;
-
     let depth = depth.map(|i| i.saturating_sub(1));
+    if include_parent {
+        let page = Page::from_url_with_other_locale_and_fallback(url, Some(locale))?;
+        write_parent_li(out, &page, locale)?;
+    }
     for sub_page in sub_pages {
         if !page_types.is_empty() && !page_types.contains(&sub_page.page_type()) {
             continue;
@@ -134,7 +147,16 @@ pub fn list_sub_pages_internal(
         } else {
             write_li_with_badges(out, &sub_page, locale, false)?;
             out.push_str("<ol>");
-            list_sub_pages_internal(out, sub_page.url(), locale, depth, sorter, page_types)?;
+
+            list_sub_pages_internal(
+                out,
+                sub_page.url(),
+                locale,
+                depth,
+                sorter,
+                page_types,
+                include_parent,
+            )?;
             out.push_str("</ol>");
             out.push_str("</li>");
         }
@@ -148,6 +170,7 @@ pub fn list_sub_pages_grouped_internal(
     locale: Locale,
     sorter: Option<SubPagesSorter>,
     page_types: &[PageType],
+    include_parent: bool,
 ) -> Result<(), DocError> {
     let sub_pages = get_sub_pages(url, None, sorter.unwrap_or_default())?;
 
@@ -171,6 +194,10 @@ pub fn list_sub_pages_grouped_internal(
             grouped.insert(sub_page.title(), vec![sub_page]);
         }
     }
+    if include_parent {
+        let page = Page::from_url_with_other_locale_and_fallback(url, Some(locale))?;
+        write_parent_li(out, &page, locale)?;
+    }
     for (prefix, group) in grouped {
         let keep_group = group.len() > 2;
         if keep_group {

Diff for: crates/rari-doc/src/helpers/subpages.rs

@@ -277,29 +277,21 @@ pub fn post_process_html<T: PageLike>(
             el.after("</div>", ContentType::Html);
             Ok(())
         }),
-        element!("div.notecard.callout > p:first-child", |el| {
-            el.prepend(
-                &concat_strs!(
-                    "<strong>",
-                    NoteCard::Callout.prefix_for_locale(page.locale()),
-                    "</strong>"
-                ),
-                ContentType::Html,
-            );
-            Ok(())
-        }),
-        element!("div.notecard.warning > p:first-child", |el| {
-            el.prepend(
-                &concat_strs!(
-                    "<strong>",
-                    NoteCard::Warning.prefix_for_locale(page.locale()),
-                    "</strong>"
-                ),
-                ContentType::Html,
-            );
-            Ok(())
-        }),
-        element!("div.notecard.note > p:first-child", |el| {
+        element!(
+            "div.notecard.warning[data-add-warning] > p:first-child",
+            |el| {
+                el.prepend(
+                    &concat_strs!(
+                        "<strong>",
+                        NoteCard::Warning.prefix_for_locale(page.locale()),
+                        "</strong>"
+                    ),
+                    ContentType::Html,
+                );
+                Ok(())
+            }
+        ),
+        element!("div.notecard.note[data-add-note] > p:first-child", |el| {
             el.prepend(
                 &concat_strs!(
                     "<strong>",
@@ -330,6 +322,14 @@ pub fn post_process_html<T: PageLike>(
             el.remove_attribute("data-sourcepos");
             Ok(())
         }),
+        element!("*[data-add-note]", |el| {
+            el.remove_attribute("data-add-note");
+            Ok(())
+        }),
+        element!("*[data-add-warning]", |el| {
+            el.remove_attribute("data-add-warning");
+            Ok(())
+        }),
     ];
     if sidebar {
         element_content_handlers.push(element!("html", |el| {

Diff for: crates/rari-doc/src/html/rewriter.rs

@@ -224,6 +224,8 @@ pub struct SubPageEntry {
     pub tags: Vec<PageType>,
     #[serde(default)]
     pub details: Details,
+    #[serde(default)]
+    pub include_parent: bool,
 }
 
 #[derive(Serialize, Deserialize, Default, Debug)]
@@ -248,8 +250,8 @@ pub enum SidebarEntry {
 #[derive(Debug, Default)]
 pub enum MetaChildren {
     Children(Vec<SidebarMetaEntry>),
-    ListSubPages(String, Vec<PageType>),
-    ListSubPagesGrouped(String, Vec<PageType>),
+    ListSubPages(String, Vec<PageType>, bool),
+    ListSubPagesGrouped(String, Vec<PageType>, bool),
     WebExtApi,
     #[default]
     None,
@@ -327,25 +329,27 @@ impl From<SidebarEntry> for SidebarMetaEntry {
                 link,
                 title,
                 path,
+                include_parent,
             }) => SidebarMetaEntry {
                 section: false,
                 details,
                 code: false,
                 content: SidebarMetaEntryContent::Link { link, title },
-                children: MetaChildren::ListSubPages(path, tags),
+                children: MetaChildren::ListSubPages(path, tags, include_parent),
             },
             SidebarEntry::ListSubPagesGrouped(SubPageEntry {
                 details,
                 tags,
                 link,
                 title,
                 path,
+                include_parent,
             }) => SidebarMetaEntry {
                 section: false,
                 details,
                 code: false,
                 content: SidebarMetaEntryContent::Link { link, title },
-                children: MetaChildren::ListSubPagesGrouped(path, tags),
+                children: MetaChildren::ListSubPagesGrouped(path, tags, include_parent),
             },
             SidebarEntry::Default(BasicEntry {
                 link,
@@ -457,11 +461,24 @@ impl SidebarMetaEntry {
                     child.render(out, locale, slug, l10n)?;
                 }
             }
-            MetaChildren::ListSubPages(url, page_types) => {
-                list_sub_pages_internal(out, url, locale, Some(1), None, page_types)?
-            }
-            MetaChildren::ListSubPagesGrouped(url, page_types) => {
-                list_sub_pages_grouped_internal(out, url, locale, None, page_types)?
+            MetaChildren::ListSubPages(url, page_types, include_parent) => list_sub_pages_internal(
+                out,
+                url,
+                locale,
+                Some(1),
+                None,
+                page_types,
+                *include_parent,
+            )?,
+            MetaChildren::ListSubPagesGrouped(url, page_types, include_parent) => {
+                list_sub_pages_grouped_internal(
+                    out,
+                    url,
+                    locale,
+                    None,
+                    page_types,
+                    *include_parent,
+                )?
             }
             MetaChildren::WebExtApi => {
                 let children = &helpers::webextapi::children(

Diff for: crates/rari-doc/src/html/sidebar.rs

@@ -166,7 +166,7 @@ fn build_interface_list(entries: &mut Vec<SidebarMetaEntry>, interfaces: &[&str]
                     .map(|interface| SidebarMetaEntry {
                         code: true,
                         content: SidebarMetaEntryContent::Link {
-                            title: None,
+                            title: Some(interface.to_string()),
                             link: Some(format!(
                                 "/Web/API/{}",
                                 interface.replace("()", "").replace('.', "/")

Diff for: crates/rari-doc/src/sidebars/apiref.rs

@@ -37,7 +37,7 @@ pub fn available_in_workers(typ: Option<String>) -> Result<String, DocError> {
     .unwrap_or(l10n_json_data("Template", default_typ, env.locale)?);
 
     Ok(concat_strs!(
-        r#"<div class="notecard note"><p> "#,
+        r#"<div class="notecard note" data-add-note><p> "#,
         copy,
         "</p></div>"
     ))

Diff for: crates/rari-doc/src/templ/templs/banners.rs

@@ -42,6 +42,7 @@ pub fn list_sub_pages(
             Some(depth),
             Some(SubPagesSorter::SlugNatural),
             &[],
+            false,
         )?;
     }
     out.push_str(if ordered { "</ol>" } else { "</ul>" });
@@ -74,6 +75,7 @@ pub fn list_sub_pages_grouped(
             })
             .as_deref()
             .unwrap_or_default(),
+        false,
     )?;
     out.push_str("</ol></details>");
     Ok(out)