fix(various): fix issues from testing

Author: fiji-flo

Diff for: crates/rari-doc/src/pages/build.rs

@@ -259,6 +259,9 @@ pub fn build_doc(doc: &Doc) -> Result<BuiltDocy, DocError> {
         })
         .collect();
 
+    let no_indexing =
+        doc.meta.slug == "MDN/Kitchensink" || doc.is_orphaned() || doc.is_conflicting();
+
     Ok(BuiltDocy::Doc(Box::new(JsonDoADoc {
         doc: JsonDoc {
             title: doc.title().to_string(),
@@ -278,6 +281,7 @@ pub fn build_doc(doc: &Doc) -> Result<BuiltDocy, DocError> {
             modified,
             summary,
             popularity,
+            no_indexing,
             sidebar_macro: doc.meta.sidebar.first().cloned(),
             source: Source {
                 folder,
@@ -287,7 +291,6 @@ pub fn build_doc(doc: &Doc) -> Result<BuiltDocy, DocError> {
             },
             browser_compat: doc.meta.browser_compat.clone(),
             other_translations,
-            ..Default::default()
         },
         url: doc.meta.url.clone(),
         ..Default::default()
@@ -333,6 +336,7 @@ pub fn build_generic_page(page: &GenericPage) -> Result<BuiltDocy, DocError> {
         },
         page_title: strcat!(page.meta.title.as_str() " | " page.meta.title_suffix.as_str()),
         url: page.meta.url.clone(),
+        id: page.meta.page.clone(),
     })))
 }
 

Diff for: crates/rari-doc/src/pages/json.rs

@@ -348,6 +348,7 @@ where
 #[serde(rename_all = "camelCase")]
 pub struct JsonHomePageSPA {
     pub slug: &'static str,
+    pub url: String,
     pub page_title: &'static str,
     pub page_description: Option<&'static str>,
     pub featured_articles: Vec<HomePageFeaturedArticle>,
@@ -370,4 +371,5 @@ pub struct JsonGenericPage {
     pub hy_data: JsonGenericHyData,
     pub page_title: String,
     pub url: String,
+    pub id: String,
 }

Diff for: crates/rari-doc/src/pages/types/doc.rs

@@ -105,6 +105,14 @@ impl Doc {
         meta.original_slug = super_doc.meta.original_slug.clone();
         meta.sidebar = super_doc.meta.sidebar.clone();
     }
+
+    pub fn is_orphaned(&self) -> bool {
+        self.meta.slug.starts_with("orphaned/")
+    }
+
+    pub fn is_conflicting(&self) -> bool {
+        self.meta.slug.starts_with("orphaned/")
+    }
 }
 
 impl PageReader for Doc {
@@ -122,7 +130,7 @@ impl PageReader for Doc {
         debug!("reading doc: {}", &path.display());
         let mut doc = read_doc(&path)?;
 
-        if doc.meta.locale != Default::default() {
+        if doc.meta.locale != Default::default() && !doc.is_conflicting() && !doc.is_orphaned() {
             let super_doc = Doc::page_from_slug(&doc.meta.slug, Default::default())?;
             if let Page::Doc(super_doc) = super_doc {
                 doc.copy_meta_from_super(&super_doc);

Diff for: crates/rari-doc/src/pages/types/generic.rs

@@ -28,6 +28,7 @@ pub struct GenericPageMeta {
     pub full_path: PathBuf,
     pub path: PathBuf,
     pub title_suffix: String,
+    pub page: String,
 }
 
 impl GenericPageMeta {
@@ -38,8 +39,9 @@ impl GenericPageMeta {
         locale: Locale,
         slug: String,
         title_suffix: &str,
-        url: String,
+        page: String,
     ) -> Result<Self, DocError> {
+        let url = strcat!("/" locale.as_url_str() "/" slug.as_str() "/" page.as_str());
         Ok(GenericPageMeta {
             title: fm.title,
             locale,
@@ -48,6 +50,7 @@ impl GenericPageMeta {
             path,
             full_path,
             title_suffix: title_suffix.to_string(),
+            page,
         })
     }
 }
@@ -195,7 +198,6 @@ fn read_generic_page(
     let path = full_path.strip_prefix(root)?.to_path_buf();
     let page = path.with_extension("");
     let page = page.to_string_lossy();
-    let url = strcat!("/" locale.as_url_str() "/" slug "/" page.as_ref());
     let slug = strcat!(slug "/" page.as_ref());
 
     Ok(GenericPage {
@@ -206,7 +208,7 @@ fn read_generic_page(
             locale,
             slug.to_string(),
             title_suffix,
-            url,
+            page.to_string(),
         )?,
         raw,
         content_start,

Diff for: crates/rari-doc/src/pages/types/spa.rs

@@ -130,6 +130,7 @@ impl SPA {
             }))),
             SPAData::HomePage => Ok(BuiltDocy::HomePageSPA(Box::new(JsonHomePageSPA {
                 slug: self.slug,
+                url: strcat!("/" self.locale().as_url_str() "/" self.slug),
                 page_title: self.page_title,
                 page_description: self.page_description,
                 featured_articles: featured_articles(

Diff for: crates/rari-doc/src/templ/render.rs

@@ -8,6 +8,9 @@ use super::parser::{parse, Token};
 use super::templs::invoke;
 use crate::error::DocError;
 
+static DELIM_START: &str = ";!::::";
+static DELIM_END: &str = ";!::::";
+
 pub struct Rendered {
     pub content: String,
     pub templs: Vec<String>,
@@ -59,7 +62,7 @@ pub fn render(env: &RariEnv, input: &str) -> Result<Rendered, DocError> {
     render_tokens(env, tokens, input)
 }
 fn encode_ref(index: usize, out: &mut String) -> Result<(), DocError> {
-    Ok(write!(out, "!::::{index}::::!",)?)
+    Ok(write!(out, "{DELIM_START}{index}{DELIM_END}",)?)
 }
 
 pub fn render_and_decode_ref(env: &RariEnv, input: &str) -> Result<String, DocError> {
@@ -71,13 +74,13 @@ pub fn render_and_decode_ref(env: &RariEnv, input: &str) -> Result<String, DocEr
 
 pub(crate) fn decode_ref(input: &str, templs: &[String]) -> Result<String, DocError> {
     let mut decoded = String::with_capacity(input.len());
-    if !input.contains("!::::") {
+    if !input.contains(";!::::") {
         return Ok(input.to_string());
     }
     let mut frags = vec![];
-    for frag in input.split("!::::") {
-        let has_ks = frag.contains("::::!");
-        for (i, sub_frag) in frag.splitn(2, "::::!").enumerate() {
+    for frag in input.split(DELIM_START) {
+        let has_ks = frag.contains(DELIM_END);
+        for (i, sub_frag) in frag.splitn(2, DELIM_END).enumerate() {
             if i == 0 && has_ks {
                 frags.push(sub_frag);
                 //decode_macro(sub_frag, &mut decoded)?;

Diff for: src/main.rs

@@ -223,11 +223,15 @@ fn main() -> Result<(), anyhow::Error> {
                 let file = File::create(out_file).unwrap();
                 let mut buffed = BufWriter::new(file);
                 urls.sort();
-                for url in urls {
+                for url in &urls {
                     buffed.write_all(url.as_bytes())?;
                     buffed.write_all(b"\n")?;
                 }
-                println!("Took: {: >10.3?} to write sitemap.txt", start.elapsed());
+                println!(
+                    "Took: {: >10.3?} to write sitemap.txt ({})",
+                    start.elapsed(),
+                    urls.len()
+                );
             }
             if let Some((recorder_handler, tx)) = templ_stats {
                 tx.send("∞".to_string())?;