feat(issues): support json_issues flag

This adds a `flaws`field to the index.json to better support
existing flaw tooling.

Cargo.toml

@@ -92,7 +92,7 @@ indoc = "2"
 base64 = "0.22"
 console = "0.15"
 sha2 = "0.10"
-dashmap = "6"
+dashmap = { version = "6", features = ["serde"] }
 serde_yaml_ng = "0.10"
 schemars = { version = "0.8", features = ["chrono"] }
 pretty_yaml = "0.5"

crates/rari-cli/main.rs

@@ -17,7 +17,7 @@ use rari_doc::build::{
     build_generic_pages, build_spas, build_top_level_meta,
 };
 use rari_doc::cached_readers::{read_and_cache_doc_pages, CACHED_DOC_PAGE_FILES};
-use rari_doc::issues::{issues_by, InMemoryLayer};
+use rari_doc::issues::IN_MEMORY;
 use rari_doc::pages::json::BuiltPage;
 use rari_doc::pages::types::doc::Doc;
 use rari_doc::reader::read_docs_parallel;
@@ -183,6 +183,8 @@ struct BuildArgs {
     issues: Option<PathBuf>,
     #[arg(long, help = "Annotate html with 'data-flaw' attributes")]
     data_issues: bool,
+    #[arg(long, help = "Add flaws field to index.json for docs")]
+    json_issues: bool,
 }
 
 #[derive(Debug)]
@@ -219,7 +221,7 @@ fn main() -> Result<(), Error> {
         .with_target("rari_doc", Level::WARN)
         .with_target("rari", Level::WARN);
 
-    let memory_layer = InMemoryLayer::default();
+    let memory_layer = IN_MEMORY.clone();
     tracing_subscriber::registry()
         .with(
             tracing_subscriber::fmt::layer()
@@ -235,6 +237,7 @@ fn main() -> Result<(), Error> {
             settings.deny_warnings = args.deny_warnings;
             settings.cache_content = !args.no_cache;
             settings.data_issues = args.data_issues;
+            settings.json_issues = args.json_issues;
             let _ = SETTINGS.set(settings);
 
             let templ_stats = if args.templ_stats {
@@ -383,37 +386,17 @@ fn main() -> Result<(), Error> {
 
             if let Some(issues_path) = args.issues {
                 let events = memory_layer.get_events();
-                let events = events.lock().unwrap();
-
-                let issues = issues_by(&events);
-
-                let mut tw = TabWriter::new(vec![]);
-                writeln!(&mut tw, "--- templ issues ---\t").expect("unable to write");
-                for (templ, templ_issues) in &issues.templ {
-                    writeln!(&mut tw, "{}\t{:5}", templ, templ_issues.len())
-                        .expect("unable to write");
-                }
-                writeln!(&mut tw, "--- other issues ---\t").expect("unable to write");
-                for (source, other_issues) in &issues.other {
-                    writeln!(&mut tw, "{}\t{:5}", source, other_issues.len())
-                        .expect("unable to write");
-                }
-                writeln!(&mut tw, "--- other issues w/o pos ---\t").expect("unable to write");
-                for (source, no_pos) in &issues.no_pos {
-                    writeln!(&mut tw, "{}\t{:5}", source, no_pos.len()).expect("unable to write");
-                }
-                print!("{}", String::from_utf8_lossy(&tw.into_inner().unwrap()));
                 let file = File::create(issues_path).unwrap();
                 let mut buffed = BufWriter::new(file);
-                serde_json::to_writer_pretty(&mut buffed, &issues).unwrap();
+                serde_json::to_writer_pretty(&mut buffed, &*events).unwrap();
             }
         }
         Commands::Serve(args) => {
             let mut settings = Settings::new()?;
             settings.cache_content = args.cache;
             settings.data_issues = true;
             let _ = SETTINGS.set(settings);
-            serve::serve(memory_layer.clone())?
+            serve::serve()?
         }
         Commands::GitHistory => {
             println!("Gathering history 📜");

crates/rari-cli/serve.rs

@@ -1,18 +1,17 @@
 use std::cmp::Ordering;
 use std::str::FromStr;
 use std::sync::atomic::AtomicU64;
-use std::sync::Arc;
 
 use axum::body::Body;
-use axum::extract::{Path, Request, State};
+use axum::extract::{Path, Request};
 use axum::http::{header, StatusCode};
 use axum::response::{IntoResponse, Response};
 use axum::routing::get;
 use axum::{Json, Router};
 use rari_doc::cached_readers::wiki_histories;
 use rari_doc::contributors::contributors_txt;
 use rari_doc::error::DocError;
-use rari_doc::issues::{to_display_issues, InMemoryLayer};
+use rari_doc::issues::{to_display_issues, IN_MEMORY};
 use rari_doc::pages::json::BuiltPage;
 use rari_doc::pages::page::{Page, PageBuilder, PageLike};
 use rari_doc::pages::types::doc::Doc;
@@ -32,18 +31,15 @@ struct SearchItem {
     url: String,
 }
 
-async fn handler(state: State<Arc<InMemoryLayer>>, req: Request) -> Response<Body> {
+async fn handler(req: Request) -> Response<Body> {
     if req.uri().path().ends_with("/contributors.txt") {
         get_contributors_handler(req).await.into_response()
     } else {
-        get_json_handler(state, req).await.into_response()
+        get_json_handler(req).await.into_response()
     }
 }
 
-async fn get_json_handler(
-    State(memory_layer): State<Arc<InMemoryLayer>>,
-    req: Request,
-) -> Result<Json<BuiltPage>, AppError> {
+async fn get_json_handler(req: Request) -> Result<Json<BuiltPage>, AppError> {
     let url = req.uri().path();
     let req_id = REQ_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
     let span = span!(Level::WARN, "serve", req = req_id);
@@ -52,22 +48,22 @@ async fn get_json_handler(
     let _enter1 = span.enter();
     let url = url.strip_suffix("/index.json").unwrap_or(url);
     let page = Page::from_url_with_fallback(url)?;
-    let slug = &page.slug();
-    let locale = page.locale();
-    let span = span!(Level::ERROR, "page", "{}:{}", locale, slug);
+    let file = page.full_path().to_string_lossy();
+    let span = span!(
+        Level::ERROR,
+        "page",
+        locale = page.locale().as_url_str(),
+        slug = page.slug(),
+        file = file.as_ref()
+    );
     let _enter2 = span.enter();
     let mut json = page.build()?;
     tracing::info!("{url}");
     if let BuiltPage::Doc(json_doc) = &mut json {
-        let m = memory_layer.get_events();
-        let mut issues = m.lock().unwrap();
-        let req_issues: Vec<_> = issues
-            .iter()
-            .filter(|issue| issue.req == req_id)
-            .cloned()
-            .collect();
-        issues.retain_mut(|i| i.req != req_id);
-        json_doc.doc.flaws = Some(to_display_issues(req_issues, &page));
+        let m = IN_MEMORY.get_events();
+        if let Some((_, req_issues)) = m.remove(page.full_path().to_string_lossy().as_ref()) {
+            json_doc.doc.flaws = Some(to_display_issues(req_issues, &page));
+        }
     }
     Ok(Json(json))
 }
@@ -180,16 +176,15 @@ where
     }
 }
 
-pub fn serve(memory_layer: InMemoryLayer) -> Result<(), anyhow::Error> {
+pub fn serve() -> Result<(), anyhow::Error> {
     tokio::runtime::Builder::new_current_thread()
         .enable_all()
         .build()
         .unwrap()
         .block_on(async {
             let app = Router::new()
                 .route("/:locale/search-index.json", get(get_search_index_handler))
-                .fallback(handler)
-                .with_state(Arc::new(memory_layer));
+                .fallback(handler);
 
             let listener = tokio::net::TcpListener::bind("0.0.0.0:8083").await.unwrap();
             axum::serve(listener, app).await.unwrap();

crates/rari-doc/src/build.rs

@@ -14,7 +14,7 @@ use chrono::NaiveDateTime;
 use itertools::Itertools;
 use rari_types::globals::{
     base_url, blog_root, build_out_root, contributor_spotlight_root, curriculum_root,
-    generic_content_root, git_history,
+    generic_content_root, git_history, settings,
 };
 use rari_types::locale::{default_locale, Locale};
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
@@ -27,6 +27,7 @@ use crate::cached_readers::{
 };
 use crate::contributors::contributors_txt;
 use crate::error::DocError;
+use crate::issues::{to_display_issues, IN_MEMORY};
 use crate::pages::build::copy_additional_files;
 use crate::pages::json::{BuiltPage, JsonDocMetadata};
 use crate::pages::page::{Page, PageBuilder, PageLike};
@@ -67,7 +68,17 @@ pub fn build_single_page(page: &Page) -> Result<(BuiltPage, String), DocError> {
         file = file.as_ref()
     );
     let _enter = span.enter();
-    let built_page = page.build()?;
+    let mut built_page = page.build()?;
+    if settings().json_issues {
+        if let BuiltPage::Doc(json_doc) = &mut built_page {
+            if let Some(issues) = IN_MEMORY
+                .get_events()
+                .get(page.full_path().to_string_lossy().as_ref())
+            {
+                json_doc.doc.flaws = Some(to_display_issues(issues.value().clone(), page));
+            }
+        }
+    }
     let out_path = build_out_root()
         .expect("No BUILD_OUT_ROOT")
         .join(url_to_folder_path(page.url().trim_start_matches('/')));

crates/rari-doc/src/issues.rs

@@ -1,8 +1,9 @@
 use std::collections::{BTreeMap, HashMap};
 use std::fmt;
 use std::sync::atomic::AtomicI64;
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, LazyLock};
 
+use dashmap::DashMap;
 use itertools::Itertools;
 use schemars::JsonSchema;
 use serde::Serialize;
@@ -20,12 +21,13 @@ pub(crate) fn get_issue_counter() -> i64 {
     ISSUE_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
 }
 
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, Serialize)]
 pub struct Issue {
     pub req: u64,
     pub ic: i64,
     pub col: i64,
     pub line: i64,
+    pub file: String,
     pub fields: Vec<(&'static str, String)>,
     pub spans: Vec<(&'static str, String)>,
 }
@@ -36,6 +38,7 @@ pub struct IssueEntries {
     ic: i64,
     col: i64,
     line: i64,
+    file: String,
     entries: Vec<(&'static str, String)>,
 }
 
@@ -77,9 +80,6 @@ impl<'a> From<&'a Issue> for TemplIssue<'a> {
         let mut tissue = DEFAULT_TEMPL_ISSUE.clone();
         for (key, value) in value.spans.iter().chain(value.fields.iter()) {
             match *key {
-                "file" => {
-                    tissue.file = value.as_str();
-                }
                 "slug" => {
                     tissue.slug = value.as_str();
                 }
@@ -95,42 +95,18 @@ impl<'a> From<&'a Issue> for TemplIssue<'a> {
         }
         tissue.col = value.col;
         tissue.line = value.line;
+        tissue.file = value.file.as_str();
         tissue
     }
 }
 
-pub fn issues_by(issues: &[Issue]) -> Issues {
-    let mut templ: BTreeMap<&str, Vec<TemplIssue>> = BTreeMap::new();
-    let mut other: BTreeMap<&str, Vec<TemplIssue>> = BTreeMap::new();
-    let mut no_pos: BTreeMap<&str, Vec<TemplIssue>> = BTreeMap::new();
-    for issue in issues.iter().map(TemplIssue::from) {
-        if let Some(templ_name) =
-            issue
-                .tail
-                .iter()
-                .find_map(|(key, value)| if *key == "templ" { Some(value) } else { None })
-        {
-            templ.entry(templ_name).or_default().push(issue);
-        } else if issue.line != -1 {
-            other.entry(issue.source).or_default().push(issue)
-        } else {
-            no_pos.entry(issue.source).or_default().push(issue);
-        }
-    }
-    Issues {
-        templ,
-        other,
-        no_pos,
-    }
-}
-
-#[derive(Clone, Default)]
+#[derive(Clone, Default, Debug)]
 pub struct InMemoryLayer {
-    events: Arc<Mutex<Vec<Issue>>>,
+    events: Arc<DashMap<String, Vec<Issue>>>,
 }
 
 impl InMemoryLayer {
-    pub fn get_events(&self) -> Arc<Mutex<Vec<Issue>>> {
+    pub fn get_events(&self) -> Arc<DashMap<String, Vec<Issue>>> {
         Arc::clone(&self.events)
     }
 }
@@ -140,7 +116,11 @@ impl Visit for IssueEntries {
         self.entries.push((field.name(), format!("{value:?}")));
     }
     fn record_str(&mut self, field: &Field, value: &str) {
-        self.entries.push((field.name(), value.to_string()));
+        if field.name() == "file" {
+            self.file = value.to_string();
+        } else {
+            self.entries.push((field.name(), value.to_string()));
+        }
     }
     fn record_u64(&mut self, field: &Field, value: u64) {
         if field.name() == "req" {
@@ -162,7 +142,11 @@ impl Visit for Issue {
         self.fields.push((field.name(), format!("{value:?}")));
     }
     fn record_str(&mut self, field: &Field, value: &str) {
-        self.fields.push((field.name(), value.to_string()));
+        if field.name() == "file" {
+            self.file = value.to_string();
+        } else {
+            self.fields.push((field.name(), value.to_string()));
+        }
     }
     fn record_u64(&mut self, field: &Field, value: u64) {
         if field.name() == "req" {
@@ -204,6 +188,7 @@ where
             ic: 0,
             col: 0,
             line: 0,
+            file: String::default(),
             fields: vec![],
             spans: vec![],
         };
@@ -221,6 +206,9 @@ where
                 if entries.line != 0 {
                     issue.line = entries.line;
                 }
+                if !entries.file.is_empty() {
+                    issue.file = entries.file.clone();
+                }
                 if entries.ic != 0 {
                     issue.ic = entries.ic;
                 } else {
@@ -231,8 +219,10 @@ where
         }
 
         event.record(&mut issue);
-        let mut events = self.events.lock().unwrap();
-        events.push(issue);
+        self.events
+            .entry(issue.file.clone())
+            .or_default()
+            .push(issue);
     }
 }
 
@@ -361,3 +351,5 @@ pub fn to_display_issues(issues: Vec<Issue>, page: &Page) -> DisplayIssues {
     }
     map
 }
+
+pub static IN_MEMORY: LazyLock<InMemoryLayer> = LazyLock::new(InMemoryLayer::default);

crates/rari-types/src/settings.rs

@@ -24,6 +24,7 @@ pub struct Settings {
     pub additional_locales_for_generics_and_spas: Vec<Locale>,
     pub reader_ignores_gitignore: bool,
     pub data_issues: bool,
+    pub json_issues: bool,
 }
 
 impl Settings {