diff --git a/Cargo.lock b/Cargo.lock index cb12b9bc..5303f664 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2644,6 +2644,7 @@ dependencies = [ "svg_metadata", "thiserror", "tracing", + "tracing-subscriber", "unescaper", "url", "validator", diff --git a/Cargo.toml b/Cargo.toml index 3542c306..54768bda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ rari-data = { path = "crates/rari-data" } rari-templ-func = { path = "crates/rari-templ-func" } tracing = "0.1" +tracing-subscriber = "0.3" thiserror = "1" serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } @@ -100,6 +101,7 @@ rari-types.workspace = true serde.workspace = true serde_json.workspace = true tracing.workspace = true +tracing-subscriber.workspace = true anyhow.workspace = true self_update = { version = "0.41", default-features = false, features = [ @@ -109,7 +111,6 @@ self_update = { version = "0.41", default-features = false, features = [ ] } clap = { version = "4.5.1", features = ["derive"] } clap-verbosity-flag = "2" -tracing-subscriber = "0.3" tracing-log = "0.2" tabwriter = "1" axum = "0.7" diff --git a/crates/rari-cli/main.rs b/crates/rari-cli/main.rs index 096a594d..3527e2b2 100644 --- a/crates/rari-cli/main.rs +++ b/crates/rari-cli/main.rs @@ -8,12 +8,12 @@ use std::thread::spawn; use anyhow::{anyhow, Error}; use clap::{Args, Parser, Subcommand}; -use issues::{issues_by, InMemoryLayer}; use rari_doc::build::{ build_blog_pages, build_contributor_spotlight_pages, build_curriculum_pages, build_docs, build_generic_pages, build_spas, }; use rari_doc::cached_readers::{read_and_cache_doc_pages, CACHED_DOC_PAGE_FILES}; +use rari_doc::issues::{issues_by, InMemoryLayer}; use rari_doc::pages::types::doc::Doc; use rari_doc::reader::read_docs_parallel; use rari_doc::search_index::build_search_index; @@ -35,7 +35,6 @@ use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{filter, Layer}; -mod issues; mod serve; #[derive(Parser)] @@ -167,9 +166,11 @@ fn main() -> Result<(), Error> { .with_target("rari_doc", cli.verbose.log_level_filter().as_trace()) .with_target("rari", cli.verbose.log_level_filter().as_trace()); - let memory_filter = filter::Targets::new().with_target("rari_doc", Level::WARN); + let memory_filter = filter::Targets::new() + .with_target("rari_doc", Level::WARN) + .with_target("rari", Level::WARN); - let memory_layer = InMemoryLayer::new(); + let memory_layer = InMemoryLayer::default(); tracing_subscriber::registry() .with( tracing_subscriber::fmt::layer() @@ -341,7 +342,7 @@ fn main() -> Result<(), Error> { settings.deny_warnings = args.deny_warnings; settings.cache_content = args.cache_content; let _ = SETTINGS.set(settings); - serve::serve()? + serve::serve(memory_layer.clone())? } Commands::GitHistory => { println!("Gathering history 📜"); diff --git a/crates/rari-cli/serve.rs b/crates/rari-cli/serve.rs index ac892b10..4ec7438d 100644 --- a/crates/rari-cli/serve.rs +++ b/crates/rari-cli/serve.rs @@ -1,15 +1,38 @@ -use axum::extract::Request; +use std::sync::atomic::AtomicU64; +use std::sync::Arc; + +use axum::extract::{Request, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use axum::{Json, Router}; use rari_doc::error::DocError; +use rari_doc::issues::{to_display_issues, InMemoryLayer}; use rari_doc::pages::json::BuiltDocy; use rari_doc::pages::page::{Page, PageBuilder, PageLike}; use tracing::{error, span, Level}; -async fn get_json_handler(req: Request) -> Result, AppError> { +static REQ_COUNTER: AtomicU64 = AtomicU64::new(1); + +async fn get_json_handler( + State(memory_layer): State>, + req: Request, +) -> Result, AppError> { + let req_id = REQ_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let span = span!(Level::WARN, "serve", req = req_id); + let _enter1 = span.enter(); let url = req.uri().path(); - let json = get_json(url)?; + let mut json = get_json(url)?; + if let BuiltDocy::Doc(json_doc) = &mut json { + let m = memory_layer.get_events(); + let mut issues = m.lock().unwrap(); + let req_isses: Vec<_> = issues + .iter() + .filter(|issue| issue.req == req_id) + .cloned() + .collect(); + issues.retain_mut(|i| i.req != req_id); + json_doc.doc.flaws = Some(to_display_issues(req_isses)); + } Ok(Json(json)) } @@ -44,13 +67,15 @@ where } } -pub fn serve() -> Result<(), anyhow::Error> { +pub fn serve(memory_layer: InMemoryLayer) -> Result<(), anyhow::Error> { tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap() .block_on(async { - let app = Router::new().fallback(get_json_handler); + let app = Router::new() + .fallback(get_json_handler) + .with_state(Arc::new(memory_layer)); let listener = tokio::net::TcpListener::bind("0.0.0.0:8083").await.unwrap(); axum::serve(listener, app).await.unwrap(); diff --git a/crates/rari-doc/Cargo.toml b/crates/rari-doc/Cargo.toml index 62dfcd6c..28cc54d2 100644 --- a/crates/rari-doc/Cargo.toml +++ b/crates/rari-doc/Cargo.toml @@ -7,6 +7,12 @@ license.workspace = true rust-version.workspace = true [dependencies] +rari-utils.workspace = true +rari-types.workspace = true +rari-md.workspace = true +rari-data.workspace = true +rari-templ-func.workspace = true + thiserror.workspace = true serde.workspace = true serde_json.workspace = true @@ -17,6 +23,7 @@ itertools.workspace = true constcat.workspace = true indexmap.workspace = true sha2.workspace = true +tracing-subscriber.workspace = true serde_yaml_ng = "0.10" yaml-rust = "0.4" @@ -48,11 +55,6 @@ phf_macros = "0.11" phf = "0.11" unescaper = "0.1" -rari-utils.workspace = true -rari-types.workspace = true -rari-md.workspace = true -rari-data.workspace = true -rari-templ-func.workspace = true css-syntax = { path = "../css-syntax", features = ["rari"] } diff --git a/crates/rari-doc/src/html/rewriter.rs b/crates/rari-doc/src/html/rewriter.rs index b6e75ddd..52b0e034 100644 --- a/crates/rari-doc/src/html/rewriter.rs +++ b/crates/rari-doc/src/html/rewriter.rs @@ -178,6 +178,13 @@ pub fn post_process_html( if let Some(pos) = el.get_attribute("data-sourcepos") { if let Some((start, _)) = pos.split_once('-') { if let Some((line, col)) = start.split_once(':') { + let line_n = + line.parse::().map(|l| l + page.fm_offset()).ok(); + let line = line_n + .map(|i| i.to_string()) + .map(Cow::Owned) + .unwrap_or(Cow::Borrowed(line)); + let line = line.as_ref(); tracing::warn!( source = "redirected-link", line = line, diff --git a/crates/rari-cli/issues.rs b/crates/rari-doc/src/issues.rs similarity index 63% rename from crates/rari-cli/issues.rs rename to crates/rari-doc/src/issues.rs index 1bd51aec..185a625f 100644 --- a/crates/rari-cli/issues.rs +++ b/crates/rari-doc/src/issues.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::sync::{Arc, Mutex}; @@ -9,14 +9,16 @@ use tracing::{Event, Subscriber}; use tracing_subscriber::registry::LookupSpan; use tracing_subscriber::Layer; -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct Issue { + pub req: u64, pub fields: Vec<(&'static str, String)>, pub spans: Vec<(&'static str, String)>, } #[derive(Debug, Default)] pub struct IssueEntries { + req: u64, entries: Vec<(&'static str, String)>, } @@ -29,6 +31,7 @@ pub struct Issues<'a> { #[derive(Clone, Debug, Serialize)] pub struct TemplIssue<'a> { + pub req: u64, pub source: &'a str, pub file: &'a str, pub slug: &'a str, @@ -40,6 +43,7 @@ pub struct TemplIssue<'a> { static UNKNOWN: &str = "unknown"; static DEFAULT_TEMPL_ISSUE: TemplIssue<'static> = TemplIssue { + req: 0, source: UNKNOWN, file: UNKNOWN, slug: UNKNOWN, @@ -101,18 +105,12 @@ pub fn issues_by(issues: &[Issue]) -> Issues { } } -#[derive(Clone)] +#[derive(Clone, Default)] pub struct InMemoryLayer { events: Arc>>, } impl InMemoryLayer { - pub fn new() -> Self { - InMemoryLayer { - events: Arc::new(Mutex::new(Vec::new())), - } - } - pub fn get_events(&self) -> Arc>> { Arc::clone(&self.events) } @@ -125,6 +123,11 @@ impl Visit for IssueEntries { fn record_str(&mut self, field: &Field, value: &str) { self.entries.push((field.name(), value.to_string())); } + fn record_u64(&mut self, field: &Field, value: u64) { + if field.name() == "req" { + self.req = value; + } + } } impl Visit for Issue { fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { @@ -133,6 +136,11 @@ impl Visit for Issue { fn record_str(&mut self, field: &Field, value: &str) { self.fields.push((field.name(), value.to_string())); } + fn record_u64(&mut self, field: &Field, value: u64) { + if field.name() == "req" { + self.req = value; + } + } } impl Layer for InMemoryLayer where @@ -155,6 +163,7 @@ where } fn on_event(&self, event: &Event, ctx: tracing_subscriber::layer::Context) { let mut issue = Issue { + req: 0, fields: vec![], spans: vec![], }; @@ -163,6 +172,9 @@ where for span in scope { let ext = span.extensions(); if let Some(entries) = ext.get::() { + if entries.req != 0 { + issue.req = entries.req + } issue.spans.extend(entries.entries.iter().rev().cloned()); } } @@ -172,3 +184,81 @@ where events.push(issue); } } + +#[derive(Serialize, Debug, Default, Clone)] +#[serde(untagged)] +pub enum Additional { + BrokenLink { + href: String, + }, + #[default] + None, +} + +#[derive(Serialize, Debug, Default, Clone)] +pub struct DisplayIssue { + pub id: usize, + pub explanation: Option, + pub suggestion: Option, + pub fixable: Option, + pub fixed: bool, + pub name: String, + pub line: Option, + pub col: Option, + #[serde(flatten)] + pub additional: Additional, +} + +pub type DisplayIssues = BTreeMap<&'static str, Vec>; + +impl From for DisplayIssue { + fn from(value: Issue) -> Self { + let mut di = DisplayIssue::default(); + let mut additional = HashMap::new(); + for (key, value) in value.spans.into_iter().chain(value.fields.into_iter()) { + match key { + "line" => di.line = value.parse().ok(), + "col" => di.col = value.parse().ok(), + "source" => { + di.name = value; + } + "message" => di.explanation = Some(value), + "redirect" => di.suggestion = Some(value), + + _ => { + additional.insert(key, value); + } + } + } + let additional = match di.name.as_str() { + "redirected-link" => { + di.fixed = false; + di.fixable = Some(true); + Additional::BrokenLink { + href: additional.remove("url").unwrap_or_default(), + } + } + _ => Additional::None, + }; + di.additional = additional; + di + } +} + +pub fn to_display_issues(issues: Vec) -> DisplayIssues { + let mut map = BTreeMap::new(); + for issue in issues { + let di = DisplayIssue::from(issue); + match &di.additional { + Additional::BrokenLink { .. } => { + let entry: &mut Vec<_> = map.entry("broken_links").or_default(); + entry.push(di); + } + Additional::None => { + let entry: &mut Vec<_> = map.entry("unknown").or_default(); + entry.push(di); + } + } + } + map +} diff --git a/crates/rari-doc/src/lib.rs b/crates/rari-doc/src/lib.rs index 6807dc8b..14bf07c3 100644 --- a/crates/rari-doc/src/lib.rs +++ b/crates/rari-doc/src/lib.rs @@ -4,6 +4,7 @@ pub mod cached_readers; pub mod error; pub mod helpers; pub mod html; +pub mod issues; pub mod pages; pub mod percent; pub mod reader; diff --git a/crates/rari-doc/src/pages/build.rs b/crates/rari-doc/src/pages/build.rs index 816d8df7..9c4f9b26 100644 --- a/crates/rari-doc/src/pages/build.rs +++ b/crates/rari-doc/src/pages/build.rs @@ -296,6 +296,7 @@ fn build_doc(doc: &Doc) -> Result { browser_compat: doc.meta.browser_compat.clone(), other_translations, page_type: doc.meta.page_type, + flaws: None, }, url: doc.meta.url.clone(), ..Default::default() diff --git a/crates/rari-doc/src/pages/json.rs b/crates/rari-doc/src/pages/json.rs index 7c40dc79..71442229 100644 --- a/crates/rari-doc/src/pages/json.rs +++ b/crates/rari-doc/src/pages/json.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use super::types::contributors::Usernames; use super::types::curriculum::{CurriculumIndexEntry, CurriculumSidebarEntry, Template, Topic}; +use crate::issues::DisplayIssues; use crate::pages::types::blog::BlogMeta; use crate::specs::Specification; use crate::utils::modified_dt; @@ -117,6 +118,8 @@ pub struct JsonDoc { pub browser_compat: Vec, #[serde(rename = "pageType")] pub page_type: PageType, + #[serde(skip_serializing_if = "Option::is_none")] + pub flaws: Option, } impl JsonDoc {