feat(popularities): move popularities to deps

crates/rari-cli/main.rs

@@ -23,7 +23,6 @@ use rari_doc::utils::TEMPL_RECORDER_SENDER;
 use rari_sitemap::Sitemaps;
 use rari_tools::add_redirect::add_redirect;
 use rari_tools::history::gather_history;
-use rari_tools::popularities::update_popularities;
 use rari_tools::r#move::r#move;
 use rari_tools::redirects::fix_redirects;
 use rari_tools::remove::remove;
@@ -63,7 +62,6 @@ enum Commands {
     Foo(BuildArgs),
     Serve(ServeArgs),
     GitHistory,
-    Popularities,
     Update(UpdateArgs),
     ExportSchema(ExportSchemaArgs),
     /// Subcommands for altering content programmatically
@@ -189,6 +187,7 @@ fn main() -> Result<(), Error> {
         rari_deps::bcd::update_bcd(rari_types::globals::data_dir())?;
         rari_deps::mdn_data::update_mdn_data(rari_types::globals::data_dir())?;
         rari_deps::web_ext_examples::update_web_ext_examples(rari_types::globals::data_dir())?;
+        rari_deps::popularities::update_popularities(rari_types::globals::data_dir())?;
     }
 
     let fmt_filter = filter::Targets::new()
@@ -377,12 +376,6 @@ fn main() -> Result<(), Error> {
             gather_history()?;
             println!("Took: {:?}", start.elapsed());
         }
-        Commands::Popularities => {
-            println!("Calculating popularities 🥇");
-            let start = std::time::Instant::now();
-            update_popularities(20000);
-            println!("Took: {:?}", start.elapsed());
-        }
         Commands::Content(content_subcommand) => match content_subcommand {
             ContentSubcommand::Move(args) => {
                 r#move(&args.old_slug, &args.new_slug, args.locale, args.assume_yes)?;

crates/rari-cli/serve.rs

@@ -17,7 +17,7 @@ use rari_doc::pages::json::BuiltPage;
 use rari_doc::pages::page::{Page, PageBuilder, PageLike};
 use rari_doc::pages::types::doc::Doc;
 use rari_doc::reader::read_docs_parallel;
-use rari_types::globals::{content_root, content_translated_root};
+use rari_types::globals::{self, content_root, content_translated_root};
 use rari_types::locale::Locale;
 use rari_types::Popularities;
 use rari_utils::io::read_to_string;
@@ -113,8 +113,8 @@ async fn get_search_index_handler(
 }
 
 fn get_search_index(locale: Locale) -> Result<Vec<SearchItem>, DocError> {
-    let in_file = content_root()
-        .join(Locale::EnUs.as_folder_str())
+    let in_file = globals::data_dir()
+        .join("popularities")
         .join("popularities.json");
     let json_str = read_to_string(in_file)?;
     let popularities: Popularities = serde_json::from_str(&json_str)?;

crates/rari-deps/Cargo.toml

@@ -8,6 +8,7 @@ rust-version.workspace = true
 
 [dependencies]
 rari-utils.workspace = true
+rari-types.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 chrono.workspace = true
@@ -17,3 +18,4 @@ reqwest.workspace = true
 css-syntax-types = { path = "../css-syntax-types" }
 tar = "0.4"
 flate2 = "1"
+csv = "1"

crates/rari-deps/src/current.rs

@@ -0,0 +1,8 @@
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+
+#[derive(Deserialize, Serialize, Default, Debug)]
+pub struct Current {
+    pub latest_last_check: Option<DateTime<Utc>>,
+    pub version: String,
+}

crates/rari-deps/src/github_release.rs

@@ -2,20 +2,14 @@ use std::fs::{self, File};
 use std::io::{BufWriter, Read, Write};
 use std::path::{Path, PathBuf};
 
-use chrono::{DateTime, Duration, Utc};
+use chrono::{Duration, Utc};
 use rari_utils::io::read_to_string;
 use reqwest::redirect::Policy;
-use serde::{Deserialize, Serialize};
 
+use crate::current::Current;
 use crate::error::DepsError;
 
-#[derive(Deserialize, Serialize, Default, Debug)]
-pub struct Current {
-    pub latest_last_check: Option<DateTime<Utc>>,
-    pub version: String,
-}
-
-/// Download and unpack an npm package for a given version (defaults to latest).
+/// Download a github release artifact for a given version (defaults to latest).
 pub fn get_artifact(
     base_url: &str,
     artifact: &str,

crates/rari-deps/src/lib.rs

@@ -1,9 +1,11 @@
 pub mod bcd;
+pub mod current;
 pub mod error;
 pub mod external_json;
 pub mod github_release;
 pub mod mdn_data;
 pub mod npm;
+pub mod popularities;
 pub mod web_ext_examples;
 pub mod web_features;
 pub mod webref_css;

crates/rari-deps/src/npm.rs

@@ -2,21 +2,15 @@ use std::fs;
 use std::io::Read;
 use std::path::{Path, PathBuf};
 
-use chrono::{DateTime, Duration, Utc};
+use chrono::{Duration, Utc};
 use flate2::read::GzDecoder;
 use rari_utils::io::read_to_string;
-use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use tar::Archive;
 
+use crate::current::Current;
 use crate::error::DepsError;
 
-#[derive(Deserialize, Serialize, Default, Debug)]
-pub struct Current {
-    pub latest_last_check: Option<DateTime<Utc>>,
-    pub version: String,
-}
-
 /// Download and unpack an npm package for a given version (defaults to latest).
 pub fn get_package(
     package: &str,

crates/rari-deps/src/popularities.rs

@@ -0,0 +1,79 @@
+use std::fs::{self, File};
+use std::io::BufWriter;
+use std::path::{Path, PathBuf};
+
+use chrono::{Datelike, Utc};
+use rari_types::Popularities;
+use rari_utils::io::read_to_string;
+use serde::Deserialize;
+
+use crate::current::Current;
+use crate::error::DepsError;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct PopularityRow {
+    #[serde(rename = "Page")]
+    pub page: String,
+    #[serde(rename = "Pageviews")]
+    pub page_views: f64,
+}
+
+const CURRENT_URL: &str = "https://popularities.mdn.mozilla.net/current.csv";
+const LIMIT: usize = 20_000;
+
+pub fn update_popularities(base_path: &Path) -> Result<Option<PathBuf>, DepsError> {
+    let version = "latest";
+    let package_path = base_path.join("popularities");
+    let last_check_path = package_path.join("last_check.json");
+    let now = Utc::now();
+    let current = read_to_string(last_check_path)
+        .ok()
+        .and_then(|current| serde_json::from_str::<Current>(&current).ok())
+        .unwrap_or_default();
+    let current_date = current.latest_last_check.unwrap_or_default().date_naive();
+    let now_date = now.date_naive();
+
+    if (current_date.year() < now_date.year() || current_date.month() < now_date.month())
+        && now_date.day() > 1
+    {
+        let mut popularities = Popularities {
+            popularities: Default::default(),
+            date: Utc::now().naive_utc(),
+        };
+
+        let mut max = f64::INFINITY;
+        let pop_csv = reqwest::blocking::get(CURRENT_URL).expect("unable to download popularities");
+        let mut rdr = csv::Reader::from_reader(pop_csv);
+        for row in rdr.deserialize::<PopularityRow>().flatten().take(LIMIT) {
+            if row.page.contains("/docs/") && !row.page.contains(['$', '?']) {
+                if max.is_infinite() {
+                    max = row.page_views;
+                }
+                popularities
+                    .popularities
+                    .insert(row.page, row.page_views / max);
+            }
+        }
+
+        let artifact_path = package_path.join("popularities.json");
+        if package_path.exists() {
+            fs::remove_dir_all(&package_path)?;
+        }
+        fs::create_dir_all(&package_path)?;
+
+        let file = File::create(artifact_path).unwrap();
+        let buffed = BufWriter::new(file);
+
+        serde_json::to_writer_pretty(buffed, &popularities).unwrap();
+
+        fs::write(
+            package_path.join("last_check.json"),
+            serde_json::to_string_pretty(&Current {
+                version: version.to_string(),
+                latest_last_check: Some(now),
+            })?,
+        )?;
+        return Ok(Some(package_path));
+    }
+    Ok(None)
+}

crates/rari-doc/src/pages/types/spa_homepage.rs

@@ -31,6 +31,10 @@ pub fn lastet_news(urls: &[&str]) -> Result<Vec<HomePageLatestNewsItem>, DocErro
                 },
                 published_at: post.meta.date,
             })),
+            Err(DocError::PageNotFound(url, category)) => {
+                tracing::warn!("page not found {url} ({category:?})");
+                None
+            }
             Err(e) => Some(Err(e)),
             x => {
                 tracing::debug!("{x:?}");
@@ -62,6 +66,10 @@ pub fn featured_articles(
                     title: doc.title().to_string(),
                     tag: parents(page).get(1).cloned(),
                 })),
+                Err(DocError::PageNotFound(url, category)) => {
+                    tracing::warn!("page not found {url} ({category:?})");
+                    None
+                }
                 Err(e) => Some(Err(e)),
                 x => {
                     tracing::debug!("{x:?}");

crates/rari-doc/src/search_index.rs

@@ -9,7 +9,7 @@ use std::collections::HashMap;
 use std::fs::File;
 use std::io::BufWriter;
 
-use rari_types::globals::{build_out_root, content_root};
+use rari_types::globals::{self, build_out_root};
 use rari_types::locale::Locale;
 use rari_types::Popularities;
 use rari_utils::error::RariIoError;
@@ -47,8 +47,8 @@ struct SearchItem<'a> {
 /// - The popularity data cannot be parsed.
 /// - An error occurs while creating or writing to the search index files.
 pub fn build_search_index(docs: &[Page]) -> Result<(), DocError> {
-    let in_file = content_root()
-        .join(Locale::EnUs.as_folder_str())
+    let in_file = globals::data_dir()
+        .join("popularities")
         .join("popularities.json");
     let json_str = read_to_string(in_file)?;
     let popularities: Popularities = serde_json::from_str(&json_str)?;

crates/rari-tools/src/lib.rs

@@ -3,7 +3,6 @@ pub mod error;
 pub mod git;
 pub mod history;
 pub mod r#move;
-pub mod popularities;
 pub mod redirects;
 pub mod remove;
 pub mod sidebars;