From 3d9f66aa8a6912c36639eb6d3a5511e68b53b67c Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sun, 30 Apr 2023 23:53:24 +0800 Subject: [PATCH 1/8] Add support for filtering by language type. --- Cargo.lock | 1 + Cargo.toml | 1 + README.md | 11 +++++++++++ src/main.rs | 55 +++++++++++++++++++++++++++++++++++++++-------------- 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c067b2..9f14a05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2329,6 +2329,7 @@ name = "tokei_rs" version = "0.1.0" dependencies = [ "actix-web", + "base64 0.21.0", "cached", "csscolorparser", "dotenv", diff --git a/Cargo.toml b/Cargo.toml index 23a5510..3a22b45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,4 @@ eyre = "0.6" env_logger = "0.10" rsbadges = "1.1" csscolorparser = "0.6" +base64 = "0.21" diff --git a/README.md b/README.md index 83d7bae..d766768 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,17 @@ Here is an example showing total number of code. [![](https://tokei.rs/b1/github/XAMPPRocky/tokei?category=code)](https://github.com/XAMPPRocky/tokei). ``` +## Type + +You can choose to count lines only for specific language type(s), by using the `?type=` query +string. Languages are to be separated by a comma. +Here is an example showing total number of lines for JSON, Rust, and Markdown. +[![lines of json, rust, and markdown](https://tokei.rs/b1/github/XAMPPRocky/tokei?type=JSON,Rust,Markdown)](https://github.com/XAMPPRocky/tokei). + +```sh +[![](https://tokei.rs/b1/github/XAMPPRocky/tokei?type=JSON,Rust,Markdown)](https://github.com/XAMPPRocky/tokei). +``` + ## Label You can customize the badge label by using the `?label=` query string. For example, [![custom label](https://tokei.rs/b1/github/XAMPPRocky/tokei?category=code&label=custom%20label)](https://github.com/XAMPPRocky/tokei). diff --git a/src/main.rs b/src/main.rs index 4937c20..877c261 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ use actix_web::{ }, web, App, HttpRequest, HttpResponse, HttpServer, }; +use base64::{engine::general_purpose, Engine as _}; use cached::Cached; use csscolorparser::parse; use once_cell::sync::Lazy; @@ -88,6 +89,7 @@ struct BadgeQuery { style: Option, color: Option, logo: Option, + r#type: Option, } #[get("/b1/{domain}/{user}/{repo}")] @@ -105,6 +107,7 @@ async fn create_badge( let style: String = query.style.unwrap_or_else(|| "plastic".to_owned()); let color: String = query.color.unwrap_or_else(|| BLUE.to_owned()); let logo: String = query.logo.unwrap_or_else(|| "".to_owned()); + let r#type: String = query.r#type.unwrap_or_else(|| "".to_owned()); let content_type = if let Ok(accept) = Accept::parse(&request) { if accept == Accept::json() { @@ -134,36 +137,48 @@ async fn create_badge( .and_then(|bytes| String::from_utf8(bytes).ok()) .ok_or_else(|| actix_web::error::ErrorBadRequest(eyre::eyre!("Invalid SHA provided.")))?; + let mut language_types: Vec = r#type + .split(',') + .map(str::parse::) + .filter_map(Result::ok) + .collect(); + language_types.sort(); + let language_types_encoded = + general_purpose::STANDARD_NO_PAD.encode(format!("{:?}", language_types)); + + let entity_hash = format!("{}#{}", sha, language_types_encoded); if let Ok(if_none_match) = IfNoneMatch::parse(&request) { - log::debug!("Checking If-None-Match: {}", sha); - let sha_tag = EntityTag::new(false, sha.clone()); + log::debug!("Checking If-None-Match: {}", entity_hash); + let entity_tag = EntityTag::new(false, entity_hash.clone()); let found_match = match if_none_match { IfNoneMatch::Any => false, - IfNoneMatch::Items(items) => items.iter().any(|etag| etag.weak_eq(&sha_tag)), + IfNoneMatch::Items(items) => items.iter().any(|etag| etag.weak_eq(&entity_tag)), }; if found_match { CACHE .lock() .unwrap() - .cache_get(&repo_identifier(&url, &sha)); - log::info!("{}#{} Not Modified", url, sha); + .cache_get(&repo_identifier(&url, &sha, &language_types)); + log::info!("{}#{} Not Modified", url, entity_hash); return Ok(respond!(NotModified)); } } - let entry = get_statistics(&url, &sha).map_err(actix_web::error::ErrorBadRequest)?; + let entry = + get_statistics(&url, &sha, &language_types).map_err(actix_web::error::ErrorBadRequest)?; if entry.was_cached { - log::info!("{}#{} Cache hit", url, sha); + log::info!("{}#{} Cache hit", url, entity_hash); } let stats = entry.value; log::info!( - "{url}#{sha} - Lines {lines} Code {code} Comments {comments} Blanks {blanks}", + "{url}#{sha}#{language_types:?} - Lines {lines} Code {code} Comments {comments} Blanks {blanks}", url = url, sha = sha, + language_types = language_types, lines = stats.lines(), code = stats.code, comments = stats.comments, @@ -181,11 +196,11 @@ async fn create_badge( no_label, )?; - Ok(respond!(Ok, content_type, badge, sha)) + Ok(respond!(Ok, content_type, badge, entity_hash)) } -fn repo_identifier(url: &str, sha: &str) -> String { - format!("{}#{}", url, sha) +fn repo_identifier(url: &str, sha: &str, language_types: &Vec) -> String { + format!("{}#{}#{:?}", url, sha, language_types) } #[cached::proc_macro::cached( @@ -194,9 +209,13 @@ fn repo_identifier(url: &str, sha: &str) -> String { with_cached_flag = true, type = "cached::TimedSizedCache>", create = "{ cached::TimedSizedCache::with_size_and_lifespan(1000, DAY_IN_SECONDS) }", - convert = r#"{ repo_identifier(url, _sha) }"# + convert = r#"{ repo_identifier(url, _sha, language_types) }"# )] -fn get_statistics(url: &str, _sha: &str) -> eyre::Result> { +fn get_statistics( + url: &str, + _sha: &str, + language_types: &Vec, +) -> eyre::Result> { log::info!("{} - Cloning", url); let temp_dir = TempDir::new()?; let temp_path = temp_dir.path().to_str().unwrap(); @@ -208,7 +227,15 @@ fn get_statistics(url: &str, _sha: &str) -> eyre::Result Date: Mon, 1 May 2023 00:11:34 +0800 Subject: [PATCH 2/8] Add some comments. --- src/main.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main.rs b/src/main.rs index 877c261..427c9a0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -142,10 +142,14 @@ async fn create_badge( .map(str::parse::) .filter_map(Result::ok) .collect(); + // Caching should be insensitive to order of languages specified by user + // e.g. "?type=Rust,JSON" and "?type=JSON,Rust" are equivalent language_types.sort(); + // Use Base64-encoding as `language_types` may contain characters disallowed by EntityTag (such as whitespace) let language_types_encoded = general_purpose::STANDARD_NO_PAD.encode(format!("{:?}", language_types)); + // Check if both git commit `sha` and `language_types` match let entity_hash = format!("{}#{}", sha, language_types_encoded); if let Ok(if_none_match) = IfNoneMatch::parse(&request) { log::debug!("Checking If-None-Match: {}", entity_hash); From 629cc326d6ae8d20d7c0127e17413d82e56da9ba Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Mon, 1 May 2023 01:33:43 +0800 Subject: [PATCH 3/8] Make caching insensitive to duplicate languages. --- src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.rs b/src/main.rs index 427c9a0..e41c7b0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -145,6 +145,7 @@ async fn create_badge( // Caching should be insensitive to order of languages specified by user // e.g. "?type=Rust,JSON" and "?type=JSON,Rust" are equivalent language_types.sort(); + language_types.dedup(); // Use Base64-encoding as `language_types` may contain characters disallowed by EntityTag (such as whitespace) let language_types_encoded = general_purpose::STANDARD_NO_PAD.encode(format!("{:?}", language_types)); From e664f68debec591b047b9941b71b319d6864739a Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Mon, 1 May 2023 16:56:16 +0800 Subject: [PATCH 4/8] Make language_types immutable --- src/main.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main.rs b/src/main.rs index e41c7b0..3e75f15 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ use cached::Cached; use csscolorparser::parse; use once_cell::sync::Lazy; use rsbadges::{Badge, Style}; +use std::collections::HashSet; use tempfile::TempDir; use tokei::{Language, Languages}; @@ -137,15 +138,16 @@ async fn create_badge( .and_then(|bytes| String::from_utf8(bytes).ok()) .ok_or_else(|| actix_web::error::ErrorBadRequest(eyre::eyre!("Invalid SHA provided.")))?; - let mut language_types: Vec = r#type - .split(',') - .map(str::parse::) - .filter_map(Result::ok) - .collect(); // Caching should be insensitive to order of languages specified by user // e.g. "?type=Rust,JSON" and "?type=JSON,Rust" are equivalent - language_types.sort(); - language_types.dedup(); + let language_types: Vec = r#type + .split(',') + .filter_map(|s| str::parse::(s).ok()) + .into_iter() + .collect::>() + .into_iter() + .collect(); + // Use Base64-encoding as `language_types` may contain characters disallowed by EntityTag (such as whitespace) let language_types_encoded = general_purpose::STANDARD_NO_PAD.encode(format!("{:?}", language_types)); From 4cbf084e1c5454d088a8ec837914204000ec6d9d Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Tue, 2 May 2023 15:44:47 +0800 Subject: [PATCH 5/8] Cache only git SHA --- Cargo.lock | 1 - Cargo.toml | 1 - src/main.rs | 92 +++++++++++++++++++++++------------------------------ 3 files changed, 39 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9f14a05..7c067b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2329,7 +2329,6 @@ name = "tokei_rs" version = "0.1.0" dependencies = [ "actix-web", - "base64 0.21.0", "cached", "csscolorparser", "dotenv", diff --git a/Cargo.toml b/Cargo.toml index 3a22b45..23a5510 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,4 +19,3 @@ eyre = "0.6" env_logger = "0.10" rsbadges = "1.1" csscolorparser = "0.6" -base64 = "0.21" diff --git a/src/main.rs b/src/main.rs index 3e75f15..2805415 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,14 +8,13 @@ use actix_web::{ }, web, App, HttpRequest, HttpResponse, HttpServer, }; -use base64::{engine::general_purpose, Engine as _}; -use cached::Cached; +use cached::{Cached, Return}; use csscolorparser::parse; use once_cell::sync::Lazy; use rsbadges::{Badge, Style}; use std::collections::HashSet; use tempfile::TempDir; -use tokei::{Language, Languages}; +use tokei::{Language, LanguageType, Languages}; const BILLION: usize = 1_000_000_000; const BLANKS: &str = "blank lines"; @@ -138,25 +137,10 @@ async fn create_badge( .and_then(|bytes| String::from_utf8(bytes).ok()) .ok_or_else(|| actix_web::error::ErrorBadRequest(eyre::eyre!("Invalid SHA provided.")))?; - // Caching should be insensitive to order of languages specified by user - // e.g. "?type=Rust,JSON" and "?type=JSON,Rust" are equivalent - let language_types: Vec = r#type - .split(',') - .filter_map(|s| str::parse::(s).ok()) - .into_iter() - .collect::>() - .into_iter() - .collect(); - - // Use Base64-encoding as `language_types` may contain characters disallowed by EntityTag (such as whitespace) - let language_types_encoded = - general_purpose::STANDARD_NO_PAD.encode(format!("{:?}", language_types)); - - // Check if both git commit `sha` and `language_types` match - let entity_hash = format!("{}#{}", sha, language_types_encoded); + // Check if git commit `sha` matches if let Ok(if_none_match) = IfNoneMatch::parse(&request) { - log::debug!("Checking If-None-Match: {}", entity_hash); - let entity_tag = EntityTag::new(false, entity_hash.clone()); + log::debug!("Checking If-None-Match: {}", sha); + let entity_tag = EntityTag::new(false, sha.clone()); let found_match = match if_none_match { IfNoneMatch::Any => false, IfNoneMatch::Items(items) => items.iter().any(|etag| etag.weak_eq(&entity_tag)), @@ -166,26 +150,39 @@ async fn create_badge( CACHE .lock() .unwrap() - .cache_get(&repo_identifier(&url, &sha, &language_types)); - log::info!("{}#{} Not Modified", url, entity_hash); + .cache_get(&repo_identifier(&url, &sha)); + log::info!("{}#{} Not Modified", url, sha); return Ok(respond!(NotModified)); } } - let entry = - get_statistics(&url, &sha, &language_types).map_err(actix_web::error::ErrorBadRequest)?; + let entry: Return> = + get_statistics(&url, &sha).map_err(actix_web::error::ErrorBadRequest)?; if entry.was_cached { - log::info!("{}#{} Cache hit", url, entity_hash); + log::info!("{}#{} Cache hit", url, sha); } - let stats = entry.value; + let language_types: HashSet = r#type + .split(',') + .filter_map(|s| str::parse::(s).ok()) + .into_iter() + .collect::>(); + + let mut stats = Language::new(); + let languages: Vec<(LanguageType, Language)> = entry.value; + + for (language_type, language) in languages { + if language_types.is_empty() || language_types.contains(&language_type) { + stats += language; + } + } log::info!( - "{url}#{sha}#{language_types:?} - Lines {lines} Code {code} Comments {comments} Blanks {blanks}", + "{url}#{sha} - Types {language_types:?} Lines {lines} Code {code} Comments {comments} Blanks {blanks}", url = url, sha = sha, - language_types = language_types, + language_types = language_types.into_iter(), lines = stats.lines(), code = stats.code, comments = stats.comments, @@ -203,26 +200,25 @@ async fn create_badge( no_label, )?; - Ok(respond!(Ok, content_type, badge, entity_hash)) + Ok(respond!(Ok, content_type, badge, sha)) } -fn repo_identifier(url: &str, sha: &str, language_types: &Vec) -> String { - format!("{}#{}#{:?}", url, sha, language_types) +fn repo_identifier(url: &str, sha: &str) -> String { + format!("{}#{}", url, sha) } #[cached::proc_macro::cached( name = "CACHE", result = true, with_cached_flag = true, - type = "cached::TimedSizedCache>", + type = "cached::TimedSizedCache>>", create = "{ cached::TimedSizedCache::with_size_and_lifespan(1000, DAY_IN_SECONDS) }", - convert = r#"{ repo_identifier(url, _sha, language_types) }"# + convert = r#"{ repo_identifier(url, _sha) }"# )] fn get_statistics( url: &str, _sha: &str, - language_types: &Vec, -) -> eyre::Result> { +) -> eyre::Result>> { log::info!("{} - Cloning", url); let temp_dir = TempDir::new()?; let temp_path = temp_dir.path().to_str().unwrap(); @@ -231,28 +227,18 @@ fn get_statistics( .args(["clone", url, temp_path, "--depth", "1"]) .output()?; - let mut stats = Language::new(); let mut languages = Languages::new(); log::info!("{} - Getting Statistics", url); - let config = tokei::Config { - types: if language_types.is_empty() { - None - } else { - Some(language_types.to_owned()) - }, - ..tokei::Config::default() - }; - languages.get_statistics(&[temp_path], &[], &config); - - for (_, language) in languages { - stats += language; - } + languages.get_statistics(&[temp_path], &[], &tokei::Config::default()); - for stat in &mut stats.reports { - stat.name = stat.name.strip_prefix(temp_path)?.to_owned(); + let mut iter = languages.iter_mut(); + while let Some((_, language)) = iter.next() { + for report in &mut language.reports { + report.name = report.name.strip_prefix(temp_path)?.to_owned(); + } } - Ok(cached::Return::new(stats)) + Ok(cached::Return::new(languages.into_iter().collect())) } fn trim_and_float(num: usize, trim: usize) -> f64 { From 44c57ad0e8889a6e136a93a7972ddf0a514bc8a7 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Tue, 2 May 2023 16:15:24 +0800 Subject: [PATCH 6/8] Revert naming of sha_tag --- src/main.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2805415..153f0f4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -137,13 +137,12 @@ async fn create_badge( .and_then(|bytes| String::from_utf8(bytes).ok()) .ok_or_else(|| actix_web::error::ErrorBadRequest(eyre::eyre!("Invalid SHA provided.")))?; - // Check if git commit `sha` matches if let Ok(if_none_match) = IfNoneMatch::parse(&request) { log::debug!("Checking If-None-Match: {}", sha); - let entity_tag = EntityTag::new(false, sha.clone()); + let sha_tag = EntityTag::new(false, sha.clone()); let found_match = match if_none_match { IfNoneMatch::Any => false, - IfNoneMatch::Items(items) => items.iter().any(|etag| etag.weak_eq(&entity_tag)), + IfNoneMatch::Items(items) => items.iter().any(|etag| etag.weak_eq(&sha_tag)), }; if found_match { From 848387fa5300efe668e42d26d324dbd9adc7b959 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Wed, 3 May 2023 00:42:36 +0800 Subject: [PATCH 7/8] Log should display only languages that exist in repo --- src/main.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 153f0f4..45f85dd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -171,17 +171,17 @@ async fn create_badge( let mut stats = Language::new(); let languages: Vec<(LanguageType, Language)> = entry.value; - for (language_type, language) in languages { + for (language_type, language) in &languages { if language_types.is_empty() || language_types.contains(&language_type) { - stats += language; + stats += language.clone(); } } log::info!( - "{url}#{sha} - Types {language_types:?} Lines {lines} Code {code} Comments {comments} Blanks {blanks}", + "{url}#{sha} - Languages (most common to least common) {languages:#?} Lines {lines} Code {code} Comments {comments} Blanks {blanks}", url = url, sha = sha, - language_types = language_types.into_iter(), + languages = languages, lines = stats.lines(), code = stats.code, comments = stats.comments, From feda2971fd2c6fb361320ef673480af51c32ee2c Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Wed, 3 May 2023 00:43:57 +0800 Subject: [PATCH 8/8] Remove inaccurate statement --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 45f85dd..29e200a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -178,7 +178,7 @@ async fn create_badge( } log::info!( - "{url}#{sha} - Languages (most common to least common) {languages:#?} Lines {lines} Code {code} Comments {comments} Blanks {blanks}", + "{url}#{sha} - Languages {languages:#?} Lines {lines} Code {code} Comments {comments} Blanks {blanks}", url = url, sha = sha, languages = languages,