chore: Add devtool text optimizer

devtools/text-optimizer/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "text-optimizer"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+cargo_metadata = "0.18"
+clap = { version = "4", features = ["derive", "string"] }
+env_logger = "0.10"
+log = "0.4"
+proc-macro2 = { version = "1", features = ["span-locations"] }
+serde = "1.0"
+serde_yaml = "0.9"
+syn = { version = "2", features = ["visit", "full"] }
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]

devtools/text-optimizer/README.md

@@ -0,0 +1,4 @@
+- compile tools
+- extract text on rev 
+- edit
+- fillback on rev

devtools/text-optimizer/src/backfill.rs

@@ -0,0 +1,73 @@
+use crate::error::MyError;
+use crate::types::TextInfo;
+use crate::yaml_processor::load_yaml;
+
+use std::collections::HashMap;
+use std::fs::read_dir;
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::PathBuf;
+
+pub fn backfill(input_dir: &PathBuf) {
+    if let Ok(text_info_lists) = load_all_text_info_files(input_dir) {
+        backfill_edited(text_info_lists)
+    } else {
+        log::error!("Backfill failed to start, please fix text info file first.");
+    }
+}
+
+fn load_all_text_info_files(input_dir: &PathBuf) -> Result<Vec<(String, Vec<TextInfo>)>, MyError> {
+    let mut text_info_lists: Vec<(String, Vec<TextInfo>)> = vec![];
+    let entries = read_dir(input_dir).expect("Read text info file failed.");
+    for entry in entries.flatten() {
+        if let Some(file_name) = entry.file_name().to_str() {
+            if file_name.ends_with(".yml") {
+                let entry_path = entry.path();
+                log::info!("Load yaml: {:?}", entry_path);
+                let list = load_yaml(&entry_path).expect("load yaml");
+                text_info_lists.push((file_name.to_owned(), list))
+            }
+        }
+    }
+    Ok(text_info_lists)
+}
+
+fn backfill_edited(text_info_lists: Vec<(String, Vec<TextInfo>)>) {
+    let mut file_to_text_infos: HashMap<String, Vec<TextInfo>> = HashMap::new();
+
+    for list in text_info_lists {
+        log::info!("Parse text info file: {:?}", list.0);
+        for text_info in list.1 {
+            let file_name = text_info
+                .metadata()
+                .file()
+                .to_str()
+                .expect("Convert file path to string.")
+                .to_owned();
+            file_to_text_infos
+                .entry(file_name)
+                .or_default()
+                .push(text_info);
+        }
+    }
+
+    for (file_name, text_infos) in file_to_text_infos {
+        let mut source_code = String::new();
+        let mut file = File::open(&file_name).expect("Failed to open file");
+        file.read_to_string(&mut source_code)
+            .expect("Failed to read file");
+
+        for text_info in text_infos {
+            // Replace the match original with the edited
+            source_code = source_code.replace(text_info.original(), text_info.editable());
+        }
+
+        // Reopen the file in write mode and write the new source code
+        let mut new_file = File::create(&file_name).expect("Failed to create file");
+        new_file
+            .write_all(source_code.as_bytes())
+            .expect("Failed to write file");
+    }
+
+    log::info!("The backfill is completed, please review the modifications in the source code.");
+}

devtools/text-optimizer/src/error.rs

@@ -0,0 +1,19 @@
+#[derive(Debug)]
+pub enum MyError {
+    CommitId,
+    Io(std::io::Error),
+    Serde(serde_yaml::Error),
+    PathError,
+}
+
+impl From<std::io::Error> for MyError {
+    fn from(error: std::io::Error) -> Self {
+        MyError::Io(error)
+    }
+}
+
+impl From<serde_yaml::Error> for MyError {
+    fn from(error: serde_yaml::Error) -> Self {
+        MyError::Serde(error)
+    }
+}

devtools/text-optimizer/src/extractors/clap_extractor.rs

@@ -0,0 +1,96 @@
+use super::{extract_contents_in_brackets, Extractor};
+use crate::types::{Category, Meta, TextInfo};
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
+use syn::Expr::{self};
+use syn::Lit::Str;
+use syn::{visit::visit_file, File};
+
+#[derive(Default)]
+pub struct ClapExtractor {
+    save_file: PathBuf,
+    map: HashMap<(String, PathBuf), TextInfo>,
+    scanning_file_path: PathBuf,
+}
+
+impl ClapExtractor {
+    pub fn new(save_file: PathBuf) -> Self {
+        ClapExtractor {
+            save_file,
+            ..Default::default()
+        }
+    }
+}
+
+impl Extractor for ClapExtractor {
+    fn reset_scanning_path(&mut self, file_path: &Path) {
+        self.scanning_file_path = file_path.to_owned();
+    }
+
+    fn add_text_info(&mut self, text_info: TextInfo) {
+        let key = text_info.original().to_owned();
+        let file = text_info.metadata().file().clone();
+
+        if let Some(existing) = self.map.get_mut(&(key.to_owned(), file.clone())) {
+            existing.append_new_line(text_info.metadata().start_lines()[0]);
+        } else {
+            self.map.insert((key, file), text_info);
+        }
+    }
+
+    fn text_list(&self) -> Vec<TextInfo> {
+        let mut text_list: Vec<TextInfo> = self.map.values().cloned().collect();
+        text_list.sort_by(|a, b| {
+            let cmp = a.metadata().file().cmp(b.metadata().file());
+            if cmp == std::cmp::Ordering::Equal {
+                a.metadata().start_lines()[0].cmp(&b.metadata().start_lines()[0])
+            } else {
+                cmp
+            }
+        });
+        text_list
+    }
+
+    fn scanning_file_path(&self) -> &PathBuf {
+        &self.scanning_file_path
+    }
+
+    fn save_file_path(&self) -> &PathBuf {
+        &self.save_file
+    }
+
+    fn visit_file(&mut self, node: &File) {
+        visit_file(self, node)
+    }
+}
+
+impl syn::visit::Visit<'_> for ClapExtractor {
+    fn visit_expr_method_call(&mut self, expr: &syn::ExprMethodCall) {
+        let method_ident = &expr.method;
+        if *method_ident == "about" || *method_ident == "help" {
+            if let Some(Expr::Lit(lit)) = expr.args.first() {
+                if let Str(lit_str) = &lit.lit {
+                    let lit = lit_str.token().to_string();
+
+                    if let Some(text) = extract_contents_in_brackets(lit) {
+                        log::trace!("Found target text: {}", text);
+
+                        let span = lit_str.span();
+                        let start_line = span.start().line;
+                        let category =
+                            Category::from_str(method_ident.to_string().as_str()).unwrap();
+                        let meta = Meta::new_line(
+                            category,
+                            self.scanning_file_path.to_owned(),
+                            start_line,
+                        );
+                        self.add_text_info(TextInfo::new(text.clone(), text, meta));
+                    }
+                }
+            }
+        }
+        syn::visit::visit_expr_method_call(self, expr);
+    }
+}

devtools/text-optimizer/src/extractors/log_extractor.rs

@@ -0,0 +1,106 @@
+use super::{extract_contents_in_brackets, Extractor};
+use crate::types::{Category, Meta, TextInfo};
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
+use syn::Macro;
+use syn::{visit::visit_file, File};
+
+#[derive(Default)]
+pub struct LogExtractor {
+    save_file: PathBuf,
+    map: HashMap<(String, PathBuf), TextInfo>,
+    scanning_file_path: PathBuf,
+}
+
+impl LogExtractor {
+    pub fn new(save_file: PathBuf) -> Self {
+        LogExtractor {
+            save_file,
+            ..Default::default()
+        }
+    }
+}
+
+impl Extractor for LogExtractor {
+    fn reset_scanning_path(&mut self, file_path: &Path) {
+        self.scanning_file_path = file_path.to_owned();
+    }
+
+    fn add_text_info(&mut self, text_info: TextInfo) {
+        let key = text_info.original().to_owned();
+        let file = text_info.metadata().file().clone();
+
+        if let Some(existing) = self.map.get_mut(&(key.to_owned(), file.clone())) {
+            existing.append_new_line(text_info.metadata().start_lines()[0]);
+        } else {
+            self.map.insert((key, file), text_info);
+        }
+    }
+
+    fn text_list(&self) -> Vec<TextInfo> {
+        let mut text_list: Vec<TextInfo> = self.map.values().cloned().collect();
+        text_list.sort_by(|a, b| {
+            let cmp = a.metadata().file().cmp(b.metadata().file());
+            if cmp == std::cmp::Ordering::Equal {
+                a.metadata().start_lines()[0].cmp(&b.metadata().start_lines()[0])
+            } else {
+                cmp
+            }
+        });
+        text_list
+    }
+
+    fn scanning_file_path(&self) -> &PathBuf {
+        &self.scanning_file_path
+    }
+
+    fn save_file_path(&self) -> &PathBuf {
+        &self.save_file
+    }
+
+    fn visit_file(&mut self, node: &File) {
+        visit_file(self, node)
+    }
+}
+
+impl syn::visit::Visit<'_> for LogExtractor {
+    fn visit_macro(&mut self, node: &Macro) {
+        if let Some(name) = get_macro_name(node) {
+            if name == "error"
+                || name == "warn"
+                || name == "info"
+                || name == "debug"
+                || name == "trace"
+            {
+                if let Some(lit) = node.tokens.clone().into_iter().next() {
+                    if let Some(text) = extract_contents_in_brackets(lit.to_string()) {
+                        log::trace!("Found target text: {}", text);
+
+                        let span = lit.span();
+                        let start_line = span.start().line;
+                        let category = Category::from_str(&name).unwrap();
+                        let meta = Meta::new_line(
+                            category,
+                            self.scanning_file_path.to_owned(),
+                            start_line,
+                        );
+                        self.add_text_info(TextInfo::new(text.clone(), text, meta));
+                    }
+                }
+            }
+        }
+    }
+}
+
+fn get_macro_name(node: &Macro) -> Option<String> {
+    if let Some(ident) = node.path.get_ident() {
+        Some(ident.to_string())
+    } else {
+        node.path
+            .segments
+            .last()
+            .map(|segment| segment.ident.to_string())
+    }
+}