Merge pull request #634 from Ivanbeethoven/main

[scorpio]: Pack and Push in scorpio.

Author: genedna

docs/images/scorpio.drawio

@@ -262,6 +262,15 @@ impl Tree {
             tree_items,
         })
     }
+
+    /// After the subdirectory is changed, the hash value of the tree is recalculated.
+    pub fn rehash(&mut self){
+        let mut data = Vec::new();
+        for item in &self.tree_items {
+            data.extend_from_slice(item.to_data().as_slice());
+        }
+        self.id = SHA1::from_type_and_data(ObjectType::Tree, &data);
+    }
 }
 impl TryFrom<&[u8]> for Tree{
     type Error = GitError;

docs/images/scorpio2.drawio

@@ -6,7 +6,7 @@ edition = "2021"
 [dependencies]
 libra = { path = "../libra" }
 mercury = { path = "../mercury" }
-
+ceres = { path = "../ceres" }
 toml = "0.8.19"
 fuser = "0.14.0"
 libc = "0.2.158"
@@ -26,6 +26,7 @@ env_logger = "0.11.5"
 sled = "0.34.7"
 bincode = "1.3.3"
 async-recursion = "1.1.1"
+bytes = "1.7.2"
 
 [features]
 async-io = []

docs/images/scorpio2.png

@@ -64,3 +64,9 @@ The structure of Sapling is achieved through a multi-layered architecture, with
 [6] [fuser](https://github.com/cberner/fuser.git) : A Rust library crate for easy implementation of FUSE filesystems in userspace.
 
 [7] [Scalar](https://github.com/microsoft/git/blob/HEAD/contrib/scalar/docs/index.md) : Scalar is a tool that helps Git scale to some of the largest Git repositories. Initially, it was a single standalone git plugin based on Vfs for git, inheriting GVFS. No longer using FUSE. It implements aware partial directory management. Users need to manage and register the required workspace directory on their own. Ease of use can be improved through the fuse mechanism.
+
+
+# TODO
+1. * BUILD : monorepo build , which dictionary ?
+2. *** API list... for add mount point .
+3. - .https://opentitan.org/

mercury/src/internal/object/tree.rs

@@ -5,9 +5,9 @@ store_path = "/home/luxian/megadir/store"
 [[works]]
 path = "third-part/mega/libra"
 node = 26
-hash = "4a3bec47e31fc5442b9f9c42f519788e565ea485"
+hash = "47a90e9262bcd023bbff32f9f4c328b43b62813a"
 
 [[works]]
 path = "third-part/mega/scorpio"
 node = 17
-hash = "1b70e8bf4d39d6f5e9dd1637aaa2c221e2d00a27"
+hash = "76d6dbd014dee9a78497ea61d1ee923e8b0f9387"

scorpio/Cargo.toml

@@ -1,5 +1,6 @@
 use std::collections::HashMap;
-use std::{ffi::CString, fs};
+use std::vec;
+use std::ffi::CString;
 use std::path::{Path, PathBuf};
 use libc::{self, stat};
 use mercury::hash::SHA1;
@@ -8,12 +9,12 @@ use mercury::internal::object::tree::{Tree, TreeItem, TreeItemMode};
 use mercury::internal::object::types::ObjectType;
 
 use crate::manager::store::TreeStore;
-use crate::util::GPath;
+
 
 fn collect_paths<P: AsRef<Path>>(path: P) -> Vec<PathBuf> {
     let mut paths = Vec::new();
 
-    if let Ok(entries) = fs::read_dir(path) {
+    if let Ok(entries) = std::fs::read_dir(path) {
         for entry in entries.filter_map(Result::ok) {
             let path = entry.path();
             if path.is_dir() {
@@ -43,74 +44,177 @@ fn is_whiteout_inode<P: AsRef<Path>>(path: P)-> bool {
 
     false
 }
+/// the output tree in hashmap , which hash value is not computed.
 #[allow(unused)]
-fn diff(lower: PathBuf, upper:PathBuf,dbpath:&str)->(HashMap<PathBuf,Tree>,Vec<Blob>){
+pub fn diff(lower: PathBuf, upper:PathBuf,dbpath:&str,monopath:PathBuf)->(HashMap<PathBuf,Tree>,Vec<Blob>){
     let db = sled::open(dbpath).unwrap();
     let upper_changes = collect_paths(&lower); 
     let _root_len = lower.clone();
     let mut map = HashMap::<PathBuf,Tree>::new();
-    let root_tree = db.get_bypath(GPath::from(String::from(lower.to_str().unwrap()))).unwrap();
+    let root_tree = db.get_bypath(monopath.clone()).unwrap();// BUG: wrong path :"lower". the store in db is not real path .
+    // dont forget get tree below
     map.insert(lower.clone(), root_tree);
     let mut blobs=Vec::<Blob>::new();
+
     for node in upper_changes {
-            let p = node.parent().unwrap().to_path_buf();
-            let t = map.get_mut(&p).unwrap();
-
-            let mut  is_new_file = true;
-            // look up the older version to delete/change . 
-            let mut i =0;
-            while i<t.tree_items.len() {
-                let item = &mut t.tree_items[i];
-                if item.name.eq(node.file_name().unwrap().to_string_lossy().as_ref()){
-                    is_new_file = false;
-                    //delete . 
-                    if is_whiteout_inode(&node){
-                        t.tree_items.remove(i);
-                    }else{
-                        // changed 
-                        // node is a changed ()blob.
-                        // just compute the NEW hash first.
-                        let content = std::fs::read(&node).unwrap();
-                        item.id= SHA1::from_type_and_data(ObjectType::Blob, &content);
-                        blobs.push(Blob { id: item.id, data: content });
+        if node.is_dir(){
+            let new_path = monopath.clone().join(node.strip_prefix(&lower).unwrap());
+            let node_tree = db.get_bypath(new_path).unwrap();
+            map.insert(node.clone(), node_tree);
+            //db.get_bypath(path);
+            //ap.insert(node.clone(), node_tree);
+        }
+        let p = node.parent().unwrap().to_path_buf();
+        let t = map.get_mut(&p).unwrap();
+
+        let mut  is_new_file = true;
+        // look up the parent tree node to delete/change . 
+        let mut i =0;
+        while i<t.tree_items.len() {
+            let item = &mut t.tree_items[i];
+            if item.name.eq(node.file_name().unwrap().to_string_lossy().as_ref()){
+                is_new_file = false;
+                //delete . 
+                if is_whiteout_inode(&node){
+                    t.tree_items.remove(i);
+                }
+                else if node.is_dir(){
+                    //pass
+                }
+                else{
+                    // changed 
+                    // node is a changed ()blob.
+                    // just compute the NEW hash first.
+                    let content = std::fs::read(&node).unwrap();
+                    item.id= SHA1::from_type_and_data(ObjectType::Blob, &content);
+                    blobs.push(Blob { id: item.id, data: content });
+                }
+                break;
+            }
+            i+=1;
+        }
+        // if new file add item.
+        let new_name =node.file_name().unwrap().to_string_lossy().as_ref().to_owned();
+        if is_new_file{
+            //is a fiel or a dictionary?
+            if node.is_dir(){
+                // is dictionary. 
+                t.tree_items.push(TreeItem {
+                    mode: TreeItemMode::Tree, 
+                    id: SHA1::default(), 
+                    name: new_name,
+                });
+            }else {
+                //is a file.
+                let content = std::fs::read(&node).unwrap();
+                let hash = SHA1::from_type_and_data(ObjectType::Blob, &content);
+                blobs.push(Blob { id: hash, data: content });
+                t.tree_items.push(TreeItem {
+                    mode: TreeItemMode::Blob, 
+                    id: hash, 
+                    name: new_name,
+                });
+            }
+
+        }
+    }
+    (map,blobs)
+}
+
+pub fn change(
+    real_path:PathBuf,
+    tree_path:PathBuf,
+    trees:&mut Vec<Tree>,
+    blobs:&mut Vec<Blob>,
+    db:&sled::Db) -> Tree{
+        let mut tree;
+        let root_tree = db.get_bypath(tree_path.clone());
+        if let Ok(root_tree) = root_tree{// exit dictionry
+            println!("exit tree:{:?}",tree_path);
+            tree = root_tree;
+        }else {// there is a new dictionary. 
+            println!("new tree:{:?}",tree_path);
+            tree = Tree{ id: SHA1::default(), tree_items: vec![] };
+        }
+
+        let entries = std::fs::read_dir(real_path).unwrap();
+        for entry in entries {
+            let entry = entry.unwrap();
+            let path = entry.path();
+            let name = path.file_name().unwrap().to_string_lossy().into_owned().to_string();
+            let mut new = true;
+            let mut i =0 ;
+            while i<tree.tree_items.len() {
+                let item: &mut TreeItem  = tree.tree_items.get_mut(i).unwrap();
+                if item.name.eq(&name) {
+                    if is_whiteout_inode(&path){
+                        println!("change: delete {}",item.name);
+                        // delete..
+                        let _ = item;//drop
+                        tree.tree_items.remove(i);
+                    }
+                    else if path.is_dir() {
+                        let new_tree_path = tree_path.join(path.file_name().unwrap());
+                        let new_tree = change(path.clone(), new_tree_path, trees, blobs, db);
+
+                        // change the hash value & push tree .
+                        item.id = new_tree.id;
+                        trees.push(new_tree);
+                        println!("change: changed tree {}",item.name);
+                    } else {
+                        println!("change: changed file {}",item.name);
+                        let content = std::fs::read(&path).unwrap();
+                        let hash = SHA1::from_type_and_data(ObjectType::Blob, &content);
+                        blobs.push(Blob { id: hash, data: content });
+                        item.id = hash;// change fiel hash .
                     }
+                    new = false;
                     break;
                 }
                 i+=1;
             }
-            // if new file add item.
-            let new_name =node.file_name().unwrap().to_string_lossy().as_ref().to_owned();
-            if is_new_file{
-                //is a fiel or a dictionary?
-                if node.is_dir(){
-                    // is dictionary. 
-                    t.tree_items.push(TreeItem {
-                        mode: TreeItemMode::Tree, 
-                        id: SHA1::default(), 
-                        name: new_name,
+            if new{// a new file or dictionary
+                if path.is_dir() {
+                    println!("change: new tree  {:?}",name);
+                    let new_tree_path = tree_path.join(path.file_name().unwrap());
+                    let new_tree = change(path.clone(), new_tree_path, trees, blobs, db);
+                    
+                    //add a new item for this tree. and push tree.
+                    tree.tree_items.push(TreeItem {
+                        mode: TreeItemMode::Tree,
+                        id: new_tree.id,
+                        name ,
                     });
-                }else {
-                    //is a file.
-                    let content = std::fs::read(&node).unwrap();
+                    trees.push(new_tree);
+                } else {
+                    println!("change: new file  {}",name);
+                    let content = std::fs::read(&path).unwrap();
                     let hash = SHA1::from_type_and_data(ObjectType::Blob, &content);
                     blobs.push(Blob { id: hash, data: content });
-                    t.tree_items.push(TreeItem {
-                        mode: TreeItemMode::Blob, 
-                        id: hash, 
-                        name: new_name,
+                    tree.tree_items.push(TreeItem {
+                        mode: TreeItemMode::Blob,
+                        id: hash,
+                        name ,
                     });
                 }
-
             }
-    }
-    (map,blobs)
+
+
+           
+        }
+        tree.rehash();//Re compute the hash value .
+        tree
+
 }
+
 #[cfg(test)]
 mod tests {
-    use fs::File;
+
+    use std::fs::File;
 
     use super::*;
 
+
     #[test]
     fn test_collect_paths_nested_directories() {
         let temp_dir = "/home/luxian/code/mega/scorpio/src";

scorpio/README.md

@@ -1,4 +1,3 @@
-
 use std::path::{Path, PathBuf};
 use std::{collections::VecDeque, sync::Arc, time::Duration};
 use mercury::hash::SHA1;
@@ -61,7 +60,7 @@ async fn worker_thread(
     send_tree :Sender<Tree>,
 ) {
     let client = Client::new();
-    let mut interval = time::interval(Duration::from_millis(50)); // 设定检查间隔时间
+    let mut interval = time::interval(Duration::from_millis(50)); 
     let timeout_duration = Duration::from_millis(300);
     loop {
         let path = tokio::select! {
@@ -165,7 +164,6 @@ async fn worker_ro_thread(
                 tokio::fs::create_dir_all(real_path).await.unwrap();
             } else {
 
-                // TODO: fetch file and save to target path. about file fetch api, refer to test_fetch_octet_stream() test func. 
                 let e = fetch_and_save_file(&item.id,real_path).await;
                 println!("{:?}",e);
             }
@@ -175,6 +173,11 @@ async fn worker_ro_thread(
         }
 
 }
+
+///
+/// the tree info is store in k-v database.
+///     monorepo path  -> Tree
+/// 
 async fn fetch_code(path:&GPath, save_path : impl AsRef<Path>){
 
     let target_path: Arc<PathBuf> = Arc::new(save_path.as_ref().to_path_buf());
@@ -342,7 +345,6 @@ mod tests {
 
         // Use the URL from environment variables or local test URL
         let url = "http://localhost:8000/api/v1/file/blob/d12d12579799a658b29808fe695abd919a033ac9";
-
         // Send a GET request
         let response = client.get(url).send().await.unwrap();
 

scorpio/config.toml

@@ -1,7 +1,8 @@
 use serde::{Deserialize, Serialize};
 use std::fs;
 
-mod diff;
+pub mod diff;
+pub mod push;
 pub mod fetch;
 pub mod store;