Add neptune

.github/workflows/base.yml

@@ -6,7 +6,7 @@
 #
 #
 
-on: [ push, pull_request ]
+on: [push, pull_request]
 
 name: Base GitHub Action for Check, Test and Lints
 
@@ -17,6 +17,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -32,6 +34,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -49,6 +53,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -69,5 +75,6 @@ jobs:
           profile: minimal
           toolchain: stable
           override: true
+          submodules: recursive
       - run: sudo apt-get update && sudo apt-get install -y fuse3 libfuse3-dev
-      - run: cd ./scorpio && cargo clippy --all-targets --all-features -- -D warnings
+      - run: cd ./scorpio && cargo clippy --all-targets --all-features -- -D warnings

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "neptune/libs/ztm"]
+	path = neptune/libs/ztm
+	url = [email protected]:flomesh-io/ztm.git
+    ignore = dirty

Cargo.toml

@@ -9,6 +9,7 @@ members = [
     "ceres",
     "libra",
     "vault",
+    "neptune",
 ]
 exclude = ["craft"]
 resolver = "1"
@@ -23,6 +24,7 @@ ceres = { path = "ceres" }
 callisto = { path = "jupiter/callisto" }
 gemini = { path = "gemini" }
 vault = { path = "vault" }
+neptune = { path = "neptune" }
 
 anyhow = "1.0.86"
 serde = "1.0.203"

gemini/Cargo.toml

@@ -21,4 +21,4 @@ tokio = { workspace = true, features = ["net"] }
 chrono = { workspace = true }
 jupiter = { workspace = true }
 callisto = { workspace = true }
-rust-ztm = { git = "https://github.com/flomesh-io/rust-ztm.git" }
+neptune = { workspace = true }

gemini/src/ztm/mod.rs

@@ -271,7 +271,7 @@ pub struct LocalZTM {
 impl LocalZTM {
     pub fn start_ztm_agent(self) {
         tokio::spawn(async move {
-            rust_ztm::start_agent("ztm_agent.db", self.agent_port);
+            neptune::start_agent("ztm_agent.db", self.agent_port);
         });
     }
 }

neptune/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "neptune"
+version = "0.1.0"
+edition = "2021"
+
+[build-dependencies]
+cmake = "0.1.50"
+num_cpus = { workspace = true }
+
+[dependencies]
+libc = "0.1.0"
+reqwest = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+tokio = { workspace = true, features = ["macros"] }
+tokio-test = { workspace = true }
+
+[features]
+agent-ui = []

neptune/README.md

@@ -0,0 +1,11 @@
+# neptune
+rust wrap of flomesh-io/ztm
+
+> [!IMPORTANT]
+>
+> Building `neptune` requires `cmake`, `clang`, `nodejs`, `npm`, and the `rust toolchain`.
+>
+> Additionally, building with the `agent-ui` features requires that `vite` be installed.
+
+# features
+- [x] `agent-ui`: build with ztm agent's web ui，for development and testing purpose.

neptune/build.rs

@@ -0,0 +1,112 @@
+use std::{fs, path::Path};
+
+use cmake::Config;
+fn build_agent_ui() {
+    let ui = Path::new("libs/ztm/agent/gui");
+    if cfg!(feature = "agent-ui") {
+        if !ui.exists() {
+            // run npm run build in the libs/ztm/gui
+            let _ = std::process::Command::new("npm")
+                .current_dir("libs/ztm/gui")
+                .arg("run")
+                .arg("build")
+                .output()
+                .expect("failed to run npm run build in ztm/gui");
+        }
+    } else if ui.exists() {
+        std::fs::remove_dir_all(ui).expect("failed to remove agent/gui");
+    }
+}
+
+fn parse_args_to_rustc(dst: &Path) {
+    // ** `cargo:rustc-*` format is used to pass information to the cargo build system
+
+    // parse to `rustc` to look for dynamic library, used in running
+    let origin_path = if cfg!(target_os = "macos") {
+        "@executable_path"
+    } else if cfg!(target_os = "linux") {
+        "$ORIGIN"
+    } else {
+        "" // windows auto search excutable path
+    };
+    println!(
+        "cargo:rustc-link-arg=-Wl,-rpath,{}/build,-rpath,{}",
+        dst.display(),
+        origin_path
+    );
+
+    // add the path to the library to the linker search path, used in build
+    println!("cargo:rustc-link-search={}/build", dst.display());
+
+    println!("cargo:rustc-link-lib=pipy");
+}
+
+/// copy `mega-app/` to `libs/ztm/agent/apps/mega`
+fn copy_mega_apps() {
+    let src = Path::new("mega_app");
+    let dst = Path::new("libs/ztm/agent/apps/mega");
+    if src.exists() {
+        if dst.exists() {
+            fs::remove_dir_all(dst).expect("failed to remove origin agent/apps");
+        }
+        // std::fs::copy(src, dst).expect("failed to copy mega to agent/apps");
+        copy_dir_all(src, dst).expect("failed to copy mega to agent/apps");
+    }
+}
+
+fn main() {
+    // check submodule exists
+    let check_file = Path::new("libs/ztm/pipy/CMakeLists.txt");
+    if !check_file.exists() {
+        panic!("Please run `git submodule update --init --recursive` to get the submodule");
+    }
+    copy_mega_apps();
+
+    build_agent_ui();
+
+    /* compile ztm & pipy */
+    // run npm install in the libs/ztm
+    let _ = std::process::Command::new("npm")
+        .current_dir("libs/ztm/pipy")
+        .arg("install")
+        .output()
+        .expect("failed to run npm install in ztm/pipy");
+
+    let mut config = Config::new("libs/ztm/pipy");
+
+    // set to use clang/clang++ to compile
+    config.define("CMAKE_C_COMPILER", "clang");
+    config.define("CMAKE_CXX_COMPILER", "clang++");
+
+    // compile ztm in pipy
+    config.define("PIPY_SHARED", "ON");
+    config.define("PIPY_GUI", "OFF");
+    config.define("PIPY_CODEBASES", "ON");
+    config.define(
+        "PIPY_CUSTOM_CODEBASES",
+        "ztm/agent:../agent,ztm/hub:../hub,ztm/ca:../ca",
+    );
+
+    config.no_build_target(true);
+
+    // build, with half of the cpu
+    let cups = num_cpus::get() - num_cpus::get() / 2;
+    std::env::set_var("CMAKE_BUILD_PARALLEL_LEVEL", cups.to_string());
+    let dst = config.build();
+
+    parse_args_to_rustc(&dst);
+}
+
+fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> std::io::Result<()> {
+    fs::create_dir_all(&dst)?;
+    for entry in fs::read_dir(src)? {
+        let entry = entry?;
+        let ty = entry.file_type()?;
+        if ty.is_dir() {
+            copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
+        } else {
+            fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
+        }
+    }
+    Ok(())
+}