Cleanup implants lib and refactor dependencies

.gitignore

@@ -3,7 +3,7 @@
 /target/
 tests/create_file_dll/target/
 tests/create_file_dll/Cargo.lock
-cmd/implants/eldritch/target/
+cmd/implants/lib/eldritch/target/
 cmd/implants/target/
 cmd/implants/Cargo.lock
 

docs/_docs/dev-guide/eldritch.md

@@ -49,7 +49,7 @@ The <b>module.function</b> describe your function and edge cases.
 
 ```
 #### Eldritch definition
-`implants/Eldritch/src/module.rs`
+`implants/lib/eldritch/src/module.rs`
 Add a function definition here, where `module.rs` is the name of the module you selected above. This is how the Eldritch language is made aware that your function exists.
 
 Add the import for your functions implementation at the top, try to keep these in alphabetical order for readability.
@@ -76,7 +76,7 @@ You may notice that some functions follow the pattern:
 This pattern is only used for none type returns since we're returning Starlark None. Returning like this in the module file is more streamlined than having each module return a special starlark type.
 
 ### Eldritch Implementation
-`implants/Eldritch/src/module/function_impl.rs`
+`implants/lib/eldritch/src/module/function_impl.rs`
 Add your function implementation here, where `/module/` is the name of the module you selected above and `/function_impl.rs` is the name of your function with `_impl.rs` appended after it. This should match what's been put in the module file.
 This file will contain the actual implementation, helper functions, and unit tests for your function.
 

implants/Cargo.toml

@@ -1,11 +1,56 @@
 [workspace]
 members = [
     "imix",
-    "eldritch",
     "golem",
-    "pkg/tavern",
+    "lib/eldritch",
+    "lib/tavern",
 ]
 
+[workspace.dependencies]
+allocative = { git = "https://github.com/facebookexperimental/starlark-rust", rev = "acf638430a00ca3855862e8c669670e1adaa42aa" }
+allocative_derive = { git = "https://github.com/facebookexperimental/starlark-rust", rev = "acf638430a00ca3855862e8c669670e1adaa42aa" }
+anyhow = "1.0.65"
+assert_cmd = "2.0.6"
+async-recursion = "1.0.0"
+async-trait = "0.1.68"
+chrono = "0.4.24"
+clap = "3.2.23"
+default-net = "0.13.1"
+derive_more = "0.99.17"
+eldritch = { path = "./lib/eldritch" }
+eval = "0.4.3"
+flate2 = "1.0.24"
+gazebo = "0.8.1"
+graphql_client = "0.12.0"
+httptest = "0.15.4"
+itertools = "0.10"
+lsp-types = "0.93.0"
+nix = "0.26.1"
+openssl = "0.10"
+predicates = "2.1"
+rand = "0.8.5"
+regex = "1.5.5"
+reqwest = "0.11.4"
+rust-embed = "6.6.0"
+serde = "1.0"
+serde_json = "1.0.87"
+sha256 = "1.0.3"
+starlark = { git = "https://github.com/facebookexperimental/starlark-rust", rev = "acf638430a00ca3855862e8c669670e1adaa42aa" }
+structopt = "0.3.23"
+sys-info = "0.9.1"
+sysinfo = "0.28.4"
+tar = "0.4.38"
+tavern = { path = "./lib/tavern" }
+tempfile = "3.3.0"
+tera = "1.17.1"
+thiserror = "1.0.30"
+tokio = "1.19.1"
+tokio-stream = "0.1.9"
+tokio-test = "*"
+uuid = "1.3.0"
+whoami = "1.3.0"
+windows-sys = "0.45.0"
+
 [profile.release]
 strip = true  # Automatically strip symbols from the binary.
 opt-level = "z"  # Optimize for size.

implants/golem/Cargo.toml

@@ -4,21 +4,21 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-eldritch = { path = "../eldritch" }
-tokio = { version = "1.19.1" , features = ["macros"] }
-clap = { version = "3.2.23" }
-starlark = { git = "https://github.com/facebookexperimental/starlark-rust", rev = "acf638430a00ca3855862e8c669670e1adaa42aa" }
-anyhow = "1.0.65"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0.87"
-structopt = "0.3.23"
-gazebo = "0.8.1"
-itertools = "0.10"
-thiserror = "1.0.30"
-lsp-types = "0.93.0"
-rust-embed = { version = "6.6.0" }
+eldritch = { workspace = true }
+tokio = { workspace = true, features = ["macros"] }
+clap = { workspace = true }
+starlark = { workspace = true }
+anyhow = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+structopt = { workspace = true }
+gazebo = { workspace = true }
+itertools = { workspace = true }
+thiserror = { workspace = true }
+lsp-types = { workspace = true }
+rust-embed = { workspace = true }
 
 [dev-dependencies]
-assert_cmd = "2.0.6"
-predicates = "2.1"
-tempfile = "3.3.0"
+assert_cmd = { workspace = true }
+predicates = { workspace = true}
+tempfile = { workspace = true }

implants/golem/tests/cli.rs

@@ -30,7 +30,7 @@ fn test_golem_main_syntax_fail() -> anyhow::Result<()> {
     cmd.arg("../../tests/golem_cli_test/syntax_fail.tome");
     cmd.assert()
         .failure()
-        .stderr(predicate::str::contains("[TASK ERROR] ../../tests/golem_cli_test/syntax_fail.tome: error: Parse error: unexpected string literal 'win' here"));
+        .stderr(predicate::str::contains("[TASK ERROR] ../../tests/golem_cli_test/syntax_fail.tome: [eldritch] Unable to parse eldritch tome: error: Parse error: unexpected string literal \'win\' here"));
 
     Ok(())
 }

implants/imix/Cargo.toml

@@ -4,22 +4,22 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-eldritch = { path = "../eldritch" }
-clap = { version = "3.0" }
-rand = "0.8.5"
-openssl = { version = "0.10", features = ["vendored"] }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-reqwest = { version = "0.11.4" , features = ["blocking", "stream", "json"] }
-tokio = { version = "1", features = ["full"] }
-anyhow = "1.0.65"
-chrono = { version = "0.4.23" , features = ["serde"] }
-whoami = "1.3.0"
-uuid = { version = "1.3.0", features = ["v4","fast-rng"] }
-default-net = "0.13.1"
-sys-info = "0.9.1"
-tavern = { path = "../pkg/tavern" }
+anyhow = { workspace = true }
+chrono = { workspace = true , features = ["serde"] }
+clap = { workspace = true }
+default-net = { workspace = true }
+eldritch = { workspace = true }
+openssl = { workspace = true, features = ["vendored"] }
+rand = {workspace = true}
+reqwest = { workspace = true, features = ["blocking", "stream", "json"] }
+serde = { workspace = true, features = ["derive"] }
+serde_json = {workspace = true}
+sys-info = { workspace = true }
+tavern = { workspace = true }
+tokio = { workspace = true, features = ["full"] }
+uuid = { workspace = true, features = ["v4","fast-rng"] }
+whoami = { workspace = true }
 
 [dev-dependencies]
-httptest = "0.15.4"
-tempfile = "3.3.0"
+httptest = { workspace = true }
+tempfile = { workspace = true }

implants/imix/src/main.rs

@@ -329,22 +329,26 @@ async fn main_loop(config_path: String, run_once: bool) -> Result<()> {
                 res.push(new_res_line);
                 // Send task response
             }
-            let task_response_exec_finished_at = match exec_future.1.future_join_handle.is_finished() {
+            let task_is_finished = exec_future.1.future_join_handle.is_finished();
+            let task_response_exec_finished_at = match task_is_finished {
                 true => Some(Utc::now()),
                 false => None,
             };
-            let task_response = SubmitTaskResultInput {
-                task_id: exec_future.1.graphql_task.id.clone(),
-                exec_started_at: exec_future.1.start_time,
-                exec_finished_at: task_response_exec_finished_at,
-                output: res.join("\n"),
-                error: None,
-            };
-            let res = tavern_client.submit_task_result(task_response).await;
-            let _submit_task_result = match res {
-                Ok(local_val) => local_val,
-                Err(local_err) => if debug { println!("Failed to submit task resluts:\n{}", local_err.to_string()) },
-            };
+            // If the task is finished or there's new data send a task_result.
+            if task_is_finished || res.len() > 0 {
+                let task_response = SubmitTaskResultInput {
+                    task_id: exec_future.1.graphql_task.id.clone(),
+                    exec_started_at: exec_future.1.start_time,
+                    exec_finished_at: task_response_exec_finished_at,
+                    output: res.join("\n"),
+                    error: None,
+                };
+                let res = tavern_client.submit_task_result(task_response).await;
+                let _submit_task_result = match res {
+                    Ok(local_val) => local_val,
+                    Err(local_err) => if debug { println!("Failed to submit task resluts:\n{}", local_err.to_string()) },
+                };
+            }
 
             // Only re-insert the runnine exec futures
             if !exec_future.1.future_join_handle.is_finished() {
@@ -380,6 +384,7 @@ pub fn main() -> Result<(), imix::Error> {
 
 
     let runtime = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(128)
         .enable_all()
         .build()
         .unwrap();

implants/lib/README.md

@@ -0,0 +1,3 @@
+# Overview
+
+Code shared by multiple implants can be defined in the `lib` folder.

implants/lib/eldritch/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "eldritch"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+allocative = { workspace = true }
+allocative_derive = { workspace = true }
+anyhow = { workspace = true }
+async-recursion = { workspace = true }
+derive_more = { workspace = true }
+eval = { workspace = true }
+flate2 = { workspace = true }
+gazebo = { workspace = true }
+nix = { workspace = true }
+regex = { workspace = true }
+reqwest = { workspace = true , features = ["blocking", "stream"] }
+rust-embed = { workspace = true }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { workspace = true }
+sha256 = { workspace = true }
+starlark = { workspace = true }
+sysinfo = { workspace = true }
+tar = { workspace = true }
+tempfile = { workspace = true }
+tera = { workspace = true }
+tokio = { workspace = true , features = ["macros", "rt-multi-thread"] }
+tokio-stream = { workspace = true }
+windows-sys = { workspace = true, features = [
+    "Win32_Foundation",
+    "Win32_System_LibraryLoader",
+    "Win32_System_Threading",
+    "Win32_System_Memory",
+    "Win32_System_Diagnostics_Debug",
+    "Win32_Security",
+]}
+
+[dev-dependencies]
+httptest = { workspace = true }

implants/eldritch/README.md → implants/lib/eldritch/README.md

@@ -1,10 +1,10 @@
 # Build 
 ```
-cd ./implants/eldritch/
+cd ./implants/lib/eldritch/
 cargo build
 ```
 # Test 
 ```
-cd ./implants/eldritch/
+cd ./implants/lib/eldritch/
 cargo test
 ```

implants/eldritch/build.rs → implants/lib/eldritch/build.rs

@@ -9,7 +9,7 @@ fn build_tests_create_file_dll(){
 
     // Get the path of the create_file_dll workspace member
     let cargo_root = env!("CARGO_MANIFEST_DIR");
-    let relative_path_to_test_dll = "..\\..\\tests\\create_file_dll\\";
+    let relative_path_to_test_dll = "..\\..\\..\\tests\\create_file_dll\\";
     let test_dll_path = Path::new(cargo_root).join(relative_path_to_test_dll);
     assert!(test_dll_path.is_dir());
 
@@ -25,7 +25,7 @@ fn build_tests_create_file_dll(){
         .filter_map(|line| line.ok())
         .for_each(|line| println!("cargo dll build: {}", line));
 
-    let relative_path_to_test_dll_file = "..\\..\\tests\\create_file_dll\\target\\debug\\create_file_dll.dll";
+    let relative_path_to_test_dll_file = "..\\..\\..\\tests\\create_file_dll\\target\\debug\\create_file_dll.dll";
     let test_dll_path = Path::new(cargo_root).join(relative_path_to_test_dll_file);
     assert!(test_dll_path.is_file());
 }

implants/eldritch/src/assets.rs → implants/lib/eldritch/src/assets.rs

@@ -14,12 +14,12 @@ use rust_embed::RustEmbed;
 
 #[cfg(debug_assertions)]
 #[derive(RustEmbed)]
-#[folder = "../../tests/embedded_files_test"]
+#[folder = "../../../tests/embedded_files_test"]
 pub struct Asset;
 
 #[cfg(not(debug_assertions))]
 #[derive(RustEmbed)]
-#[folder = "../../implants/golem/embed_files_golem_prod"]
+#[folder = "../../../implants/golem/embed_files_golem_prod"]
 pub struct Asset;