fetch data back from s3

applications/datamanager/.claude/settings.local.json

@@ -12,7 +12,9 @@
       "Bash(kill:*)",
       "Bash(RUST_LOG=info cargo run)",
       "Bash(pkill:*)",
-      "Bash(RUST_LOG=debug cargo run)"
+      "Bash(RUST_LOG=debug cargo run)",
+      "Bash(brew install:*)",
+      "Bash(export:*)"
     ],
     "deny": [],
     "defaultMode": "acceptEdits"

applications/datamanager/Cargo.toml

@@ -1,7 +1,15 @@
 [package]
 name = "datamanager"
 version = "0.1.0"
-edition = "2024"
+edition = "2021"
+
+[lib]
+name = "datamanager"
+path = "src/lib.rs"
+
+[[bin]]
+name = "datamanager"
+path = "src/main.rs"
 
 [dependencies]
 axum = "0.8.4"
@@ -17,4 +25,13 @@ tracing = "0.1.41"
 tracing-subscriber = { version = "0.3.20", features = ["env-filter", "fmt"] }
 aws-config = "1.5.8"
 aws-sdk-s3 = "1.48.0"
+aws-credential-types = "1.2.6"
 duckdb = { version = "1.0", features = ["r2d2", "chrono"] }
+
+[dev-dependencies]
+tokio-test = "0.4"
+serde_json = "1.0"
+tower = { version = "0.5", features = ["util"] }
+hyper = { version = "1.0", features = ["full"] }
+http-body-util = "0.1"
+reqwest = { version = "0.12", features = ["json"] }

applications/datamanager/src/lib.rs

@@ -0,0 +1,76 @@
+use aws_sdk_s3::Client as S3Client;
+use axum::{routing::get, Router};
+use reqwest::Client;
+use tower_http::trace::TraceLayer;
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
+
+pub mod routes;
+use routes::equity;
+use routes::health;
+
+#[derive(Clone)]
+pub struct AlpacaSecrets {
+    pub base: String,
+    pub key: String,
+    pub secret: String,
+}
+
+#[derive(Clone)]
+pub struct PolygonSecrets {
+    pub base: String,
+    pub key: String,
+}
+
+#[derive(Clone)]
+pub struct AppState {
+    pub client: Client,
+    pub polygon: PolygonSecrets,
+    pub alpaca: AlpacaSecrets,
+    pub s3_client: S3Client,
+    pub bucket_name: String,
+}
+
+impl AppState {
+    pub async fn from_env() -> Self {
+        let client = Client::builder()
+            .timeout(std::time::Duration::from_secs(10))
+            .build()
+            .unwrap();
+
+        let config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await;
+        let s3_client = S3Client::new(&config);
+        let bucket_name =
+            std::env::var("S3_BUCKET_NAME").unwrap_or("pocketsizefund-ml-data".to_string());
+
+        Self {
+            client,
+            polygon: PolygonSecrets {
+                base: std::env::var("POLYGON_BASE_URL")
+                    .unwrap_or("https://api.polygon.io".to_string()),
+                key: std::env::var("POLYGON_API_KEY")
+                    .expect("POLYGON_API_KEY must be set in environment"),
+            },
+            alpaca: AlpacaSecrets {
+                base: std::env::var("ALPACA_BASE_URL")
+                    .unwrap_or("https://data.alpaca.markets".to_string()),
+                key: std::env::var("ALPACA_API_KEY")
+                    .expect("ALPACA_API_KEY must be set in environment"),
+                secret: std::env::var("ALPACA_API_SECRET")
+                    .expect("ALPACA_API_SECRET must be set in environment"),
+            },
+            s3_client,
+            bucket_name,
+        }
+    }
+}
+
+pub async fn create_app() -> Router<AppState> {
+    let state = AppState::from_env().await;
+
+    Router::<AppState>::new()
+        .route("/health", get(health::check))
+        .merge(equity::router())
+        .with_state(state)
+        .layer(TraceLayer::new_for_http())
+}
+

applications/datamanager/src/main.rs

@@ -1,68 +1,5 @@
-use axum::{Router, routing::get};
-use reqwest::Client;
-use tower_http::trace::TraceLayer;
+use datamanager::create_app;
 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
-use aws_sdk_s3::Client as S3Client;
-
-mod routes;
-use routes::equity;
-use routes::health;
-
-#[derive(Clone)]
-struct AlpacaSecrets {
-    base: String,
-    key: String,
-    secret: String,
-}
-
-#[derive(Clone)]
-struct PolygonSecrets {
-    base: String,
-    key: String,
-}
-
-#[derive(Clone)]
-struct AppState {
-    client: Client,
-    polygon: PolygonSecrets,
-    alpaca: AlpacaSecrets,
-    s3_client: S3Client,
-    bucket_name: String,
-}
-
-impl AppState {
-    async fn from_env() -> Self {
-        let client = Client::builder()
-            .timeout(std::time::Duration::from_secs(10))
-            .build()
-            .unwrap();
-
-        let config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await;
-        let s3_client = S3Client::new(&config);
-        let bucket_name = std::env::var("S3_BUCKET_NAME")
-            .unwrap_or("pocketsizefund-ml-data".to_string());
-
-        Self {
-            client,
-            polygon: PolygonSecrets {
-                base: std::env::var("POLYGON_BASE_URL")
-                    .unwrap_or("https://api.polygon.io".to_string()),
-                key: std::env::var("POLYGON_API_KEY")
-                    .expect("POLYGON_API_KEY must be set in environment"),
-            },
-            alpaca: AlpacaSecrets {
-                base: std::env::var("ALPACA_BASE_URL")
-                    .unwrap_or("https://data.alpaca.markets".to_string()),
-                key: std::env::var("ALPACA_API_KEY")
-                    .expect("ALPACA_API_KEY must be set in environment"),
-                secret: std::env::var("ALPACA_API_SECRET")
-                    .expect("ALPACA_API_SECRET must be set in environment"),
-            },
-            s3_client,
-            bucket_name,
-        }
-    }
-}
 
 #[tokio::main]
 async fn main() {
@@ -74,14 +11,7 @@ async fn main() {
         .with(tracing_subscriber::fmt::layer())
         .init();
 
-    let state = AppState::from_env().await;
-
-    let app = Router::new()
-        .route("/health", get(health::check))
-        .merge(equity::router())
-        .with_state(state)
-        .layer(TraceLayer::new_for_http());
-
+    let app = create_app().await;
     let listener = tokio::net::TcpListener::bind("0.0.0.0:9000").await.unwrap();
 
     axum::serve(listener, app).await.unwrap();