Codebase reorganization and cleanup

.github/workflows/launch_infrastructure.yaml

@@ -24,12 +24,12 @@ jobs:
     strategy:
       matrix:
         include:
-          - service: datamanager
-            paths: applications/datamanager/**
-          - service: portfoliomanager
-            paths: applications/portfoliomanager/**
-          - service: equitypricemodel
-            paths: applications/equitypricemodel/**
+          - service: data_manager
+            paths: applications/data_manager/**
+          - service: portfolio_manager
+            paths: applications/portfolio_manager/**
+          - service: ensemble_manager
+            paths: applications/ensemble_manager/**
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

CLAUDE.md

@@ -48,6 +48,7 @@ This is a collection of guidelines and references.
 - `applications/` folder contains deployable services and training workflows
 - `libraries/` folder contains shared code resources
 - `infrastructure/` folder contains Pulumi infrastructure as code
+- `models/` folder contains model definitions and training code
 - See `README.md` "Principles" section for developer philosophy
 - If something goes wrong during a task, stop immediately and re-plan rather than continuing
 - Use subagents to keep main context window clean and offload research, exploration, and analysis work

Cargo.toml

@@ -1,3 +1,3 @@
 [workspace]
 resolver = "2"
-members = ["applications/datamanager"]
+members = ["applications/data_manager"]

applications/datamanager/Cargo.toml → applications/data_manager/Cargo.toml

@@ -1,14 +1,14 @@
 [package]
-name = "datamanager"
+name = "data_manager"
 version = "0.0.1"
 edition = "2021"
 
 [lib]
-name = "datamanager"
+name = "data_manager"
 path = "src/lib.rs"
 
 [[bin]]
-name = "datamanager"
+name = "data_manager"
 path = "src/main.rs"
 
 [dependencies]

applications/datamanager/Dockerfile → applications/data_manager/Dockerfile

@@ -31,7 +31,7 @@ FROM chef AS planner
 
 COPY Cargo.toml Cargo.lock ./
 
-COPY applications/datamanager/Cargo.toml ./applications/datamanager/Cargo.toml
+COPY applications/data_manager/Cargo.toml ./applications/data_manager/Cargo.toml
 
 RUN cargo chef prepare --recipe-path recipe.json
 
@@ -45,9 +45,9 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
 
 COPY Cargo.toml Cargo.lock ./
 
-COPY applications/datamanager/Cargo.toml ./applications/datamanager/Cargo.toml
+COPY applications/data_manager/Cargo.toml ./applications/data_manager/Cargo.toml
 
-COPY applications/datamanager/src/ applications/datamanager/src/
+COPY applications/data_manager/src/ applications/data_manager/src/
 
 ENV DUCKDB_LIB_DIR=/usr/local/lib
 ENV DUCKDB_INCLUDE_DIR=/usr/local/include
@@ -56,8 +56,8 @@ ENV LD_LIBRARY_PATH=/usr/local/lib
 
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/app/target \
-    cargo build --release --bin datamanager && \
-    cp /app/target/release/datamanager /tmp/datamanager
+    cargo build --release --bin data_manager && \
+    cp /app/target/release/data_manager /tmp/data_manager
 
 FROM debian:trixie-slim AS server
 
@@ -77,9 +77,8 @@ ENV SSL_CERT_DIR=/etc/ssl/certs
 
 WORKDIR /app
 
-COPY --from=builder /tmp/datamanager /usr/local/bin/datamanager
+COPY --from=builder /tmp/data_manager /usr/local/bin/data_manager
 
 EXPOSE 8080
 
-ENTRYPOINT ["/usr/local/bin/datamanager"]
-
+ENTRYPOINT ["/usr/local/bin/data_manager"]

applications/datamanager/src/main.rs → applications/data_manager/src/main.rs

@@ -1,4 +1,4 @@
-use datamanager::startup::{initialize_sentry, initialize_tracing, run_server};
+use data_manager::startup::{initialize_sentry, initialize_tracing, run_server};
 
 async fn run_with_bind_address(bind_address: &str) -> i32 {
     let _sentry_guard = initialize_sentry();
@@ -53,8 +53,8 @@ mod tests {
             std::env::set_var("MASSIVE_BASE_URL", "http://test");
             std::env::set_var("MASSIVE_API_KEY", "test-key");
             std::env::set_var("SENTRY_DSN", "");
-            std::env::set_var("ENVIRONMENT", "test");
-            std::env::set_var("RUST_LOG", "datamanager=debug,tower_http=debug");
+            std::env::set_var("FUND_ENVIRONMENT", "test");
+            std::env::set_var("RUST_LOG", "data_manager=debug,tower_http=debug");
         }
 
         let exit_code = run_with_bind_address("invalid-address").await;
@@ -66,7 +66,7 @@ mod tests {
             std::env::remove_var("MASSIVE_BASE_URL");
             std::env::remove_var("MASSIVE_API_KEY");
             std::env::remove_var("SENTRY_DSN");
-            std::env::remove_var("ENVIRONMENT");
+            std::env::remove_var("FUND_ENVIRONMENT");
             std::env::remove_var("RUST_LOG");
         }
     }

applications/datamanager/src/startup.rs → applications/data_manager/src/startup.rs

@@ -10,7 +10,7 @@ pub fn initialize_sentry() -> sentry::ClientInitGuard {
         sentry::ClientOptions {
             release: sentry::release_name!(),
             environment: Some(
-                env::var("ENVIRONMENT")
+                env::var("FUND_ENVIRONMENT")
                     .unwrap_or_else(|_| "development".to_string()) // Defaults to development so local runs don't require this
                     .into(),
             ),
@@ -45,7 +45,7 @@ pub async fn serve_app(listener: TcpListener, app: Router) -> std::io::Result<()
 }
 
 pub async fn run_server(bind_address: &str) -> std::io::Result<()> {
-    tracing::info!("Starting datamanager service");
+    tracing::info!("Starting data_manager service");
 
     let app = create_app().await;
     let listener = TcpListener::bind(bind_address).await?;
@@ -153,10 +153,10 @@ mod tests {
     #[test]
     #[serial]
     fn test_initialize_observability_functions() {
-        let _environment_guard = EnvironmentVariableGuard::set("ENVIRONMENT", "test");
+        let _environment_guard = EnvironmentVariableGuard::set("FUND_ENVIRONMENT", "test");
         let _sentry_dsn_guard = EnvironmentVariableGuard::set("SENTRY_DSN", "");
         let _rust_log_guard =
-            EnvironmentVariableGuard::set("RUST_LOG", "datamanager=debug,tower_http=debug");
+            EnvironmentVariableGuard::set("RUST_LOG", "data_manager=debug,tower_http=debug");
         let _sentry_guard = initialize_sentry();
         let _ = initialize_tracing();
         let _ = initialize_tracing();

applications/datamanager/tests/test_data.rs → applications/data_manager/tests/test_data.rs

@@ -1,7 +1,7 @@
 mod common;
 
 use common::initialize_test_tracing;
-use datamanager::data::{
+use data_manager::data::{
     create_equity_bar_dataframe, create_equity_details_dataframe, create_portfolio_dataframe,
     create_predictions_dataframe, EquityBar, Portfolio, Prediction,
 };

applications/datamanager/tests/test_errors.rs → applications/data_manager/tests/test_errors.rs

@@ -1,4 +1,4 @@
-use datamanager::errors::Error;
+use data_manager::errors::Error;
 
 #[test]
 fn test_error_display_formats_messages() {

applications/datamanager/tests/test_handlers.rs → applications/data_manager/tests/test_handlers.rs

@@ -1,6 +1,6 @@
 mod common;
 
-use datamanager::{
+use data_manager::{
     router::create_app_with_state,
     state::{MassiveSecrets, State},
 };

applications/datamanager/tests/test_state_and_health.rs → applications/data_manager/tests/test_state_and_health.rs

@@ -1,6 +1,6 @@
 mod common;
 
-use datamanager::{
+use data_manager::{
     router::create_app_with_state,
     state::{MassiveSecrets, State},
 };

applications/datamanager/tests/test_storage.rs → applications/data_manager/tests/test_storage.rs

@@ -1,7 +1,7 @@
 mod common;
 
 use chrono::{TimeZone, Utc};
-use datamanager::{
+use data_manager::{
     data::{
         create_equity_bar_dataframe, create_portfolio_dataframe, create_predictions_dataframe,
         EquityBar, Portfolio, Prediction,

applications/ensemble_manager/Dockerfile

@@ -0,0 +1,38 @@
+FROM python:3.12.10-slim AS builder
+
+COPY --from=ghcr.io/astral-sh/uv:0.7.2 /uv /bin/uv
+
+WORKDIR /app
+
+COPY pyproject.toml uv.lock ./
+
+COPY applications/ensemble_manager/ applications/ensemble_manager/
+
+COPY libraries/python/ libraries/python/
+
+COPY models/tide/ models/tide/
+
+RUN uv sync --no-dev
+
+FROM python:3.12.10-slim AS server
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends build-essential clang && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV PYTHONPATH=/app/applications/ensemble_manager/src
+
+WORKDIR /app
+
+COPY --from=ghcr.io/astral-sh/uv:0.7.2 /uv /bin/uv
+
+COPY --from=builder /app /app
+
+RUN useradd --system --uid 10001 --create-home appuser && \
+    chown -R appuser:appuser /app
+
+USER appuser
+
+EXPOSE 8080
+
+ENTRYPOINT ["uv", "run", "--package", "ensemble_manager", "uvicorn", "ensemble_manager.server:application", "--host", "0.0.0.0", "--port", "8080", "--app-dir", "applications/ensemble_manager/src"]

applications/equitypricemodel/pyproject.toml → applications/ensemble_manager/pyproject.toml

@@ -1,10 +1,11 @@
 [project]
-name = "equitypricemodel"
+name = "ensemble_manager"
 version = "0.0.1"
-description = "Equity price time-series dense encoder model architecture"
+description = "Ensemble manager service for combining model predictions"
 requires-python = "==3.12.10"
 dependencies = [
   "internal>=0.0.1",
+  "tide>=0.0.1",
   "boto3>=1.35.0",
   "fastapi>=0.115.0",
   "uvicorn>=0.34.0",
@@ -13,15 +14,14 @@ dependencies = [
   "requests>=2.32.5",
   "sentry-sdk[fastapi]>=2.0.0",
   "structlog>=25.5.0",
-  "tinygrad>=0.10.3",
-  "numpy>=1.26.4",
 ]
 
 [dependency-groups]
 dev = ["boto3-stubs[s3,ssm]>=1.38.0"]
 
 [tool.uv.sources]
 internal = { workspace = true }
+tide = { workspace = true }
 
 [tool.uv]
 package = true

applications/equitypricemodel/src/equitypricemodel/server.py → applications/ensemble_manager/src/ensemble_manager/server.py

@@ -23,15 +23,16 @@
 if TYPE_CHECKING:
     from mypy_boto3_s3 import S3Client
 
-from .equity_details_schema import equity_details_schema
+from internal.equity_details_schema import equity_details_schema
+from tide.tide_data import Data
+from tide.tide_model import Model
+
 from .predictions_schema import predictions_schema
 from .preprocess import filter_equity_bars
-from .tide_data import Data
-from .tide_model import Model
 
 sentry_sdk.init(
     dsn=os.environ.get("SENTRY_DSN"),
-    environment=os.environ.get("ENVIRONMENT", "development"),
+    environment=os.environ.get("FUND_ENVIRONMENT", "development"),
     traces_sample_rate=1.0,
     profiles_sample_rate=1.0,
     enable_tracing=True,
@@ -58,8 +59,11 @@
 
 logger = structlog.get_logger()
 
-DATAMANAGER_BASE_URL = os.getenv("FUND_DATAMANAGER_BASE_URL", "http://datamanager:8080")
-MODEL_VERSION_SSM_PARAMETER = "/fund/equitypricemodel/model_version"
+DATAMANAGER_BASE_URL = os.getenv(
+    "FUND_DATAMANAGER_BASE_URL", "http://data-manager:8080"
+)
+_environment = os.environ.get("FUND_ENVIRONMENT", "development")
+MODEL_VERSION_SSM_PARAMETER = f"/fund/{_environment}/ensemble-manager/model-version"
 
 
 def find_latest_artifact_key(

applications/equitypricemodel/tests/test_predictions_schema.py → applications/ensemble_manager/tests/test_predictions_schema.py

@@ -2,7 +2,7 @@
 
 import polars as pl
 import pytest
-from equitypricemodel.predictions_schema import predictions_schema
+from ensemble_manager.predictions_schema import predictions_schema
 from pandera.errors import SchemaError
 
 

applications/equitypricemodel/tests/test_preprocess.py → applications/ensemble_manager/tests/test_preprocess.py

@@ -1,5 +1,5 @@
 import polars as pl
-from equitypricemodel.preprocess import filter_equity_bars
+from ensemble_manager.preprocess import filter_equity_bars
 
 
 def test_filter_equity_bars_above_thresholds() -> None:

applications/equitypricemodel/tests/test_server.py → applications/ensemble_manager/tests/test_server.py

@@ -1,7 +1,7 @@
 from unittest.mock import MagicMock, patch
 
 from botocore.exceptions import ClientError
-from equitypricemodel.server import _resolve_artifact_key
+from ensemble_manager.server import _resolve_artifact_key
 
 
 def test_resolve_artifact_key_uses_latest_by_default() -> None:
@@ -18,8 +18,8 @@ def test_resolve_artifact_key_uses_latest_by_default() -> None:
     )
 
     with (
-        patch("equitypricemodel.server.boto3") as mock_boto3,
-        patch("equitypricemodel.server.find_latest_artifact_key") as mock_find,
+        patch("ensemble_manager.server.boto3") as mock_boto3,
+        patch("ensemble_manager.server.find_latest_artifact_key") as mock_find,
     ):
         mock_boto3.client.return_value = mock_ssm
         mock_find.return_value = "artifacts/model-2026/output/model.tar.gz"
@@ -40,7 +40,7 @@ def test_resolve_artifact_key_uses_ssm_version() -> None:
         "Parameter": {"Value": "equitypricemodel-trainer-2026-01-15"}
     }
 
-    with patch("equitypricemodel.server.boto3") as mock_boto3:
+    with patch("ensemble_manager.server.boto3") as mock_boto3:
         mock_boto3.client.return_value = mock_ssm
         result = _resolve_artifact_key(
             s3_client=mock_s3,
@@ -59,7 +59,7 @@ def test_resolve_artifact_key_ssm_tar_gz_value() -> None:
         "Parameter": {"Value": "custom/path/model.tar.gz"}
     }
 
-    with patch("equitypricemodel.server.boto3") as mock_boto3:
+    with patch("ensemble_manager.server.boto3") as mock_boto3:
         mock_boto3.client.return_value = mock_ssm
         result = _resolve_artifact_key(
             s3_client=mock_s3,
@@ -75,7 +75,7 @@ def test_resolve_artifact_key_explicit_tar_gz_path() -> None:
     mock_ssm = MagicMock()
     mock_ssm.get_parameter.return_value = {"Parameter": {"Value": "latest"}}
 
-    with patch("equitypricemodel.server.boto3") as mock_boto3:
+    with patch("ensemble_manager.server.boto3") as mock_boto3:
         mock_boto3.client.return_value = mock_ssm
         result = _resolve_artifact_key(
             s3_client=mock_s3,