update infra

.flox/env/manifest.lock

@@ -12,9 +12,6 @@
       "pulumi-python": {
         "pkg-path": "pulumiPackages.pulumi-python"
       },
-      "pulumictl": {
-        "pkg-path": "pulumictl"
-      },
       "ruff": {
         "pkg-path": "ruff"
       },
@@ -398,126 +395,6 @@
       "group": "toplevel",
       "priority": 5
     },
-    {
-      "attr_path": "pulumictl",
-      "broken": false,
-      "derivation": "/nix/store/kx43jzcfslw28byvs6h5ngsgl432pvvv-pulumictl-0.0.49.drv",
-      "description": "Swiss Army Knife for Pulumi Development",
-      "install_id": "pulumictl",
-      "license": "Apache-2.0",
-      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
-      "name": "pulumictl-0.0.49",
-      "pname": "pulumictl",
-      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
-      "rev_count": 793735,
-      "rev_date": "2025-05-04T03:14:55Z",
-      "scrape_date": "2025-05-05T04:19:37.687142Z",
-      "stabilities": [
-        "staging",
-        "unstable"
-      ],
-      "unfree": false,
-      "version": "0.0.49",
-      "outputs_to_install": [
-        "out"
-      ],
-      "outputs": {
-        "out": "/nix/store/ny69c9bfkf4w179240ch45injfb2ajqr-pulumictl-0.0.49"
-      },
-      "system": "aarch64-darwin",
-      "group": "toplevel",
-      "priority": 5
-    },
-    {
-      "attr_path": "pulumictl",
-      "broken": false,
-      "derivation": "/nix/store/www9nfncvv7l339n8dks22x5vs5lz1mk-pulumictl-0.0.49.drv",
-      "description": "Swiss Army Knife for Pulumi Development",
-      "install_id": "pulumictl",
-      "license": "Apache-2.0",
-      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
-      "name": "pulumictl-0.0.49",
-      "pname": "pulumictl",
-      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
-      "rev_count": 793735,
-      "rev_date": "2025-05-04T03:14:55Z",
-      "scrape_date": "2025-05-05T04:37:42.118866Z",
-      "stabilities": [
-        "staging",
-        "unstable"
-      ],
-      "unfree": false,
-      "version": "0.0.49",
-      "outputs_to_install": [
-        "out"
-      ],
-      "outputs": {
-        "out": "/nix/store/xpdh5dijdki4cngh7k7n4rg84i6c28zs-pulumictl-0.0.49"
-      },
-      "system": "aarch64-linux",
-      "group": "toplevel",
-      "priority": 5
-    },
-    {
-      "attr_path": "pulumictl",
-      "broken": false,
-      "derivation": "/nix/store/17wf5x1kk3v5ch5npwhamnix629y07wg-pulumictl-0.0.49.drv",
-      "description": "Swiss Army Knife for Pulumi Development",
-      "install_id": "pulumictl",
-      "license": "Apache-2.0",
-      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
-      "name": "pulumictl-0.0.49",
-      "pname": "pulumictl",
-      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
-      "rev_count": 793735,
-      "rev_date": "2025-05-04T03:14:55Z",
-      "scrape_date": "2025-05-05T04:54:38.447587Z",
-      "stabilities": [
-        "staging",
-        "unstable"
-      ],
-      "unfree": false,
-      "version": "0.0.49",
-      "outputs_to_install": [
-        "out"
-      ],
-      "outputs": {
-        "out": "/nix/store/6wmig1w7f3vmfrlyg2qzv21bvacj3as8-pulumictl-0.0.49"
-      },
-      "system": "x86_64-darwin",
-      "group": "toplevel",
-      "priority": 5
-    },
-    {
-      "attr_path": "pulumictl",
-      "broken": false,
-      "derivation": "/nix/store/ib7hqxg7xdf5kyh78jqggzdcs97q1224-pulumictl-0.0.49.drv",
-      "description": "Swiss Army Knife for Pulumi Development",
-      "install_id": "pulumictl",
-      "license": "Apache-2.0",
-      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
-      "name": "pulumictl-0.0.49",
-      "pname": "pulumictl",
-      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
-      "rev_count": 793735,
-      "rev_date": "2025-05-04T03:14:55Z",
-      "scrape_date": "2025-05-05T05:16:19.858098Z",
-      "stabilities": [
-        "staging",
-        "unstable"
-      ],
-      "unfree": false,
-      "version": "0.0.49",
-      "outputs_to_install": [
-        "out"
-      ],
-      "outputs": {
-        "out": "/nix/store/rmh9mjkxijxcc7cvjhsqc9657fbw0yyg-pulumictl-0.0.49"
-      },
-      "system": "x86_64-linux",
-      "group": "toplevel",
-      "priority": 5
-    },
     {
       "attr_path": "ruff",
       "broken": false,

application/datamanager/features/environment.py

@@ -4,5 +4,4 @@
 
 
 def before_all(context: Context) -> None:
-    """Set up test environment."""
     context.base_url = os.environ.get("BASE_URL", "http://datamanager:8080")

application/datamanager/pyproject.toml

@@ -22,3 +22,8 @@ packages = ["datamanager"]
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "behave>=1.2.6",
+]

application/datamanager/src/datamanager/config.py

@@ -19,7 +19,8 @@ class Bucket(BaseModel):
     @computed_field
     def daily_bars_path(self) -> str:
         if self.name is None:
-            raise ValueError("DATA_BUCKET environment variable is required")
+            msg = "DATA_BUCKET environment variable is required"
+            raise ValueError(msg)
         return f"gs://{self.name}/equity/bars/"
 
 

application/datamanager/src/datamanager/main.py

@@ -47,7 +47,7 @@ def bars_query(*, bucket: str, start_date: date, end_date: date) -> str:
 
 
 @asynccontextmanager
-async def lifespan(app: FastAPI) -> AsyncGenerator[None]:
+async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
     app.state.settings = Settings()
     app.state.bucket = storage.Client(os.getenv("GCP_PROJECT")).bucket(
         app.state.settings.gcp.bucket.name,

application/positionmanager/pyproject.toml

@@ -13,6 +13,7 @@ dependencies = [
     "pandas>=2.1.0",
     "pyportfolioopt>=1.5.6",
     "ecos>=2.0.14",
+    "prometheus-fastapi-instrumentator>=7.1.0",
 ]
 
 [tool.hatch.build.targets.wheel]

application/positionmanager/src/positionmanager/clients.py

@@ -13,8 +13,6 @@ class AlpacaClient:
     def __init__(
         self,
         *,
-        api_key: str | None = "",
-        api_secret: str | None = "",
         api_key: str | None = None,
         api_secret: str | None = None,
         paper: bool = True,
@@ -23,14 +21,17 @@ def __init__(
             msg = "Alpaca API key and secret are required"
             raise ValueError(msg)
 
-        self.trading_client = TradingClient(api_key, api_secret, paper=paper)
+        self.trading_client: TradingClient = TradingClient(
+            api_key, api_secret, paper=paper
+        )
 
     def get_cash_balance(self) -> Money:
         account = self.trading_client.get_account()
         cash_balance = getattr(account, "cash", None)
 
         if cash_balance is None:
-            raise ValueError("Cash balance is not available")
+            msg = "Cash balance is not available"
+            raise ValueError(msg)
 
         return Money.from_float(float(cash_balance))
 

application/positionmanager/src/positionmanager/main.py

@@ -1,11 +1,6 @@
 import os
-from datetime import datetime, timedelta, timezone
-import polars as pl
-from typing import Dict, Any
-from .models import Money, DateRange, PredictionPayload
-from .clients import AlpacaClient, DataClient
-from .portfolio import PortfolioOptimizer
-from prometheus_fastapi_instrumentator import Instrumentator
+from datetime import UTC, datetime, timedelta
+from typing import Any
 
 import polars as pl
 import requests

application/predictionengine/compose.yaml

@@ -0,0 +1,20 @@
+name: predictionengine integration tests
+
+services:
+  predictionengine:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - 8080:8080
+    environment:
+      - DATAMANAGER_BASE_URL=${DATAMANAGER_BASE_URL}
+    volumes:
+      - ./:/app/predictionengine
+      - ~/.config/gcloud/application_default_credentials.json:/root/.config/gcloud/application_default_credentials.json:ro
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://0.0.0.0:8080/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 1s

application/predictionengine/pyproject.toml

@@ -10,6 +10,8 @@ dependencies = [
     "polars>=1.29.0",
     "category-encoders>=2.8.1",
     "requests>=2.31.0",
+    "prometheus-fastapi-instrumentator>=7.1.0",
+    "loguru>=0.7.3",
 ]
 
 [tool.hatch.build.targets.wheel]

application/predictionengine/src/predictionengine/dataset.py

@@ -1,8 +1,9 @@
-from typing import Dict, List, Any, Tuple, Generator
-from tinygrad.tensor import Tensor
+from collections.abc import Generator
+from typing import Any
+
 import polars as pl
 from category_encoders import OrdinalEncoder
-
+from tinygrad.tensor import Tensor
 
 continuous_variable_columns = [
     "open_price",
@@ -20,13 +21,13 @@ def __init__(
         batch_size: int,
         sequence_length: int,
         sample_count: int,
-        scalers: Dict[str, Dict[str, Tensor]] = {},
+        scalers: dict[str, dict[str, Tensor]] | None = None,
     ) -> None:
-        self.batch_size = batch_size
-        self.sequence_length = sequence_length
-        self.sample_count = sample_count
-        self.scalers = scalers if scalers is not None else {}
-        self.preprocessors: Dict[str, Any] = {}
+        self.batch_size: int = batch_size
+        self.sequence_length: int = sequence_length
+        self.sample_count: int = sample_count
+        self.scalers: dict[str, dict[str, Tensor]] = scalers or {}
+        self.preprocessors: dict[str, Any] = {}
 
     def __len__(self) -> int:
         return (self.sample_count + self.batch_size - 1) // self.batch_size
@@ -106,7 +107,7 @@ def _encode_tickers(self, data: pl.DataFrame) -> pl.DataFrame:
 
     def _compute_scalers(self, data: pl.DataFrame) -> None:
         if len(self.scalers) == 0:
-            self.scalers: Dict[str, Dict[str, Tensor]] = {}
+            self.scalers: dict[str, dict[str, Tensor]] = {}
             for ticker_key, group in data.group_by("ticker"):
                 ticker = ticker_key[0]
                 means = group[continuous_variable_columns].mean()
@@ -118,7 +119,7 @@ def _compute_scalers(self, data: pl.DataFrame) -> None:
                 }
 
     def _scale_data(self, data: pl.DataFrame) -> Tensor:
-        groups: List[Tensor] = []
+        groups: list[Tensor] = []
         for ticker_key, group in data.group_by("ticker"):
             ticker = ticker_key[0]
             means = self.scalers[str(ticker)]["means"]
@@ -133,7 +134,8 @@ def _scale_data(self, data: pl.DataFrame) -> Tensor:
             groups.append(combined_group)
 
         if not groups:
-            raise ValueError("No data available after preprocessing")
+            msg = "No data available after preprocessing"
+            raise ValueError(msg)
 
         output_data = Tensor.empty(groups[0].shape)
         return output_data.cat(*groups, dim=0)
@@ -150,9 +152,10 @@ def load_data(self, data: pl.DataFrame) -> None:
         self._compute_scalers(data)
         self.data = self._scale_data(data)
 
-    def get_preprocessors(self) -> Dict[str, Any]:
+    def get_preprocessors(self) -> dict[str, Any]:
         if not self.preprocessors:
-            raise ValueError("Preprocessors have not been initialized.")
+            msg = "Preprocessors have not been initialized."
+            raise ValueError(msg)
 
         means_by_ticker = {
             ticker: values["means"] for ticker, values in self.scalers.items()
@@ -169,7 +172,7 @@ def get_preprocessors(self) -> Dict[str, Any]:
             "indices": self.preprocessors["indices"],
         }
 
-    def batches(self) -> Generator[Tuple[Tensor, Tensor, Tensor], None, None]:
+    def batches(self) -> Generator[tuple[Tensor, Tensor, Tensor], None, None]:
         close_price_idx = self.preprocessors["indices"]["close_price"]
 
         for i in range(0, self.sample_count, self.batch_size):
@@ -193,9 +196,8 @@ def batches(self) -> Generator[Tuple[Tensor, Tensor, Tensor], None, None]:
             ]
 
             if not batch_tensors:
-                raise ValueError(
-                    "Cannot stack empty batch tensors (batch_size must be ≥ 1)"
-                )
+                msg = "Cannot stack empty batch tensors (batch_size must be ≥ 1)"
+                raise ValueError(msg)
             if len(batch_tensors) == 1:
                 historical_features = batch_tensors[0].unsqueeze(0)
             else:

application/predictionengine/src/predictionengine/gated_residual_network.py

@@ -1,15 +1,15 @@
 from typing import cast
+
+from tinygrad.nn import LayerNorm, Linear
 from tinygrad.tensor import Tensor
-from tinygrad.nn import Linear, LayerNorm
-from typing import Optional
 
 
 class GatedResidualNetwork:
     def __init__(
         self,
         input_size: int,
         hidden_size: int,
-        output_size: Optional[int] = None,
+        output_size: int | None = None,
     ) -> None:
         output_size = output_size if output_size is not None else input_size
 
@@ -30,18 +30,18 @@ def __init__(
 
     def forward(
         self,
-        input: Tensor,
+        input_: Tensor,
     ) -> Tensor:
-        hidden_state = self.dense_input(input).relu()
+        hidden_state = self.dense_input(input_).relu()
 
         output_state = self.dense_output(hidden_state)
 
         gate_state = self.gate(hidden_state).sigmoid()
 
         if self.residual_projection is not None:
-            residual = self.residual_projection(input)
+            residual = self.residual_projection(input_)
         else:
-            residual = input
+            residual = input_
 
-        gated_output = cast(Tensor, gate_state * output_state + residual)
+        gated_output = cast("Tensor", gate_state * output_state + residual)
         return self.layer_normalizer(gated_output)