Fix datamanager Dockerfile and improve error handling

.flox/env/manifest.lock

@@ -12,6 +12,9 @@
       "pulumi-python": {
         "pkg-path": "pulumiPackages.pulumi-python"
       },
+      "pulumictl": {
+        "pkg-path": "pulumictl"
+      },
       "ruff": {
         "pkg-path": "ruff"
       },
@@ -395,6 +398,126 @@
       "group": "toplevel",
       "priority": 5
     },
+    {
+      "attr_path": "pulumictl",
+      "broken": false,
+      "derivation": "/nix/store/kx43jzcfslw28byvs6h5ngsgl432pvvv-pulumictl-0.0.49.drv",
+      "description": "Swiss Army Knife for Pulumi Development",
+      "install_id": "pulumictl",
+      "license": "Apache-2.0",
+      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "name": "pulumictl-0.0.49",
+      "pname": "pulumictl",
+      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "rev_count": 793735,
+      "rev_date": "2025-05-04T03:14:55Z",
+      "scrape_date": "2025-05-05T04:19:37.687142Z",
+      "stabilities": [
+        "staging",
+        "unstable"
+      ],
+      "unfree": false,
+      "version": "0.0.49",
+      "outputs_to_install": [
+        "out"
+      ],
+      "outputs": {
+        "out": "/nix/store/ny69c9bfkf4w179240ch45injfb2ajqr-pulumictl-0.0.49"
+      },
+      "system": "aarch64-darwin",
+      "group": "toplevel",
+      "priority": 5
+    },
+    {
+      "attr_path": "pulumictl",
+      "broken": false,
+      "derivation": "/nix/store/www9nfncvv7l339n8dks22x5vs5lz1mk-pulumictl-0.0.49.drv",
+      "description": "Swiss Army Knife for Pulumi Development",
+      "install_id": "pulumictl",
+      "license": "Apache-2.0",
+      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "name": "pulumictl-0.0.49",
+      "pname": "pulumictl",
+      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "rev_count": 793735,
+      "rev_date": "2025-05-04T03:14:55Z",
+      "scrape_date": "2025-05-05T04:37:42.118866Z",
+      "stabilities": [
+        "staging",
+        "unstable"
+      ],
+      "unfree": false,
+      "version": "0.0.49",
+      "outputs_to_install": [
+        "out"
+      ],
+      "outputs": {
+        "out": "/nix/store/xpdh5dijdki4cngh7k7n4rg84i6c28zs-pulumictl-0.0.49"
+      },
+      "system": "aarch64-linux",
+      "group": "toplevel",
+      "priority": 5
+    },
+    {
+      "attr_path": "pulumictl",
+      "broken": false,
+      "derivation": "/nix/store/17wf5x1kk3v5ch5npwhamnix629y07wg-pulumictl-0.0.49.drv",
+      "description": "Swiss Army Knife for Pulumi Development",
+      "install_id": "pulumictl",
+      "license": "Apache-2.0",
+      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "name": "pulumictl-0.0.49",
+      "pname": "pulumictl",
+      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "rev_count": 793735,
+      "rev_date": "2025-05-04T03:14:55Z",
+      "scrape_date": "2025-05-05T04:54:38.447587Z",
+      "stabilities": [
+        "staging",
+        "unstable"
+      ],
+      "unfree": false,
+      "version": "0.0.49",
+      "outputs_to_install": [
+        "out"
+      ],
+      "outputs": {
+        "out": "/nix/store/6wmig1w7f3vmfrlyg2qzv21bvacj3as8-pulumictl-0.0.49"
+      },
+      "system": "x86_64-darwin",
+      "group": "toplevel",
+      "priority": 5
+    },
+    {
+      "attr_path": "pulumictl",
+      "broken": false,
+      "derivation": "/nix/store/ib7hqxg7xdf5kyh78jqggzdcs97q1224-pulumictl-0.0.49.drv",
+      "description": "Swiss Army Knife for Pulumi Development",
+      "install_id": "pulumictl",
+      "license": "Apache-2.0",
+      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "name": "pulumictl-0.0.49",
+      "pname": "pulumictl",
+      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "rev_count": 793735,
+      "rev_date": "2025-05-04T03:14:55Z",
+      "scrape_date": "2025-05-05T05:16:19.858098Z",
+      "stabilities": [
+        "staging",
+        "unstable"
+      ],
+      "unfree": false,
+      "version": "0.0.49",
+      "outputs_to_install": [
+        "out"
+      ],
+      "outputs": {
+        "out": "/nix/store/rmh9mjkxijxcc7cvjhsqc9657fbw0yyg-pulumictl-0.0.49"
+      },
+      "system": "x86_64-linux",
+      "group": "toplevel",
+      "priority": 5
+    },
     {
       "attr_path": "ruff",
       "broken": false,

.github/workflows/test.yaml

@@ -4,20 +4,13 @@ on:
   push
 jobs:
   python_test_and_coverage:
-    name: Run Python tests (${{ matrix.os }})
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
+    name: Run Python tests
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install Flox
         uses: flox/install-flox-action@v2
-      - name: Install Python dependencies
-        uses: flox/activate-action@v1
-        with:
-          command: mise tasks run python:install
       - name: Run unit tests with coverage
         uses: flox/activate-action@v1
         with:

.gitignore

@@ -6,6 +6,7 @@
 .python_coverage.*
 __pycache__/
 infrastructure/Pulumi.infrastructure.yaml
-
-**/.claude/settings.local.json
+.coverage*
+.coverage/
+coverage.xml
 .envrc

.mise.toml

@@ -3,7 +3,7 @@ COMPOSE_BAKE=true
 
 [tasks."python:install"]
 description = "Install Python dependencies"
-run = "uv sync --all-packages"
+run = "uv sync --all-groups"
 
 [tasks."python:format"]
 description = "Format Python code"
@@ -30,22 +30,14 @@ ruff check \
 
 [tasks."python:test"]
 description = "Run Python tests"
-run = """
-uv run coverage run \
-    --parallel-mode \
-    --omit '*/__init__.py,**/test_*.py' \
-    --data-file .python_coverage.output \
-    --module pytest
-
-uv run coverage combine \
-    --data-file .python_coverage.output
+run = "docker compose run tests"
 
-uv run coverage report \
-    --data-file .python_coverage.output
 
-uv run coverage xml \
-    --data-file .python_coverage.output \
-    -o .python_coverage.xml
+[tasks."python:test:behave"]
+description = "Run behave end-to-end tests"
+run = """
+cd application/{{arg(name="application_name")}}
+uv run behave features/
 """
 
 
@@ -62,19 +54,19 @@ description = "Build the application service"
 run = """
 TIMESTAMP=$(date +%Y%m%d) 
 docker build \
---file application/{{arg(name="service_name")}}/Dockerfile \
---tag pocketsizefund/{{arg(name="service_name")}}:latest \
---tag pocketsizefund/{{arg(name="service_name")}}:${TIMESTAMP} \
+--file application/{{arg(name="application_name")}}/Dockerfile \
+--tag pocketsizefund/{{arg(name="application_name")}}:latest \
+--tag pocketsizefund/{{arg(name="application_name")}}:${TIMESTAMP} \
 .
 """
 
-[tasks."application:service:run"]
+[tasks."application:run"]
 description = "Run the application service"
 run = """
 docker run \
 --env-file .env \
 --publish 8080:8080 \
-pocketsizefund/{{arg(name="service_name")}}:latest \
+pocketsizefund/{{arg(name="application_name")}}:latest \
 """
 
 [tasks."application:service:development"]

AGENTS.md

@@ -0,0 +1,19 @@
+# Codex Agent Instructions
+
+This project uses **mise** tasks for code quality and testing. Before committing, run the following steps:
+
+1. Install dependencies with `mise run python:install`.
+2. Format code with `mise run python:format`.
+3. Run lint checks with `mise run python:lint`.
+4. Execute tests with `mise run python:test`.
+
+## Code Style
+- Write tested, self-documenting code. Avoid comments unless the code is complex.
+
+## Pull Requests
+- Use the Pull Request template: `.github/PULL_REQUEST_TEMPLATE.md`.
+
+## GitHub Issues
+- When creating an issue, use the templates under `.github/ISSUE_TEMPLATE`.
+- Include a clear title, the reason the feature or fix is needed, and two implementation options with cost-benefit analysis.
+

Dockerfile.test

@@ -0,0 +1,9 @@
+FROM python:3.13
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
+
+ENV PYTEST_ADDOPTS="--rootdir=/tests"
+
+WORKDIR /tests
+
+COPY pyproject.toml uv.lock .
+RUN uv sync --all-groups

application/datamanager/Dockerfile

@@ -1,6 +1,8 @@
 FROM python:3.13-slim 
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
 
+ENV PYTHONPATH=/app/src
+
 RUN apt-get update && \
     apt-get install -y --no-install-recommends libgomp1 curl \
     && rm -rf /var/lib/apt/lists/*
@@ -10,8 +12,8 @@ WORKDIR /app
 COPY pyproject.toml ./
 RUN uv sync --no-dev
 
-COPY ./src/ ./app
+COPY ./src ./src
 
 EXPOSE 8080
 
-ENTRYPOINT ["uv", "run", "uvicorn", "datamanager.src.datamanager.main:application", "--host", "0.0.0.0", "--port", "8080"]
+ENTRYPOINT ["uv", "run", "uvicorn", "datamanager.main:application", "--host", "0.0.0.0", "--port", "8080", "--app-dir", "src"]

application/datamanager/features/steps/equity_bars_steps.py

@@ -60,4 +60,4 @@ def step_impl_equity_bars(context, date_str):
         expected_file = Path(f"equity_bars_{date_str}.parquet")
         assert not expected_file.exists(), (
             f"Parquet file {expected_file} still exists after deletion"
-        )
+        )

application/datamanager/features/steps/health_steps.py

@@ -1,9 +1,7 @@
 from behave import when
 import requests
 
-
 @when('I send a GET request to "{endpoint}"')
 def step_impl(context, endpoint):
-    """Send a GET request to the specified endpoint."""
     url = f"{context.api_url}{endpoint}"
     context.response = requests.get(url)

application/datamanager/pyproject.toml

@@ -2,7 +2,7 @@
 name = "datamanager"
 version = "0.1.0"
 description = "Data management service"
-requires-python = ">=3.13"
+requires-python = ">=3.12"
 dependencies = [
     "fastapi>=0.115.12",
     "uvicorn>=0.34.2",

application/datamanager/src/datamanager/bars.sql

@@ -0,0 +1 @@
+

application/datamanager/src/datamanager/config.py

@@ -1,7 +1,9 @@
 import os
 import json
 from functools import cached_property
+from datetime import date, datetime
 from pydantic import BaseModel, Field, computed_field
+from loguru import logger
 
 
 class Polygon(BaseModel):

application/datamanager/src/datamanager/main.py

@@ -165,5 +165,4 @@ async def delete_equity_bars(request: Request, summary_date: SummaryDate):
 
     logger.info(f"deleting {prefix=}")
     bucket.delete_blobs(blobs)
-
     return Response(status_code=status.HTTP_204_NO_CONTENT)

application/datamanager/src/datamanager/query.py

@@ -0,0 +1,28 @@
+from datetime import timedelta, date
+from polars import DataFrame
+import duckdb
+from pathlib import Path
+
+
+async def list_file_paths(
+    *, bucket: str, start_date: date, end_date: date
+) -> list[Path]:
+    filepaths = []
+    current = start_date
+    while current <= end_date:
+        filepaths.append(
+            f"gs://{bucket}/equity/bars/{current.strftime('%Y-%m-%d')}/data.parquet"
+        )
+        current += timedelta(days=1)
+
+    return filepaths
+
+
+async def query_bars(*, filepaths: list[Path]) -> DataFrame:
+    query = f"""
+    SELECT * FROM read_parquet({filepaths})
+    """
+    try:
+        return duckdb.sql(query).pl()
+    except duckdb.duckdb.HTTPException as e:
+    return 

application/positionmanager/Dockerfile

@@ -1,15 +1,15 @@
-FROM python:3.13-slim 
-
+FROM python:3.13
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
 
+ENV PYTHONPATH=/app/src
+
 WORKDIR /app
 
 COPY pyproject.toml ./
+RUN uv sync --no-dev
 
-COPY application/positionmanager/ ./application/positionmanager/
-
-RUN uv sync --all-packages
+COPY ./src ./src
 
 EXPOSE 8080
 
-ENTRYPOINT ["uv", "run", "uvicorn", "application.positionmanager.src.positionmanager.main:application", "--host", "0.0.0.0", "--port", "8080"]
+ENTRYPOINT ["uv", "run", "uvicorn", "positionmanager.main:application", "--host", "0.0.0.0", "--port", "8080", "--app-dir", "src"]

compose.yaml

@@ -0,0 +1,16 @@
+
+services:
+  tests:
+    build:
+      context: .
+      dockerfile: Dockerfile.test
+    volumes:
+      - .:/tests
+    command:
+      - /bin/sh
+      - -euxc
+      - |
+        uv run coverage run --parallel-mode -m pytest && \
+        uv run coverage combine && \
+        uv run coverage report && \
+        uv run coverage xml