Update Pulumi infrastructure to build Docker images

.gitignore

@@ -6,6 +6,7 @@
 .python_coverage.*
 __pycache__/
 infrastructure/Pulumi.infrastructure.yaml
-.coverage/
+.envrc
 .coverage*
-coverage.xml
+.coverage/
+coverage.xml

.mise.toml

@@ -1,3 +1,6 @@
+[env]
+COMPOSE_BAKE=true
+
 [tasks."python:install"]
 description = "Install Python dependencies"
 run = "uv sync --all-groups"
@@ -29,6 +32,7 @@ ruff check \
 description = "Run Python tests"
 run = "docker compose run tests"
 
+
 [tasks."application:service:run"]
 description = "Run the application service"
 run = """
@@ -38,12 +42,25 @@ docker run \
 pocketsizefund/{{arg(name="service_name")}}:latest \
 """
 
+[tasks."application:service:development"]
+description = "Run the application service locally with hot reloading"
+run = """
+cd application/{{arg(name="service_name")}}
+uv run uvicorn src.{{arg(name="service_name")}}.main:application --reload
+"""
+
+[tasks."application:service:test"]
+description = "Run integration tests"
+run = """
+cd application/{{arg(name="service_name")}}
+docker-compose up --build --abort-on-container-exit --remove-orphans
+"""
 
 [tasks."lint"]
 depends = ["python:lint"]
 description = "Run code quality checks"
 run = """
-yamllint .
+yamllint -d "{extends: relaxed, rules: {line-length: {max: 110}}}" . 
 """
 
 [tasks."infrastructure:up"]
@@ -52,3 +69,4 @@ run = """
 cd infrastructure
 uv run pulumi up --yes
 """
+

application/datamanager/Dockerfile.test

@@ -0,0 +1,11 @@
+FROM python:3.13-slim
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
+
+WORKDIR /tests
+
+COPY pyproject.toml .
+RUN uv sync --only-group dev
+
+COPY features /tests
+
+CMD ["uv", "run", "behave", "features/"]

application/datamanager/compose.yaml

@@ -0,0 +1,37 @@
+name: Integration Tests for datamanager service
+
+services:
+  datamanager:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - 8080:8080
+    environment:
+      - POLYGON_API_KEY=${POLYGON_API_KEY}
+      - DATA_BUCKET=${DATA_BUCKET}
+      - GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json
+      - DUCKDB_ACCESS_KEY=${DUCKDB_ACCESS_KEY}
+      - DUCKDB_SECRET=${DUCKDB_SECRET}
+    volumes:
+      - ./:/app/datamanager
+      - ~/.config/gcloud/application_default_credentials.json:/root/.config/gcloud/application_default_credentials.json:ro
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://0.0.0.0:8080/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 1s
+
+  tests:
+    build:
+      context: .
+      dockerfile: Dockerfile.test
+    volumes:
+      - ./:/app/datamanager
+    depends_on:
+      datamanager:
+        condition: service_healthy
+    environment:
+      - BASE_URL=http://datamanager:8080
+    command: ["uv", "run", "behave"]

application/datamanager/features/environment.py

@@ -0,0 +1,6 @@
+import os
+
+
+def before_all(context):
+    """Set up test environment."""
+    context.base_url = os.environ.get("BASE_URL", "http://datamanager:8080")

application/datamanager/features/equity_bars.feature

@@ -0,0 +1,24 @@
+Feature: Equity Bars Data Management
+    As a datamanager application
+    I want to fetch, store, retrieve, and delete equity bars data
+    So that I can manage market data efficiently
+
+    Background:
+        Given the datamanager API is running
+
+    Scenario Outline: Manage bucket data for <start_date> to <end_date>
+        Given I have date ranges:
+            | start_date | end_date |
+            | <start_date> | <end_date> |
+        When I send a POST request to "/equity-bars" for date range
+        Then the response status code should be 200
+        When I send a GET request to "/equity-bars" for date range
+        Then the response status code should be 200
+        When I send a DELETE request to "/equity-bars" for date "<start_date>"
+        Then the response status code should be 204
+
+        Examples: dates
+            | start_date | end_date   |
+            | 2025-05-20 | 2025-05-20 |
+
+    Scenario Outline: Skip weekends

application/datamanager/features/health.feature

@@ -0,0 +1,11 @@
+Feature: Health Check Endpoint
+    As a client
+    I want to check the health of the datamanager API
+    So that I can ensure the service is running
+
+    Background:
+        Given the datamanager API is running
+
+    Scenario: Health endpoint responds successfully
+        When I send a GET request to "/health"
+        Then the response status code should be 200

application/datamanager/features/steps/equity_bars_steps.py

@@ -0,0 +1,63 @@
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+import requests
+from behave import given, when, then
+
+
+@given("I have date ranges")
+def step_impl_date_ranges(context):
+    for row in context.table:
+        context.start_date = row["start_date"]
+        context.end_date = row["end_date"]
+
+
+@given("the datamanager API is running")
+def step_impl_api_url(context):
+    context.api_url = context.base_url
+
+
+@when('I send a POST request to "{endpoint}" for date range')
+def step_impl_post_request(context, endpoint):
+    url = f"{context.api_url}{endpoint}"
+    response = requests.post(url, json={"date": context.start_date})
+    context.response = response
+
+
+@when('I send a GET request to "{endpoint}" for date range')
+def step_imp_get_request(context, endpoint):
+    url = f"{context.api_url}{endpoint}"
+    response = requests.get(
+        url,
+        params={"start_date": context.start_date, "end_date": context.end_date},
+    )
+    context.response = response
+
+
+@then("the response status code should be {status_code}")
+def step_impl_response_status_code(context, status_code):
+    assert context.response.status_code == int(status_code), (
+        f"Expected status code {status_code}, got {context.response.status_code}"
+    )
+
+
+@when('I send a DELETE request to "{endpoint}" for date "{date_str}"')
+def step_impl(context, endpoint, date_str):
+    url = f"{context.api_url}{endpoint}"
+    response = requests.delete(url, json={"date": date_str})
+    context.response = response
+    context.test_date = date_str
+
+
+@then('the equity bars data for "{date_str}" should be deleted')
+def step_impl_equity_bars(context, date_str):
+    if os.environ.get("GCP_GCS_BUCKET"):
+        assert True, "GCS bucket deletion check would go here"
+    else:
+        expected_file = Path(f"equity_bars_{date_str}.parquet")
+        assert not expected_file.exists(), (
+            f"Parquet file {expected_file} still exists after deletion"
+        )

application/datamanager/features/steps/health_steps.py

@@ -0,0 +1,9 @@
+from behave import when
+import requests
+
+
+@when('I send a GET request to "{endpoint}"')
+def step_impl(context, endpoint):
+    """Send a GET request to the specified endpoint."""
+    url = f"{context.api_url}{endpoint}"
+    context.response = requests.get(url)

application/datamanager/mise.toml

@@ -0,0 +1,11 @@
+[tasks."test:docker:behave"]
+description = "Run behave tests with Docker Compose"
+run = """
+docker-compose up --build --abort-on-container-exit
+"""
+
+[tasks."test:docker:behave:cleanup"]
+description = "Clean up after Docker Compose tests"
+run = """
+docker-compose down -v
+"""

application/datamanager/pyproject.toml

@@ -9,6 +9,9 @@ dependencies = [
     "duckdb>=1.2.2",
     "polars>=1.29.0",
     "pyarrow>=20.0.0",
+    "google-cloud-storage>=2.16.0",
+    "loguru>=0.7.3",
+    "httpx>=0.28.1",
     "prometheus-fastapi-instrumentator>=7.1.0",
     "datamanager"
 ]
@@ -19,3 +22,9 @@ packages = ["datamanager"]
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "behave>=1.2.6",
+    "requests>=2.31.0",
+]

application/datamanager/src/datamanager/config.py

@@ -0,0 +1,43 @@
+import os
+import json
+from functools import cached_property
+from pydantic import BaseModel, Field, computed_field
+
+
+class Polygon(BaseModel):
+    api_key: str = Field(default=os.getenv("POLYGON_API_KEY"))
+    base_url: str = "https://api.polygon.io"
+    daily_bars: str = "/v2/aggs/grouped/locale/us/market/stocks/"
+
+
+class Bucket(BaseModel):
+    name: str = Field(default=os.getenv("DATA_BUCKET"))
+    project: str = Field(default=os.getenv("GCP_PROJECT"))
+
+    @computed_field
+    def daily_bars_path(self) -> str:
+        return f"gs://{self.name}/equity/bars/"
+
+
+class GCP(BaseModel):
+    bucket: Bucket = Bucket()
+    credentials_path: str = os.getenv("GOOGLE_APPLICATION_CREDENTIALS", "")
+
+    @cached_property
+    def _creds(self) -> dict:
+        with open(self.credentials_path) as f:
+            creds = json.load(f)
+        return creds
+
+    @computed_field
+    def key_id(self) -> str | None:
+        return self._creds.get("client_email")
+
+    @computed_field
+    def secret(self) -> str:
+        return json.dumps(self._creds).replace("'", "\\'")
+
+
+class Settings(BaseModel):
+    gcp: GCP = GCP()
+    polygon: Polygon = Polygon()