Cleanup Docker image handling, ECS service management, and model training resources

.github/workflows/launch_infrastructure.yaml

@@ -2,20 +2,15 @@
 name: Launch infrastructure
 run-name: Launch infrastructure
 on:
-  schedule:
-    # Runs at 12:00 UTC weekdays (7:00 AM EST / 8:00 AM EDT)
-    # Launches infrastructure 2.5 hours before market open (EST) or 1.5 hours before (EDT)
-    - cron: 0 12 * * 1-5
   push:
     branches:
       - master
-  workflow_dispatch:
 concurrency:
   group: infrastructure-deployment
   cancel-in-progress: false
 jobs:
   build_and_push_images:
-    name: Build and push ${{ matrix.service }}
+    name: Build and push ${{ matrix.application }}-${{ matrix.stage }}
     runs-on: ubuntu-latest
     environment: pulumi
     permissions:
@@ -24,12 +19,18 @@ jobs:
     strategy:
       matrix:
         include:
-          - service: data_manager
+          - application: data-manager
+            stage: server
             paths: applications/data_manager/**
-          - service: portfolio_manager
+          - application: portfolio-manager
+            stage: server
             paths: applications/portfolio_manager/**
-          - service: ensemble_manager
+          - application: ensemble-manager
+            stage: server
             paths: applications/ensemble_manager/**
+          - application: model-trainer
+            stage: server-worker
+            paths: models/**
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -46,27 +47,27 @@ jobs:
               - 'pyproject.toml'
               - 'uv.lock'
       - name: Configure AWS credentials
-        if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule'
+        if: steps.changes.outputs.service == 'true'
         uses: aws-actions/configure-aws-credentials@v5
         with:
           role-to-assume: ${{ secrets.AWS_IAM_INFRASTRUCTURE_ROLE_ARN }}
           aws-region: ${{ secrets.AWS_REGION }}
       - name: Set up Docker Buildx
-        if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule'
+        if: steps.changes.outputs.service == 'true'
         uses: docker/setup-buildx-action@v3
       - name: Install Flox
-        if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule'
+        if: steps.changes.outputs.service == 'true'
         uses: flox/install-flox-action@v2
-      - name: Build ${{ matrix.service }} image
-        if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule'
+      - name: Build ${{ matrix.application }}-${{ matrix.stage }} image
+        if: steps.changes.outputs.service == 'true'
         uses: flox/activate-action@v1
         with:
-          command: mask infrastructure images build ${{ matrix.service }} server
-      - name: Push ${{ matrix.service }} image
-        if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule'
+          command: mask infrastructure image build ${{ matrix.application }} ${{ matrix.stage }}
+      - name: Push ${{ matrix.application }}-${{ matrix.stage }} image
+        if: steps.changes.outputs.service == 'true'
         uses: flox/activate-action@v1
         with:
-          command: mask infrastructure images push ${{ matrix.service }} server
+          command: mask infrastructure image push ${{ matrix.application }} ${{ matrix.stage }}
   launch_infrastructure:
     name: Deploy with Pulumi
     needs: build_and_push_images
@@ -91,15 +92,60 @@ jobs:
           PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
         with:
           command: mask infrastructure stack up
-  trigger_data_sync:
-    name: Trigger sync data workflow
+  deploy_images:
+    name: Deploy ${{ matrix.application }}-${{ matrix.stage }}
     needs: launch_infrastructure
-    if: github.event_name == 'schedule'
     runs-on: ubuntu-latest
+    environment: pulumi
     permissions:
-      contents: write
+      id-token: write
+      contents: read
+    strategy:
+      matrix:
+        include:
+          - application: data-manager
+            stage: server
+            paths: applications/data_manager/**
+          - application: portfolio-manager
+            stage: server
+            paths: applications/portfolio_manager/**
+          - application: ensemble-manager
+            stage: server
+            paths: applications/ensemble_manager/**
+          - application: model-trainer
+            stage: server
+            paths: models/**
+          - application: model-trainer
+            stage: worker
+            paths: models/**
     steps:
-      - name: Trigger sync data workflow
-        uses: peter-evans/repository-dispatch@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Check for service changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          filters: |
+            service:
+              - '${{ matrix.paths }}'
+              - 'libraries/python/**'
+              - 'Cargo.toml'
+              - 'Cargo.lock'
+              - 'pyproject.toml'
+              - 'uv.lock'
+      - name: Configure AWS credentials
+        if: steps.changes.outputs.service == 'true'
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ secrets.AWS_IAM_INFRASTRUCTURE_ROLE_ARN }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Install Flox
+        if: steps.changes.outputs.service == 'true'
+        uses: flox/install-flox-action@v2
+      - name: Deploy ${{ matrix.application }}-${{ matrix.stage }}
+        if: steps.changes.outputs.service == 'true'
+        uses: flox/activate-action@v1
+        env:
+          PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
         with:
-          event-type: sync-data-after-scheduled-launch-infrastructure
+          command: mask infrastructure image deploy ${{ matrix.application }} ${{ matrix.stage }}

.github/workflows/teardown_infrastructure.yaml

@@ -2,10 +2,6 @@
 name: Teardown infrastructure
 run-name: Teardown infrastructure
 on:
-  schedule:
-    # Runs at 23:00 UTC weekdays (6:00 PM EST / 7:00 PM EDT)
-    # Tears down infrastructure 2 hours after market close (EST) or 3 hours after (EDT)
-    - cron: 0 23 * * 1-5
   workflow_dispatch:
 jobs:
   teardown_infrastructure:

applications/data_manager/Dockerfile

@@ -37,6 +37,8 @@ RUN cargo chef prepare --recipe-path recipe.json
 
 FROM chef AS builder
 
+ENV CARGO_BUILD_JOBS=4
+
 COPY --from=planner /app/recipe.json recipe.json
 
 RUN --mount=type=cache,target=/usr/local/cargo/registry \

applications/ensemble_manager/Dockerfile

@@ -11,6 +11,7 @@ COPY applications/ensemble_manager/ applications/ensemble_manager/
 COPY libraries/python/ libraries/python/
 
 COPY models/tide/ models/tide/
+COPY tools/ tools/
 
 RUN uv sync --no-dev
 

applications/portfolio_manager/pyproject.toml

@@ -12,6 +12,7 @@ dependencies = [
   "polars>=1.29.0",
   "requests>=2.32.5",
   "alpaca-py>=0.42.1",
+  "pytz>=2025.1",
   "sentry-sdk[fastapi]>=2.0.0",
   "structlog>=25.5.0",
   "scipy>=1.17.1",

infrastructure/Pulumi.production.yaml

@@ -7,7 +7,7 @@ config:
   fund:sagemakerExecutionRoleName: fund-sagemaker-execution-role
   fund:githubRepository: oscmcompany/fund
   fund:githubBranch: master
-  fund:monthlyBudgetLimitUsd: "25"
+  fund:monthlyBudgetLimitUsd: "250"
   fund:githubWorkflowFiles:
     - launch_infrastructure.yaml
     - teardown_infrastructure.yaml

infrastructure/__main__.py

@@ -10,12 +10,12 @@
     ensemble_manager_image_uri,
     ensemble_manager_repository,
     model_artifacts_bucket,
+    model_trainer_server_worker_image_uri,
+    model_trainer_server_worker_repository,
     portfolio_manager_image_uri,
     portfolio_manager_repository,
-    tide_trainer_image_uri,
-    tide_trainer_repository,
-    training_worker_image_uri,
-    training_worker_repository,
+    tide_runner_image_uri,
+    tide_runner_repository,
 )
 
 protocol = "https://" if acm_certificate_arn else "http://"
@@ -54,18 +54,21 @@
     pulumi.Output.unsecret(model_artifacts_bucket.bucket),
 )
 pulumi.export(
-    "aws_ecr_tide_trainer_repository",
-    tide_trainer_repository.repository_url,
+    "aws_ecr_tide_runner_repository",
+    tide_runner_repository.repository_url,
 )
-pulumi.export("aws_ecr_tide_trainer_image", tide_trainer_image_uri)
+pulumi.export("aws_ecr_tide_runner_image", tide_runner_image_uri)
 pulumi.export(
-    "aws_ecr_training_worker_repository", training_worker_repository.repository_url
+    "aws_ecr_model_trainer_server_worker_repository",
+    model_trainer_server_worker_repository.repository_url,
+)
+pulumi.export(
+    "aws_ecr_model_trainer_server_worker_image", model_trainer_server_worker_image_uri
 )
-pulumi.export("aws_ecr_training_worker_image", training_worker_image_uri)
 pulumi.export(
     "training_api_url",
     pulumi.Output.concat(
-        "http://training-server.", service_discovery_namespace.name, ":4200/api"
+        "http://model-trainer-server.", service_discovery_namespace.name, ":4200/api"
     ),
 )
 training_ui_url = (