From 5b9b8d1afab8664eb440e18f3b94f9f93c77aec5 Mon Sep 17 00:00:00 2001 From: John Forstmeier Date: Fri, 26 Dec 2025 13:23:17 -0500 Subject: [PATCH 1/2] Add service image build performance improvements --- .github/workflows/launch_infrastructure.yaml | 58 ++++++++++++++++---- applications/datamanager/Dockerfile | 11 +++- applications/equitypricemodel/Dockerfile | 2 +- applications/portfoliomanager/Dockerfile | 2 +- 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/.github/workflows/launch_infrastructure.yaml b/.github/workflows/launch_infrastructure.yaml index 47f6bb6fa..72fb48d65 100644 --- a/.github/workflows/launch_infrastructure.yaml +++ b/.github/workflows/launch_infrastructure.yaml @@ -19,35 +19,69 @@ jobs: contents: read strategy: matrix: - service: - - datamanager - - portfoliomanager - - equitypricemodel + include: + - service: datamanager + paths: applications/datamanager/** + - service: portfoliomanager + paths: applications/portfoliomanager/** + - service: equitypricemodel + paths: applications/equitypricemodel/** steps: - name: Checkout code uses: actions/checkout@v4 + - name: Check for service changes + uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + service: + - '${{ matrix.paths }}' + - 'libraries/python/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'pyproject.toml' + - 'uv.lock' - name: Configure AWS credentials + if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule' uses: aws-actions/configure-aws-credentials@v5 with: role-to-assume: ${{ secrets.AWS_IAM_INFRASTRUCTURE_ROLE_ARN }} aws-region: ${{ secrets.AWS_REGION }} + - name: Login to Amazon ECR + if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule' + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule' + uses: docker/setup-buildx-action@v3 - name: Install Flox + if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule' uses: flox/install-flox-action@v2 - name: Conditionally download artifacts - if: ${{ matrix.service == 'equitypricemodel' }} + if: (steps.changes.outputs.service == 'true' || github.event_name == 'schedule') && matrix.service == 'equitypricemodel' uses: flox/activate-action@v1 env: AWS_S3_ARTIFACTS_BUCKET_NAME: ${{ secrets.AWS_S3_ARTIFACTS_BUCKET_NAME }} with: command: mask models artifacts download equitypricemodel - - name: Build ${{ matrix.service }} image - uses: flox/activate-action@v1 - with: - command: mask infrastructure images build ${{ matrix.service }} server - - name: Push ${{ matrix.service }} image - uses: flox/activate-action@v1 + - name: Get AWS Account ID + if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule' + id: aws-account + run: echo "account_id=$(aws sts get-caller-identity --query Account --output text)" >> $GITHUB_OUTPUT + - name: Build and push ${{ matrix.service }} image + if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule' + uses: docker/build-push-action@v5 with: - command: mask infrastructure images push ${{ matrix.service }} server + context: . + file: applications/${{ matrix.service }}/Dockerfile + target: server + platforms: linux/amd64 + push: true + tags: | + pocketsizefund/${{ matrix.service }}-server:latest + ${{ steps.aws-account.outputs.account_id }}.dkr.ecr.us-east-1.amazonaws.com/pocketsizefund/${{ matrix.service }}-server:latest + cache-from: type=registry,ref=${{ steps.aws-account.outputs.account_id }}.dkr.ecr.us-east-1.amazonaws.com/pocketsizefund/${{ matrix.service }}-server:buildcache + cache-to: type=registry,ref=${{ steps.aws-account.outputs.account_id }}.dkr.ecr.us-east-1.amazonaws.com/pocketsizefund/${{ matrix.service }}-server:buildcache,mode=max deploy: name: Deploy with Pulumi needs: build_and_push diff --git a/applications/datamanager/Dockerfile b/applications/datamanager/Dockerfile index 28cf2ed0f..90d2434e6 100644 --- a/applications/datamanager/Dockerfile +++ b/applications/datamanager/Dockerfile @@ -26,11 +26,16 @@ FROM chef AS builder COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --release --recipe-path recipe.json +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/app/target \ + cargo chef cook --release --recipe-path recipe.json COPY applications/datamanager/src/ applications/datamanager/src/ -RUN cargo build --release --bin datamanager +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/app/target \ + cargo build --release --bin datamanager && \ + cp /app/target/release/datamanager /tmp/datamanager FROM debian:bullseye-slim AS server @@ -45,7 +50,7 @@ ENV SSL_CERT_DIR=/etc/ssl/certs WORKDIR /app -COPY --from=builder /app/target/release/datamanager /usr/local/bin +COPY --from=builder /tmp/datamanager /usr/local/bin/datamanager EXPOSE 8080 diff --git a/applications/equitypricemodel/Dockerfile b/applications/equitypricemodel/Dockerfile index 6ed76bf11..9d6c1a156 100644 --- a/applications/equitypricemodel/Dockerfile +++ b/applications/equitypricemodel/Dockerfile @@ -10,7 +10,7 @@ COPY applications/equitypricemodel/ applications/equitypricemodel/ COPY libraries/python/ libraries/python/ -RUN uv sync --no-dev --no-cache +RUN uv sync --no-dev FROM python:3.12.10-slim AS trainer diff --git a/applications/portfoliomanager/Dockerfile b/applications/portfoliomanager/Dockerfile index dd6e33593..1cdd39bd7 100644 --- a/applications/portfoliomanager/Dockerfile +++ b/applications/portfoliomanager/Dockerfile @@ -10,7 +10,7 @@ COPY applications/portfoliomanager/ applications/portfoliomanager/ COPY libraries/python/ libraries/python/ -RUN uv sync --no-dev --no-cache +RUN uv sync --no-dev FROM python:3.12.10-slim AS server From c4362980e5129dffd05f2c720bbf1e95db9f93ab Mon Sep 17 00:00:00 2001 From: John Forstmeier Date: Fri, 26 Dec 2025 15:00:54 -0500 Subject: [PATCH 2/2] Add pull request feedback/add Pulumi stack reference to launch infrastructure workflow --- .github/workflows/launch_infrastructure.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/launch_infrastructure.yaml b/.github/workflows/launch_infrastructure.yaml index 72fb48d65..0aea090e9 100644 --- a/.github/workflows/launch_infrastructure.yaml +++ b/.github/workflows/launch_infrastructure.yaml @@ -79,9 +79,9 @@ jobs: push: true tags: | pocketsizefund/${{ matrix.service }}-server:latest - ${{ steps.aws-account.outputs.account_id }}.dkr.ecr.us-east-1.amazonaws.com/pocketsizefund/${{ matrix.service }}-server:latest - cache-from: type=registry,ref=${{ steps.aws-account.outputs.account_id }}.dkr.ecr.us-east-1.amazonaws.com/pocketsizefund/${{ matrix.service }}-server:buildcache - cache-to: type=registry,ref=${{ steps.aws-account.outputs.account_id }}.dkr.ecr.us-east-1.amazonaws.com/pocketsizefund/${{ matrix.service }}-server:buildcache,mode=max + ${{ steps.aws-account.outputs.account_id }}.dkr.ecr.{{ secrets.AWS_REGION }}.amazonaws.com/pocketsizefund/${{ matrix.service }}-server:latest + cache-from: type=registry,ref=${{ steps.aws-account.outputs.account_id }}.dkr.ecr.{{ secrets.AWS_REGION }}.amazonaws.com/pocketsizefund/${{ matrix.service }}-server:buildcache + cache-to: type=registry,ref=${{ steps.aws-account.outputs.account_id }}.dkr.ecr.{{ secrets.AWS_REGION }}.amazonaws.com/pocketsizefund/${{ matrix.service }}-server:buildcache,mode=max deploy: name: Deploy with Pulumi needs: build_and_push @@ -100,6 +100,12 @@ jobs: aws-region: ${{ secrets.AWS_REGION }} - name: Install Flox uses: flox/install-flox-action@v2 + - name: Initialize Pulumi stack + uses: flox/activate-action@v1 + env: + PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} + with: + command: cd infrastructure && pulumi stack select production --create - name: Deploy with Pulumi uses: flox/activate-action@v1 env: