Skip to content

Add S3/ECR "down" command resource handling #772

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chrisaddy merged 2 commits into from
Feb 25, 2026
Merged

Add S3/ECR "down" command resource handling #772

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
629fb84
Add S3/ECR "down" command resource handling
forstmeier Feb 25, 2026
5cbb2f7
Add bot pull request feedback
forstmeier Feb 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion infrastructure/Pulumi.production.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
---
# yamllint disable rule:line-length
Comment thread
forstmeier marked this conversation as resolved.
config:
aws:region: us-east-1
Expand All @@ -24,3 +23,5 @@ config:
secure: AAABAN+AOhwp6jr/FKgLoyOBHVEeiG8kxzGJrsTr30gnj5JiuQs+YJmmFRSd0cdOHJotfJ8Y
fund:sharedSecretValue:
secure: AAABACKTBgsKXMGiDo/WXf5/WTwxHIKAKYUGOMhCecEe09+g/huViXxO1fYA+I2EdIcxBk8zerAoxkOGNUMkik+45skuj3vUYHraOLiKmzSt9h7Z1R56ixPoNMrTSbCMjpHOHZSji0G7lH2qCdCj6jGH/aouZjZRsnPLGa5/pxjhe+1aUtuvwLoqr6IlyuPEkw==
fund:randomSuffix:
secure: AAABAPl47ORoO6t8NqEn8I/e49nYu7cuGBwRgA59mBFIKX7RyHTk1w==
26 changes: 22 additions & 4 deletions infrastructure/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ def serialize_secret_config_object(

region = aws_config.require("region")

random_suffix = stack_config.require_secret("randomSuffix")
Comment thread
forstmeier marked this conversation as resolved.
Comment thread
forstmeier marked this conversation as resolved.

github_actions_role_name = stack_config.require("githubActionsRoleName")
github_repository = stack_config.require("githubRepository")
github_branch = stack_config.require("githubBranch")
Expand Down Expand Up @@ -235,7 +237,7 @@ def serialize_secret_config_object(
# S3 Data Bucket for storing equity bars, predictions, portfolios
data_bucket = aws.s3.Bucket(
"data_bucket",
bucket_prefix="fund-data-",
bucket=pulumi.Output.concat("fund-data-", random_suffix),
opts=pulumi.ResourceOptions(retain_on_delete=True),
tags=tags,
)
Expand All @@ -250,6 +252,7 @@ def serialize_secret_config_object(
),
)
],
opts=pulumi.ResourceOptions(retain_on_delete=True),
)

aws.s3.BucketPublicAccessBlock(
Expand All @@ -259,6 +262,7 @@ def serialize_secret_config_object(
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=True,
opts=pulumi.ResourceOptions(retain_on_delete=True),
)

aws.s3.BucketVersioning(
Expand All @@ -267,12 +271,13 @@ def serialize_secret_config_object(
versioning_configuration=aws.s3.BucketVersioningVersioningConfigurationArgs(
status="Enabled",
),
opts=pulumi.ResourceOptions(retain_on_delete=True),
)

# S3 Model Artifacts Bucket for storing trained model weights and checkpoints
model_artifacts_bucket = aws.s3.Bucket(
"model_artifacts_bucket",
bucket_prefix="fund-model-artifacts-",
bucket=pulumi.Output.concat("fund-model-artifacts-", random_suffix),
opts=pulumi.ResourceOptions(retain_on_delete=True),
tags=tags,
)
Expand All @@ -287,6 +292,7 @@ def serialize_secret_config_object(
),
)
],
opts=pulumi.ResourceOptions(retain_on_delete=True),
)

aws.s3.BucketPublicAccessBlock(
Expand All @@ -296,6 +302,7 @@ def serialize_secret_config_object(
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=True,
opts=pulumi.ResourceOptions(retain_on_delete=True),
)

aws.s3.BucketVersioning(
Expand All @@ -304,13 +311,18 @@ def serialize_secret_config_object(
versioning_configuration=aws.s3.BucketVersioningVersioningConfigurationArgs(
status="Enabled",
),
opts=pulumi.ResourceOptions(retain_on_delete=True),
)

# ECR Repositories - these must exist before images can be pushed
# force_delete allows repositories containing images to be deleted on stack teardown.
# If image rebuild and push times become prohibitive on daily down/up cycles, switch to
# retain_on_delete=True and add pulumi import statements to the maskfile up command.
datamanager_repository = aws.ecr.Repository(
"datamanager_repository",
name="fund/datamanager-server",
image_tag_mutability="MUTABLE",
force_delete=True,
image_scanning_configuration=aws.ecr.RepositoryImageScanningConfigurationArgs(
scan_on_push=True,
),
Expand All @@ -321,6 +333,7 @@ def serialize_secret_config_object(
"portfoliomanager_repository",
name="fund/portfoliomanager-server",
image_tag_mutability="MUTABLE",
force_delete=True,
image_scanning_configuration=aws.ecr.RepositoryImageScanningConfigurationArgs(
scan_on_push=True,
),
Expand All @@ -331,6 +344,7 @@ def serialize_secret_config_object(
"equitypricemodel_repository",
name="fund/equitypricemodel-server",
image_tag_mutability="MUTABLE",
force_delete=True,
image_scanning_configuration=aws.ecr.RepositoryImageScanningConfigurationArgs(
scan_on_push=True,
),
Expand All @@ -341,6 +355,7 @@ def serialize_secret_config_object(
"equitypricemodel_trainer_repository",
name="fund/equitypricemodel-trainer",
image_tag_mutability="MUTABLE",
force_delete=True,
image_scanning_configuration=aws.ecr.RepositoryImageScanningConfigurationArgs(
scan_on_push=True,
),
Expand Down Expand Up @@ -1781,8 +1796,11 @@ def serialize_secret_config_object(
pulumi.export(
"aws_ecr_equitypricemodel_repository", equitypricemodel_repository.repository_url
)
pulumi.export("aws_s3_data_bucket_name", data_bucket.bucket)
pulumi.export("aws_s3_model_artifacts_bucket_name", model_artifacts_bucket.bucket)
pulumi.export("aws_s3_data_bucket_name", pulumi.Output.unsecret(data_bucket.bucket))
pulumi.export(
"aws_s3_model_artifacts_bucket_name",
pulumi.Output.unsecret(model_artifacts_bucket.bucket),
)
pulumi.export(
"aws_ecr_equitypricemodel_trainer_repository",
equitypricemodel_trainer_repository.repository_url,
Expand Down
26 changes: 21 additions & 5 deletions maskfile.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,22 +193,38 @@ fi

pulumi stack select ${organization_name}/fund/production --create

if ! pulumi config get fund:randomSuffix >/dev/null 2>&1; then
pulumi config set --secret fund:randomSuffix "$(openssl rand -hex 4)"
fi

RANDOM_SUFFIX=$(pulumi config get fund:randomSuffix)
Comment thread
forstmeier marked this conversation as resolved.

echo "Importing existing resources into Pulumi state (if they exist)"

pulumi import --yes aws:iam/role:Role github_actions_infrastructure_role fund-github-actions-infrastructure-role 2>/dev/null || true
pulumi import --yes --generate-code=false aws:iam/role:Role github_actions_infrastructure_role fund-github-actions-infrastructure-role 2>/dev/null || true

GITHUB_POLICY_ARN=$(aws iam list-policies --scope Local --query 'Policies[?PolicyName==`fund-github-actions-infrastructure-policy`].Arn' --output text 2>/dev/null || echo "")
if [ -n "$GITHUB_POLICY_ARN" ]; then
pulumi import --yes aws:iam/policy:Policy github_actions_infrastructure_policy "$GITHUB_POLICY_ARN" 2>/dev/null || true
pulumi import --yes --generate-code=false aws:iam/policy:Policy github_actions_infrastructure_policy "$GITHUB_POLICY_ARN" 2>/dev/null || true
fi

pulumi import --yes aws:iam/role:Role sagemaker_execution_role fund-sagemaker-execution-role 2>/dev/null || true
pulumi import --yes --generate-code=false aws:iam/role:Role sagemaker_execution_role fund-sagemaker-execution-role 2>/dev/null || true

SAGEMAKER_POLICY_ARN=$(aws iam list-policies --scope Local --query 'Policies[?PolicyName==`fund-sagemaker-execution-policy`].Arn' --output text 2>/dev/null || echo "")
if [ -n "$SAGEMAKER_POLICY_ARN" ]; then
pulumi import --yes aws:iam/policy:Policy sagemaker_execution_policy "$SAGEMAKER_POLICY_ARN" 2>/dev/null || true
pulumi import --yes --generate-code=false aws:iam/policy:Policy sagemaker_execution_policy "$SAGEMAKER_POLICY_ARN" 2>/dev/null || true
fi

pulumi import --yes --generate-code=false aws:s3/bucket:Bucket data_bucket "fund-data-${RANDOM_SUFFIX}" 2>/dev/null || true
pulumi import --yes --generate-code=false aws:s3/bucketServerSideEncryptionConfiguration:BucketServerSideEncryptionConfiguration data_bucket_encryption "fund-data-${RANDOM_SUFFIX}" 2>/dev/null || true
pulumi import --yes --generate-code=false aws:s3/bucketPublicAccessBlock:BucketPublicAccessBlock data_bucket_public_access_block "fund-data-${RANDOM_SUFFIX}" 2>/dev/null || true
pulumi import --yes --generate-code=false aws:s3/bucketVersioning:BucketVersioning data_bucket_versioning "fund-data-${RANDOM_SUFFIX}" 2>/dev/null || true

pulumi import --yes --generate-code=false aws:s3/bucket:Bucket model_artifacts_bucket "fund-model-artifacts-${RANDOM_SUFFIX}" 2>/dev/null || true
pulumi import --yes --generate-code=false aws:s3/bucketServerSideEncryptionConfiguration:BucketServerSideEncryptionConfiguration model_artifacts_bucket_encryption "fund-model-artifacts-${RANDOM_SUFFIX}" 2>/dev/null || true
pulumi import --yes --generate-code=false aws:s3/bucketPublicAccessBlock:BucketPublicAccessBlock model_artifacts_bucket_public_access_block "fund-model-artifacts-${RANDOM_SUFFIX}" 2>/dev/null || true
pulumi import --yes --generate-code=false aws:s3/bucketVersioning:BucketVersioning model_artifacts_bucket_versioning "fund-model-artifacts-${RANDOM_SUFFIX}" 2>/dev/null || true

echo "Importing resources complete"

pulumi up --diff --yes
Expand Down Expand Up @@ -699,7 +715,7 @@ set -euo pipefail

echo "Running Markdown lint checks"

markdownlint "**/*.md" --ignore ".flox" --ignore ".venv" --ignore "target"
markdownlint "**/*.md" --ignore ".flox" --ignore ".venv" --ignore "target" --ignore ".scratchpad"

echo "Markdown linting completed successfully"
```
Expand Down
Loading