diff --git a/.flox/env/manifest.lock b/.flox/env/manifest.lock index 6237063cc..954502996 100644 --- a/.flox/env/manifest.lock +++ b/.flox/env/manifest.lock @@ -127,6 +127,9 @@ "hook": { "on-activate": "if [[ -f .env ]]; then\n set -a\n source .env\n set +a\nfi\n" }, + "profile": { + "common": "if [[ -f .env ]]; then\n set -a\n source .env\n set +a\nfi\n" + }, "options": { "systems": [ "aarch64-darwin", diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml index 6e2bf0ae3..9bf207fa4 100644 --- a/.github/workflows/run_code_checks.yaml +++ b/.github/workflows/run_code_checks.yaml @@ -3,10 +3,14 @@ name: Code checks run-name: Code checks on: pull_request: + push: + branches: + - master jobs: run_rust_code_checks: name: Run Rust code checks runs-on: ubuntu-latest + timeout-minutes: 60 steps: - name: Checkout code uses: actions/checkout@v4 @@ -20,6 +24,14 @@ jobs: sudo rm -rf "$AGENT_TOOLSDIRECTORY" echo "Disk space after cleanup:" df -h + - name: Install DuckDB system library + run: | + wget -q https://github.com/duckdb/duckdb/releases/download/v1.4.3/libduckdb-linux-amd64.zip + unzip libduckdb-linux-amd64.zip -d /tmp/duckdb + sudo mv /tmp/duckdb/libduckdb.so /usr/local/lib/ + sudo mv /tmp/duckdb/*.h /usr/local/include/ + sudo ldconfig + rm -rf /tmp/duckdb libduckdb-linux-amd64.zip - name: Install Flox uses: flox/install-flox-action@v2 - name: Cache Rust dependencies @@ -28,7 +40,7 @@ jobs: workspaces: . cache-on-failure: false shared-key: rust-continuous-integration - save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + save-if: ${{ github.ref == 'refs/heads/master' }} - name: Run Rust code checks uses: flox/activate-action@v1 with: diff --git a/Cargo.lock b/Cargo.lock index 485ede241..f3720a8c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2804,7 +2804,6 @@ version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d78bacb8933586cee3b550c39b610d314f9b7a48701ac7a914a046165a4ad8da" dependencies = [ - "cc", "flate2", "pkg-config", "reqwest", diff --git a/applications/datamanager/Cargo.toml b/applications/datamanager/Cargo.toml index 1015a871f..30c980c80 100644 --- a/applications/datamanager/Cargo.toml +++ b/applications/datamanager/Cargo.toml @@ -34,7 +34,7 @@ tracing-subscriber = { version = "0.3.20", features = ["env-filter", "fmt"] } aws-config = "1.5" aws-sdk-s3 = "1.112" aws-credential-types = "1.2.6" -duckdb = { version = "1.4.3", features = ["bundled", "r2d2", "chrono"] } +duckdb = { version = "1.4.3", features = ["r2d2", "chrono"] } validator = { version = "0.20", features = ["derive"] } thiserror = "2.0.3" sentry = { version = "0.35", features = ["tracing", "reqwest", "rustls"] } diff --git a/applications/datamanager/src/state.rs b/applications/datamanager/src/state.rs index c422623fc..36208bcb3 100644 --- a/applications/datamanager/src/state.rs +++ b/applications/datamanager/src/state.rs @@ -38,7 +38,7 @@ impl State { let s3_client = S3Client::new(&config); let bucket_name = - std::env::var("AWS_S3_DATA_BUCKET_NAME").unwrap_or_else(|_| "oscm-data".to_string()); + std::env::var("AWS_S3_DATA_BUCKET_NAME").unwrap_or_else(|_| "fund-data".to_string()); info!("Using S3 bucket: {}", bucket_name); let massive_base_url = std::env::var("MASSIVE_BASE_URL") diff --git a/applications/datamanager/tests/common/mod.rs b/applications/datamanager/tests/common/mod.rs index 341c400b8..78d759f16 100644 --- a/applications/datamanager/tests/common/mod.rs +++ b/applications/datamanager/tests/common/mod.rs @@ -4,7 +4,7 @@ use aws_credential_types::Credentials; use aws_sdk_s3::{config::Region, primitives::ByteStream, Client as S3Client}; use axum::Router; use std::{net::SocketAddr, sync::OnceLock, time::Duration}; -use testcontainers::runners::AsyncRunner; +use testcontainers::{runners::AsyncRunner, ContainerAsync}; use testcontainers_modules::localstack::LocalStack; use tokio::{net::TcpListener, sync::oneshot, task::JoinHandle}; @@ -14,6 +14,7 @@ const TEST_SECRET_KEY: &str = "test"; const TEST_REGION: &str = "us-east-1"; static LOCALSTACK_ENDPOINT: OnceLock = OnceLock::new(); +static LOCALSTACK_CONTAINER: OnceLock<&'static ContainerAsync> = OnceLock::new(); static TRACING_INIT: std::sync::Once = std::sync::Once::new(); pub struct EnvironmentVariableGuard { @@ -97,6 +98,9 @@ pub async fn get_localstack_endpoint() -> String { .await .expect("Failed to start LocalStack container — is Docker running?"); + // Give LocalStack additional time to fully initialize services + tokio::time::sleep(Duration::from_secs(5)).await; + let host = container.get_host().await.unwrap(); let port = container.get_host_port_ipv4(4566).await.unwrap(); let endpoint = format!("http://{}:{}", host, port); @@ -107,14 +111,14 @@ pub async fn get_localstack_endpoint() -> String { // - Tests use #[serial] for sequential execution within this process // - All tests share the same LocalStack container for performance // - Container cleanup happens automatically when process exits - // - Alternative (proper Drop cleanup) requires complex lifetime management - // across static OnceLock, creating more complexity than benefit + // - Storing container reference prevents testcontainers from losing port mapping // // Trade-off: Small memory leak during test execution vs architectural complexity // Impact: Container memory is reclaimed when test process terminates - Box::leak(Box::new(container)); - + let leaked_container: &'static ContainerAsync = Box::leak(Box::new(container)); + let _ = LOCALSTACK_CONTAINER.set(leaked_container); let _ = LOCALSTACK_ENDPOINT.set(endpoint.clone()); + endpoint } diff --git a/applications/datamanager/tests/test_state_and_health.rs b/applications/datamanager/tests/test_state_and_health.rs index 8f51ae2aa..7330495e8 100644 --- a/applications/datamanager/tests/test_state_and_health.rs +++ b/applications/datamanager/tests/test_state_and_health.rs @@ -85,7 +85,7 @@ async fn test_state_from_env_uses_defaults_when_variables_are_missing() { let state = State::from_env().await; - assert_eq!(state.bucket_name, "oscm-data"); + assert_eq!(state.bucket_name, "fund-data"); assert_eq!(state.massive.base, "https://api.massive.io"); assert!(state.massive.key.is_empty()); } diff --git a/applications/equitypricemodel/src/equitypricemodel/server.py b/applications/equitypricemodel/src/equitypricemodel/server.py index 1d1172569..9d3916f28 100644 --- a/applications/equitypricemodel/src/equitypricemodel/server.py +++ b/applications/equitypricemodel/src/equitypricemodel/server.py @@ -58,7 +58,7 @@ logger = structlog.get_logger() -DATAMANAGER_BASE_URL = os.getenv("OSCM_DATAMANAGER_BASE_URL", "http://datamanager:8080") +DATAMANAGER_BASE_URL = os.getenv("FUND_DATAMANAGER_BASE_URL", "http://datamanager:8080") def find_latest_artifact_key( @@ -314,14 +314,24 @@ def create_predictions(request: Request) -> Response: ) ) - processed_predictions = predictions_schema.validate(processed_predictions) + try: + validated_result = predictions_schema.validate(processed_predictions) + validated_predictions = cast( + "pl.DataFrame", + validated_result.collect() + if isinstance(validated_result, pl.LazyFrame) + else validated_result, + ) + except Exception: + logger.exception("Predictions failed schema validation") + return Response(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) try: save_predictions_response = requests.post( url=f"{DATAMANAGER_BASE_URL}/predictions", json={ "timestamp": current_timestamp.isoformat(), - "data": processed_predictions.to_dicts(), + "data": validated_predictions.to_dicts(), }, timeout=60, ) @@ -339,7 +349,7 @@ def create_predictions(request: Request) -> Response: logger.info("Successfully generated and saved predictions") return Response( - content=json.dumps({"data": processed_predictions.to_dicts()}).encode("utf-8"), + content=json.dumps({"data": validated_predictions.to_dicts()}).encode("utf-8"), status_code=status.HTTP_200_OK, ) diff --git a/applications/portfoliomanager/src/portfoliomanager/risk_management.py b/applications/portfoliomanager/src/portfoliomanager/risk_management.py index a68626228..2235a6a76 100644 --- a/applications/portfoliomanager/src/portfoliomanager/risk_management.py +++ b/applications/portfoliomanager/src/portfoliomanager/risk_management.py @@ -9,7 +9,7 @@ logger = structlog.get_logger() -UNCERTAINTY_THRESHOLD = float(os.getenv("OSCM_UNCERTAINTY_THRESHOLD", "0.1")) +UNCERTAINTY_THRESHOLD = float(os.getenv("FUND_UNCERTAINTY_THRESHOLD", "0.1")) REQUIRED_PORTFOLIO_SIZE = 20 # 10 long + 10 short SIDE_SIZE = REQUIRED_PORTFOLIO_SIZE // 2 diff --git a/applications/portfoliomanager/src/portfoliomanager/server.py b/applications/portfoliomanager/src/portfoliomanager/server.py index 7d8fe0c70..9ac378fd9 100644 --- a/applications/portfoliomanager/src/portfoliomanager/server.py +++ b/applications/portfoliomanager/src/portfoliomanager/server.py @@ -56,11 +56,11 @@ application: FastAPI = FastAPI() -DATAMANAGER_BASE_URL = os.getenv("OSCM_DATAMANAGER_BASE_URL", "http://datamanager:8080") +DATAMANAGER_BASE_URL = os.getenv("FUND_DATAMANAGER_BASE_URL", "http://datamanager:8080") HTTP_NOT_FOUND = 404 HTTP_BAD_REQUEST = 400 EQUITYPRICEMODEL_BASE_URL = os.getenv( - "OSCM_EQUITYPRICEMODEL_BASE_URL", + "FUND_EQUITYPRICEMODEL_BASE_URL", "http://equitypricemodel:8080", ) diff --git a/infrastructure/Pulumi.production.yaml b/infrastructure/Pulumi.production.yaml index 1a38cefb1..b63e995e4 100644 --- a/infrastructure/Pulumi.production.yaml +++ b/infrastructure/Pulumi.production.yaml @@ -1,2 +1,24 @@ config: aws:region: us-east-1 + fund:budgetAlertEmailAddresses: + secure: AAABAEVOj8Wk2uRR16QLK4GVV/0Q7NqIgcpgPSEGeWeIDeq2l9JANRiPzcqKk0HfPYnJ1+x+DR0GeVYwvSENSURnq/Or5Bc0pMqMYHyMFbXqXp45VE6J2XuyaUSF8WQK3IyNcJr7HAoK/9VtRrttbQ== + fund:githubActionsRoleName: fund-github-actions-infrastructure-role + fund:sagemakerExecutionRoleName: fund-sagemaker-execution-role + fund:githubRepository: oscmcompany/fund + fund:githubBranch: master + fund:monthlyBudgetLimitUsd: "25" + fund:githubWorkflowFiles: + - launch_infrastructure.yaml + - teardown_infrastructure.yaml + fund:datamanagerSecretName: + secure: AAABALDuCuT721er5Kxz4DeVhpYFYB1U2k87kWMSWiIxDWE9nTss9OeI4Nv+Tu0w39hdQlcquPbZ330= + fund:datamanagerSecretValue: + secure: AAABAChRujETi2wJ+Tw2jj64uVvght6AMW5E+nuePkXqKXWyrMSVEIVQ5o9NRIHRRQ0wh1F3WzRW+BysSZhpdH/fULYOz9gtMmKlcG01uAApHdy8cWBk + fund:portfoliomanagerSecretName: + secure: AAABAJatHFOp4tQTL8lf14JfSYVM1GlUyFASwA+eLhdZ9Fe/xwzYDe8LrkFwjW8H6Uc8heRdNdKOur7OplyrJA== + fund:portfoliomanagerSecretValue: + secure: AAABAKW6YZV5HEHJBO04eh9QW/woEl/cvAgoDiURRH5e+YUwr+Muw6IR4hny+NQXrftQhk0EGS/E6OA1n9nElWqvjycm23MV/xR4CTlvW60CRQoQrOSWZuDdqmwv4KUa7qxVEZqYrPO86qJDBfXDIDJ5lkW8R1XmDR/+ubIV3nrkgvkEnXEtZWx1hzeXSsgfR7759Q0QXbSs8AfhGT8hxztiDxwDsSFQOA== + fund:sharedSecretName: + secure: AAABAMd0Q1ggmUrYldoYecdCllBpZ30N8F4Wnfvla9iX8FNIfC+3b+plSxGcI3s+JvIMOpoo + fund:sharedSecretValue: + secure: AAABACKTBgsKXMGiDo/WXf5/WTwxHIKAKYUGOMhCecEe09+g/huViXxO1fYA+I2EdIcxBk8zerAoxkOGNUMkik+45skuj3vUYHraOLiKmzSt9h7Z1R56ixPoNMrTSbCMjpHOHZSji0G7lH2qCdCj6jGH/aouZjZRsnPLGa5/pxjhe+1aUtuvwLoqr6IlyuPEkw== diff --git a/infrastructure/Pulumi.yaml b/infrastructure/Pulumi.yaml index c706dbb81..b25136abe 100644 --- a/infrastructure/Pulumi.yaml +++ b/infrastructure/Pulumi.yaml @@ -1,4 +1,4 @@ -name: oscm +name: fund runtime: name: python options: diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index 8aeb320d5..d914a5932 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -1,35 +1,263 @@ import json +from typing import cast import parameters import pulumi import pulumi_aws as aws -current_identity = aws.get_caller_identity() -account_id = current_identity.account_id +def require_secret_config_object( + config: pulumi.Config, + key: str, +) -> pulumi.Output[dict[str, str]]: + config_full_key = config.full_key(key) + if not pulumi.runtime.is_config_secret(config_full_key): + message = f"Pulumi config '{key}' must be configured as a secret object." + raise ValueError(message) + + return cast( + "pulumi.Output[dict[str, str]]", + config.require_secret_object(key), + ) + + +def serialize_secret_config_object( + secret_values: dict[str, str], + config_key: str, + required_keys: set[str], +) -> str: + missing_secret_keys = sorted(required_keys.difference(secret_values)) + if missing_secret_keys: + message = ( + f"Pulumi config '{config_key}' is missing required keys: " + f"{', '.join(missing_secret_keys)}." + ) + raise ValueError(message) + + return json.dumps(secret_values, sort_keys=True) -region = aws.get_region().name -secret = aws.secretsmanager.get_secret( - name="oscm/production/environment_variables", +stack_name = pulumi.get_stack() +if stack_name != "production": + message = "Only the production Pulumi stack is supported." + raise ValueError(message) + +stack_config = pulumi.Config("fund") +aws_config = pulumi.Config("aws") + +region = aws_config.require("region") + +github_actions_role_name = stack_config.require("githubActionsRoleName") +github_repository = stack_config.require("githubRepository") +github_branch = stack_config.require("githubBranch") +github_workflow_files = cast( + "list[str]", + stack_config.require_object("githubWorkflowFiles"), +) +if not github_workflow_files: + message = ( + "Pulumi config 'githubWorkflowFiles' must include at least one workflow file." + ) + raise ValueError(message) + +budget_alert_email_addresses_full_key = stack_config.full_key( + "budgetAlertEmailAddresses" ) +if not pulumi.runtime.is_config_secret(budget_alert_email_addresses_full_key): + message = ( + "Pulumi config 'budgetAlertEmailAddresses' must be configured as a secret list." + ) + raise ValueError(message) +budget_alert_email_addresses = cast( + "list[str]", + stack_config.require_secret_object("budgetAlertEmailAddresses"), +) +if not budget_alert_email_addresses: + message = ( + "Pulumi config 'budgetAlertEmailAddresses' must include at least one email " + "address." + ) + raise ValueError(message) + +monthly_budget_limit_usd = stack_config.require_float("monthlyBudgetLimitUsd") +sagemaker_execution_role_name = stack_config.require("sagemakerExecutionRoleName") + +datamanager_secret_name = stack_config.require_secret("datamanagerSecretName") +portfoliomanager_secret_name = stack_config.require_secret("portfoliomanagerSecretName") +shared_secret_name = stack_config.require_secret("sharedSecretName") + +datamanager_secret_values = require_secret_config_object( + stack_config, + "datamanagerSecretValue", +) +portfoliomanager_secret_values = require_secret_config_object( + stack_config, + "portfoliomanagerSecretValue", +) +shared_secret_values = require_secret_config_object( + stack_config, + "sharedSecretValue", +) + +github_oidc_audience_claim = "token.actions.githubusercontent.com:aud" +github_oidc_repository_claim = "token.actions.githubusercontent.com:repository" +github_oidc_ref_claim = "token.actions.githubusercontent.com:ref" +github_oidc_sub_claim = "token.actions.githubusercontent.com:sub" +github_oidc_workflow_ref_claim = "token.actions.githubusercontent.com:job_workflow_ref" + +github_workflow_refs = [ + ( + f"{github_repository}/.github/workflows/{github_workflow_file}" + f"@refs/heads/{github_branch}" + ) + for github_workflow_file in github_workflow_files +] + +current_identity = aws.get_caller_identity() + +account_id = current_identity.account_id availability_zone_a = f"{region}a" availability_zone_b = f"{region}b" tags = { - "project": "oscm", - "stack": pulumi.get_stack(), + "project": "fund", + "stack": stack_name, "manager": "pulumi", } +github_oidc_provider_arn = pulumi.Output.concat( + "arn:aws:iam::", + account_id, + ":oidc-provider/token.actions.githubusercontent.com", +) + +datamanager_secret = aws.secretsmanager.Secret( + "datamanager_secret", + name=datamanager_secret_name, + tags=tags, +) + +portfoliomanager_secret = aws.secretsmanager.Secret( + "portfoliomanager_secret", + name=portfoliomanager_secret_name, + tags=tags, +) + +shared_secret = aws.secretsmanager.Secret( + "shared_secret", + name=shared_secret_name, + tags=tags, +) + +aws.secretsmanager.SecretVersion( + "datamanager_secret_version", + secret_id=datamanager_secret.id, + secret_string=datamanager_secret_values.apply( + lambda values: serialize_secret_config_object( + values, + "datamanagerSecretValue", + {"MASSIVE_API_KEY"}, + ) + ), +) + +aws.secretsmanager.SecretVersion( + "portfoliomanager_secret_version", + secret_id=portfoliomanager_secret.id, + secret_string=portfoliomanager_secret_values.apply( + lambda values: serialize_secret_config_object( + values, + "portfoliomanagerSecretValue", + {"ALPACA_API_KEY_ID", "ALPACA_API_SECRET", "ALPACA_IS_PAPER"}, + ) + ), +) + +aws.secretsmanager.SecretVersion( + "shared_secret_version", + secret_id=shared_secret.id, + secret_string=shared_secret_values.apply( + lambda values: serialize_secret_config_object( + values, + "sharedSecretValue", + {"SENTRY_DSN"}, + ) + ), +) + +infrastructure_alerts_topic = aws.sns.Topic( + "infrastructure_alerts_topic", + name="fund-infrastructure-alerts", + tags=tags, +) + +for notification_email_index, notification_email_address in enumerate( + budget_alert_email_addresses, + start=1, +): + aws.sns.TopicSubscription( + f"infrastructure_alert_email_subscription_{notification_email_index}", + topic=infrastructure_alerts_topic.arn, + protocol="email", + endpoint=notification_email_address, + ) + +aws.budgets.Budget( + "production_cost_budget", + account_id=account_id, + name="fund-monthly-cost", + budget_type="COST", + time_unit="MONTHLY", + limit_amount=f"{monthly_budget_limit_usd:.2f}", + limit_unit="USD", + notifications=[ + aws.budgets.BudgetNotificationArgs( + comparison_operator="GREATER_THAN", + notification_type="ACTUAL", + threshold=monthly_budget_limit_usd, + threshold_type="ABSOLUTE_VALUE", + subscriber_email_addresses=budget_alert_email_addresses, + ), + aws.budgets.BudgetNotificationArgs( + comparison_operator="GREATER_THAN", + notification_type="FORECASTED", + threshold=monthly_budget_limit_usd, + threshold_type="ABSOLUTE_VALUE", + subscriber_email_addresses=budget_alert_email_addresses, + ), + ], +) + # S3 Data Bucket for storing equity bars, predictions, portfolios data_bucket = aws.s3.Bucket( "data_bucket", - bucket_prefix="oscm-data-", + bucket_prefix="fund-data-", + opts=pulumi.ResourceOptions(retain_on_delete=True), tags=tags, ) +aws.s3.BucketServerSideEncryptionConfigurationV2( + "data_bucket_encryption", + bucket=data_bucket.id, + rules=[ + aws.s3.BucketServerSideEncryptionConfigurationV2RuleArgs( + apply_server_side_encryption_by_default=aws.s3.BucketServerSideEncryptionConfigurationV2RuleApplyServerSideEncryptionByDefaultArgs( + sse_algorithm="AES256", + ), + ) + ], +) + +aws.s3.BucketPublicAccessBlock( + "data_bucket_public_access_block", + bucket=data_bucket.id, + block_public_acls=True, + block_public_policy=True, + ignore_public_acls=True, + restrict_public_buckets=True, +) + aws.s3.BucketVersioning( "data_bucket_versioning", bucket=data_bucket.id, @@ -41,10 +269,32 @@ # S3 Model Artifacts Bucket for storing trained model weights and checkpoints model_artifacts_bucket = aws.s3.Bucket( "model_artifacts_bucket", - bucket_prefix="oscm-model-artifacts-", + bucket_prefix="fund-model-artifacts-", + opts=pulumi.ResourceOptions(retain_on_delete=True), tags=tags, ) +aws.s3.BucketServerSideEncryptionConfigurationV2( + "model_artifacts_bucket_encryption", + bucket=model_artifacts_bucket.id, + rules=[ + aws.s3.BucketServerSideEncryptionConfigurationV2RuleArgs( + apply_server_side_encryption_by_default=aws.s3.BucketServerSideEncryptionConfigurationV2RuleApplyServerSideEncryptionByDefaultArgs( + sse_algorithm="AES256", + ), + ) + ], +) + +aws.s3.BucketPublicAccessBlock( + "model_artifacts_bucket_public_access_block", + bucket=model_artifacts_bucket.id, + block_public_acls=True, + block_public_policy=True, + ignore_public_acls=True, + restrict_public_buckets=True, +) + aws.s3.BucketVersioning( "model_artifacts_bucket_versioning", bucket=model_artifacts_bucket.id, @@ -56,7 +306,7 @@ # ECR Repositories - these must exist before images can be pushed datamanager_repository = aws.ecr.Repository( "datamanager_repository", - name="oscm/datamanager-server", + name="fund/datamanager-server", image_tag_mutability="MUTABLE", image_scanning_configuration=aws.ecr.RepositoryImageScanningConfigurationArgs( scan_on_push=True, @@ -66,7 +316,7 @@ portfoliomanager_repository = aws.ecr.Repository( "portfoliomanager_repository", - name="oscm/portfoliomanager-server", + name="fund/portfoliomanager-server", image_tag_mutability="MUTABLE", image_scanning_configuration=aws.ecr.RepositoryImageScanningConfigurationArgs( scan_on_push=True, @@ -76,7 +326,7 @@ equitypricemodel_repository = aws.ecr.Repository( "equitypricemodel_repository", - name="oscm/equitypricemodel-server", + name="fund/equitypricemodel-server", image_tag_mutability="MUTABLE", image_scanning_configuration=aws.ecr.RepositoryImageScanningConfigurationArgs( scan_on_push=True, @@ -86,7 +336,7 @@ equitypricemodel_trainer_repository = aws.ecr.Repository( "equitypricemodel_trainer_repository", - name="oscm/equitypricemodel-trainer", + name="fund/equitypricemodel-trainer", image_tag_mutability="MUTABLE", image_scanning_configuration=aws.ecr.RepositoryImageScanningConfigurationArgs( scan_on_push=True, @@ -201,6 +451,27 @@ tags=tags, ) +aws.cloudwatch.MetricAlarm( + "nat_gateway_bytes_out_to_destination_alarm", + name="fund-nat-gateway-bytes-out-to-destination", + alarm_description=( + "Triggers when NAT gateway outbound traffic exceeds 500 MB per hour for " + "2 consecutive hours." + ), + namespace="AWS/NATGateway", + metric_name="BytesOutToDestination", + statistic="Sum", + period=3600, + evaluation_periods=2, + threshold=500_000_000, + comparison_operator="GreaterThanThreshold", + treat_missing_data="notBreaching", + dimensions={"NatGatewayId": nat.id}, + alarm_actions=[infrastructure_alerts_topic.arn], + ok_actions=[infrastructure_alerts_topic.arn], + tags=tags, +) + private_route_table = aws.ec2.RouteTable( "private_route_table", vpc_id=vpc.id, @@ -228,7 +499,7 @@ alb_security_group = aws.ec2.SecurityGroup( "alb_sg", - name="oscm-alb", + name="fund-alb", vpc_id=vpc.id, description="Security group for ALB", ingress=[ @@ -261,7 +532,7 @@ ecs_security_group = aws.ec2.SecurityGroup( "ecs_sg", - name="oscm-ecs-tasks", + name="fund-ecs-tasks", vpc_id=vpc.id, description="Security group for ECS tasks", tags=tags, @@ -306,7 +577,7 @@ # VPC Endpoints Security Group vpc_endpoints_security_group = aws.ec2.SecurityGroup( "vpc_endpoints_sg", - name="oscm-vpc-endpoints", + name="fund-vpc-endpoints", vpc_id=vpc.id, description="Security group for VPC endpoints", tags=tags, @@ -359,7 +630,7 @@ cluster = aws.ecs.Cluster( "ecs_cluster", - name="oscm-application", + name="fund-application", settings=[aws.ecs.ClusterSettingArgs(name="containerInsights", value="enabled")], tags=tags, ) @@ -367,15 +638,15 @@ # Service Discovery Namespace for inter-service communication service_discovery_namespace = aws.servicediscovery.PrivateDnsNamespace( "service_discovery", - name="oscm.local", + name="fund.local", vpc=vpc.id, - description="Service discovery for oscm services", + description="Service discovery for fund services", tags=tags, ) alb = aws.lb.LoadBalancer( "alb", - name="oscm-alb", + name="fund-alb", subnets=[public_subnet_1.id, public_subnet_2.id], security_groups=[alb_security_group.id], internal=False, @@ -385,7 +656,7 @@ datamanager_tg = aws.lb.TargetGroup( "datamanager_tg", - name="oscm-datamanager", + name="fund-datamanager", port=8080, protocol="HTTP", vpc_id=vpc.id, @@ -402,7 +673,7 @@ portfoliomanager_tg = aws.lb.TargetGroup( "portfoliomanager_tg", - name="oscm-portfoliomanager", + name="fund-portfoliomanager", port=8080, protocol="HTTP", vpc_id=vpc.id, @@ -419,7 +690,7 @@ equitypricemodel_tg = aws.lb.TargetGroup( "equitypricemodel_tg", - name="oscm-equitypricemodel", + name="fund-equitypricemodel", port=8080, protocol="HTTP", vpc_id=vpc.id, @@ -565,10 +836,384 @@ tags=tags, ) +github_actions_oidc_provider = aws.iam.OpenIdConnectProvider( + "github_actions_oidc_provider", + url="https://token.actions.githubusercontent.com", + client_id_lists=["sts.amazonaws.com"], + tags=tags, +) + +github_actions_infrastructure_role = aws.iam.Role( + "github_actions_infrastructure_role", + name=github_actions_role_name, + assume_role_policy=github_actions_oidc_provider.arn.apply( + lambda github_actions_oidc_provider_arn: json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": github_actions_oidc_provider_arn, + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + github_oidc_audience_claim: "sts.amazonaws.com", + github_oidc_repository_claim: github_repository, + github_oidc_ref_claim: f"refs/heads/{github_branch}", + github_oidc_workflow_ref_claim: github_workflow_refs, + }, + "StringLike": { + github_oidc_sub_claim: f"repo:{github_repository}:*", + }, + }, + } + ], + }, + sort_keys=True, + ) + ), + opts=pulumi.ResourceOptions(retain_on_delete=True), + tags=tags, +) + +github_actions_infrastructure_policy = aws.iam.Policy( + "github_actions_infrastructure_policy", + name="fund-github-actions-infrastructure-policy", + description=( + "Least-privilege policy for GitHub Actions infrastructure deployments." + ), + policy=pulumi.Output.all( + datamanager_secret_name, + portfoliomanager_secret_name, + shared_secret_name, + ).apply( + lambda args: json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + # These list/describe APIs are account-scoped and require wildcard + # resources. + { + "Sid": "ReadGlobalMetadata", + "Effect": "Allow", + "Action": [ + "sts:GetCallerIdentity", + "tag:GetResources", + "tag:GetTagKeys", + "tag:GetTagValues", + "iam:Get*", + "iam:List*", + "ec2:Describe*", + "ecs:Describe*", + "ecs:List*", + "elasticloadbalancing:Describe*", + "ecr:Describe*", + "ecr:ListTagsForResource", + "s3:GetBucketLocation", + "s3:ListAllMyBuckets", + "ssm:DescribeParameters", + "secretsmanager:ListSecrets", + "logs:Describe*", + "cloudwatch:Describe*", + "cloudwatch:Get*", + "sns:Get*", + "sns:List*", + "budgets:Describe*", + "budgets:ViewBudget", + "servicediscovery:Get*", + "servicediscovery:List*", + ], + "Resource": "*", + }, + # These control-plane APIs rely on generated identifiers and do not + # support practical resource-level scoping for stack create/update/ + # delete operations. + { + "Sid": "ManageEC2ECSELBBudgetsAndServiceDiscovery", + "Effect": "Allow", + "Action": [ + "ec2:*", + "ecs:*", + "elasticloadbalancing:*", + "budgets:*", + "servicediscovery:*", + ], + "Resource": "*", + }, + # CreateRepository/GetAuthorizationToken require wildcard resources. + { + "Sid": "CreateAndAuthenticateECRRepositories", + "Effect": "Allow", + "Action": [ + "ecr:CreateRepository", + "ecr:GetAuthorizationToken", + ], + "Resource": "*", + }, + { + "Sid": "ManageECRRepositories", + "Effect": "Allow", + "Action": "ecr:*", + "Resource": ( + f"arn:aws:ecr:{region}:{account_id}:repository/fund/*" + ), + }, + # CreateBucket requires wildcard resources. + { + "Sid": "CreateBuckets", + "Effect": "Allow", + "Action": "s3:CreateBucket", + "Resource": "*", + }, + { + "Sid": "ManageBuckets", + "Effect": "Allow", + "Action": "s3:*", + "Resource": [ + "arn:aws:s3:::fund-data-*", + "arn:aws:s3:::fund-data-*/*", + "arn:aws:s3:::fund-model-artifacts-*", + "arn:aws:s3:::fund-model-artifacts-*/*", + ], + }, + # CreateSecret requires wildcard resources before an ARN exists. + { + "Sid": "CreateSecrets", + "Effect": "Allow", + "Action": "secretsmanager:CreateSecret", + "Resource": "*", + }, + { + "Sid": "ManageConfiguredSecrets", + "Effect": "Allow", + "Action": "secretsmanager:*", + "Resource": [ + f"arn:aws:secretsmanager:{region}:{account_id}:secret:{args[0]}*", + f"arn:aws:secretsmanager:{region}:{account_id}:secret:{args[1]}*", + f"arn:aws:secretsmanager:{region}:{account_id}:secret:{args[2]}*", + ], + }, + { + "Sid": "ManageParameters", + "Effect": "Allow", + "Action": "ssm:*", + "Resource": ( + f"arn:aws:ssm:{region}:{account_id}:parameter/fund/*" + ), + }, + { + "Sid": "ManageLogGroups", + "Effect": "Allow", + "Action": "logs:*", + "Resource": [ + f"arn:aws:logs:{region}:{account_id}:log-group:/ecs/fund/*", + f"arn:aws:logs:{region}:{account_id}:log-group:/ecs/fund/*:*", + ], + }, + # Alarm mutation APIs require wildcard resources. + { + "Sid": "ManageAlarms", + "Effect": "Allow", + "Action": [ + "cloudwatch:DeleteAlarms", + "cloudwatch:ListTagsForResource", + "cloudwatch:PutMetricAlarm", + "cloudwatch:TagResource", + "cloudwatch:UntagResource", + ], + "Resource": "*", + }, + # CreateTopic requires wildcard resources. + { + "Sid": "CreateInfrastructureAlertsTopic", + "Effect": "Allow", + "Action": "sns:CreateTopic", + "Resource": "*", + }, + { + "Sid": "ManageInfrastructureAlertsTopic", + "Effect": "Allow", + "Action": "sns:*", + "Resource": [ + f"arn:aws:sns:{region}:{account_id}:fund-infrastructure-alerts", + f"arn:aws:sns:{region}:{account_id}:fund-infrastructure-alerts:*", + ], + }, + { + "Sid": "CreateGithubActionsOIDCProvider", + "Effect": "Allow", + "Action": "iam:CreateOpenIDConnectProvider", + "Resource": github_oidc_provider_arn, + }, + # CreateRole uses wildcard resources by API design. + { + "Sid": "CreateRoles", + "Effect": "Allow", + "Action": "iam:CreateRole", + "Resource": "*", + "Condition": { + "StringEquals": { + "iam:RoleName": [ + "fund-ecs-execution-role", + "fund-ecs-task-role", + github_actions_role_name, + sagemaker_execution_role_name, + ] + } + }, + }, + # CreatePolicy uses wildcard resources by API design. + { + "Sid": "CreatePolicies", + "Effect": "Allow", + "Action": "iam:CreatePolicy", + "Resource": "*", + "Condition": { + "StringLike": { + "iam:PolicyName": "fund-*", + } + }, + }, + # CreateServiceLinkedRole uses wildcard resources by API design. + { + "Sid": "CreateServiceLinkedRolesForStack", + "Effect": "Allow", + "Action": "iam:CreateServiceLinkedRole", + "Resource": "*", + "Condition": { + "StringEquals": { + "iam:AWSServiceName": [ + "ecs.amazonaws.com", + "elasticloadbalancing.amazonaws.com", + ] + } + }, + }, + { + "Sid": "ManageRoles", + "Effect": "Allow", + "Action": [ + "iam:AttachRolePolicy", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:PassRole", + "iam:TagRole", + "iam:UntagRole", + "iam:UpdateAssumeRolePolicy", + ], + "Resource": [ + f"arn:aws:iam::{account_id}:role/fund-ecs-execution-role", + f"arn:aws:iam::{account_id}:role/fund-ecs-task-role", + f"arn:aws:iam::{account_id}:role/{github_actions_role_name}", + f"arn:aws:iam::{account_id}:role/{sagemaker_execution_role_name}", + ], + "Condition": { + "ArnLikeIfExists": { + "iam:PolicyARN": [ + "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", + f"arn:aws:iam::{account_id}:policy/fund-*", + ] + }, + "StringLikeIfExists": { + "iam:PassedToService": [ + "ecs-tasks.amazonaws.com", + "ecs.amazonaws.com", + "sagemaker.amazonaws.com", + ] + }, + }, + }, + { + "Sid": "ManageInlineRolePolicies", + "Effect": "Allow", + "Action": [ + "iam:DeleteRolePolicy", + "iam:PutRolePolicy", + ], + "Resource": [ + f"arn:aws:iam::{account_id}:role/fund-ecs-execution-role", + f"arn:aws:iam::{account_id}:role/fund-ecs-task-role", + f"arn:aws:iam::{account_id}:role/{sagemaker_execution_role_name}", + ], + "Condition": { + "StringEquals": { + "iam:PolicyName": [ + "fund-ecs-execution-role-secrets-policy", + "fund-ecs-task-role-s3-policy", + "fund-ecs-task-role-ssm-policy", + "fund-sagemaker-s3-policy", + "fund-sagemaker-ecr-policy", + "fund-sagemaker-cloudwatch-policy", + ] + } + }, + }, + { + "Sid": "ManagePolicies", + "Effect": "Allow", + "Action": [ + "iam:CreatePolicyVersion", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:SetDefaultPolicyVersion", + "iam:TagPolicy", + "iam:UntagPolicy", + ], + "Resource": f"arn:aws:iam::{account_id}:policy/fund-*", + }, + { + "Sid": "ManageGithubActionsOIDCProvider", + "Effect": "Allow", + "Action": [ + "iam:AddClientIDToOpenIDConnectProvider", + "iam:DeleteOpenIDConnectProvider", + "iam:RemoveClientIDFromOpenIDConnectProvider", + "iam:TagOpenIDConnectProvider", + "iam:UntagOpenIDConnectProvider", + "iam:UpdateOpenIDConnectProviderThumbprint", + ], + "Resource": github_oidc_provider_arn, + }, + # Service-linked role teardown APIs are wildcard-resource only. + { + "Sid": "DeleteServiceLinkedRoles", + "Effect": "Allow", + "Action": [ + "iam:DeleteServiceLinkedRole", + "iam:GetServiceLinkedRoleDeletionStatus", + ], + "Resource": "*", + "Condition": { + "StringLikeIfExists": { + "iam:AWSServiceName": [ + "ecs.amazonaws.com", + "elasticloadbalancing.amazonaws.com", + ] + } + }, + }, + ], + }, + sort_keys=True, + ) + ), + opts=pulumi.ResourceOptions(retain_on_delete=True), + tags=tags, +) + +aws.iam.RolePolicyAttachment( + "github_actions_infrastructure_role_custom_policy", + role=github_actions_infrastructure_role.name, + policy_arn=github_actions_infrastructure_policy.arn, + opts=pulumi.ResourceOptions(retain_on_delete=True), +) + # IAM Role for ECS to perform infrastructure tasks execution_role = aws.iam.Role( "execution_role", - name="oscm-ecs-execution-role", + name="fund-ecs-execution-role", assume_role_policy=json.dumps( { "Version": "2012-10-17", @@ -579,7 +1224,8 @@ "Principal": {"Service": "ecs-tasks.amazonaws.com"}, } ], - } + }, + sort_keys=True, ), tags=tags, ) @@ -593,9 +1239,13 @@ # Allow ECS tasks to read secrets from Secrets Manager aws.iam.RolePolicy( "execution_role_secrets_policy", - name="oscm-ecs-execution-role-secrets-policy", + name="fund-ecs-execution-role-secrets-policy", role=execution_role.id, - policy=pulumi.Output.all(secret.arn).apply( + policy=pulumi.Output.all( + datamanager_secret.arn, + portfoliomanager_secret.arn, + shared_secret.arn, + ).apply( lambda args: json.dumps( { "Version": "2012-10-17", @@ -603,10 +1253,11 @@ { "Effect": "Allow", "Action": ["secretsmanager:GetSecretValue"], - "Resource": args[0], + "Resource": [account_id, args[0], args[1]], } ], - } + }, + sort_keys=True, ) ), ) @@ -615,7 +1266,7 @@ # IAM Role for ECS tasks to access AWS resources task_role = aws.iam.Role( "task_role", - name="oscm-ecs-task-role", + name="fund-ecs-task-role", assume_role_policy=json.dumps( { "Version": "2012-10-17", @@ -626,14 +1277,15 @@ "Principal": {"Service": "ecs-tasks.amazonaws.com"}, } ], - } + }, + sort_keys=True, ), tags=tags, ) aws.iam.RolePolicy( "task_role_s3_policy", - name="oscm-ecs-task-role-s3-policy", + name="fund-ecs-task-role-s3-policy", role=task_role.id, policy=pulumi.Output.all(data_bucket.arn, model_artifacts_bucket.arn).apply( lambda args: json.dumps( @@ -644,21 +1296,22 @@ "Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:ListBucket"], "Resource": [ + account_id, + f"{account_id}/*", args[0], f"{args[0]}/*", - args[1], - f"{args[1]}/*", ], } ], - } + }, + sort_keys=True, ) ), ) aws.iam.RolePolicy( "task_role_ssm_policy", - name="oscm-ecs-task-role-ssm-policy", + name="fund-ecs-task-role-ssm-policy", role=task_role.id, policy=json.dumps( { @@ -667,17 +1320,18 @@ { "Effect": "Allow", "Action": ["ssm:GetParameter", "ssm:GetParameters"], - "Resource": (f"arn:aws:ssm:{region}:{account_id}:parameter/oscm/*"), + "Resource": f"arn:aws:ssm:{region}:{account_id}:parameter/fund/*", } ], - } + }, + sort_keys=True, ), ) # SageMaker Execution Role for training jobs sagemaker_execution_role = aws.iam.Role( "sagemaker_execution_role", - name="oscm-sagemaker-execution-role", + name=sagemaker_execution_role_name, assume_role_policy=json.dumps( { "Version": "2012-10-17", @@ -688,14 +1342,16 @@ "Principal": {"Service": "sagemaker.amazonaws.com"}, } ], - } + }, + sort_keys=True, ), + opts=pulumi.ResourceOptions(retain_on_delete=True), tags=tags, ) aws.iam.RolePolicy( "sagemaker_s3_policy", - name="oscm-sagemaker-s3-policy", + name="fund-sagemaker-s3-policy", role=sagemaker_execution_role.id, policy=pulumi.Output.all(data_bucket.arn, model_artifacts_bucket.arn).apply( lambda args: json.dumps( @@ -711,21 +1367,23 @@ "s3:ListBucket", ], "Resource": [ + account_id, + f"{account_id}/*", args[0], f"{args[0]}/*", - args[1], - f"{args[1]}/*", ], } ], - } + }, + sort_keys=True, ) ), + opts=pulumi.ResourceOptions(retain_on_delete=True), ) aws.iam.RolePolicy( "sagemaker_ecr_policy", - name="oscm-sagemaker-ecr-policy", + name="fund-sagemaker-ecr-policy", role=sagemaker_execution_role.id, policy=pulumi.Output.all(equitypricemodel_trainer_repository.arn).apply( lambda args: json.dumps( @@ -747,14 +1405,16 @@ "Resource": "*", }, ], - } + }, + sort_keys=True, ) ), + opts=pulumi.ResourceOptions(retain_on_delete=True), ) aws.iam.RolePolicy( "sagemaker_cloudwatch_policy", - name="oscm-sagemaker-cloudwatch-policy", + name="fund-sagemaker-cloudwatch-policy", role=sagemaker_execution_role.id, policy=json.dumps( { @@ -771,27 +1431,29 @@ "Resource": "arn:aws:logs:*:*:log-group:/aws/sagemaker/*", } ], - } + }, + sort_keys=True, ), + opts=pulumi.ResourceOptions(retain_on_delete=True), ) datamanager_log_group = aws.cloudwatch.LogGroup( "datamanager_logs", - name="/ecs/oscm/datamanager", + name="/ecs/fund/datamanager", retention_in_days=7, tags=tags, ) portfoliomanager_log_group = aws.cloudwatch.LogGroup( "portfoliomanager_logs", - name="/ecs/oscm/portfoliomanager", + name="/ecs/fund/portfoliomanager", retention_in_days=7, tags=tags, ) equitypricemodel_log_group = aws.cloudwatch.LogGroup( "equitypricemodel_logs", - name="/ecs/oscm/equitypricemodel", + name="/ecs/fund/equitypricemodel", retention_in_days=7, tags=tags, ) @@ -808,14 +1470,15 @@ container_definitions=pulumi.Output.all( datamanager_log_group.name, datamanager_image_uri, - secret.arn, + datamanager_secret.arn, + shared_secret.arn, data_bucket.bucket, ).apply( lambda args: json.dumps( [ { "name": "datamanager", - "image": args[1], + "image": args[0], "portMappings": [{"containerPort": 8080, "protocol": "tcp"}], "environment": [ { @@ -824,7 +1487,7 @@ }, { "name": "AWS_S3_DATA_BUCKET_NAME", - "value": args[3], + "value": args[4], }, { "name": "ENVIRONMENT", @@ -834,7 +1497,7 @@ "secrets": [ { "name": "MASSIVE_API_KEY", - "valueFrom": f"{args[2]}:MASSIVE_API_KEY::", + "valueFrom": f"{args[1]}:MASSIVE_API_KEY::", }, { "name": "SENTRY_DSN", @@ -844,14 +1507,15 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": args[0], + "awslogs-group": account_id, "awslogs-region": region, "awslogs-stream-prefix": "datamanager", }, }, "essential": True, } - ] + ], + sort_keys=True, ) ), tags=tags, @@ -870,62 +1534,64 @@ portfoliomanager_log_group.name, service_discovery_namespace.name, portfoliomanager_image_uri, - secret.arn, + portfoliomanager_secret.arn, + shared_secret.arn, parameters.uncertainty_threshold.value, ).apply( lambda args: json.dumps( [ { "name": "portfoliomanager", - "image": args[2], + "image": args[1], "portMappings": [{"containerPort": 8080, "protocol": "tcp"}], "environment": [ { - "name": "OSCM_DATAMANAGER_BASE_URL", - "value": f"http://datamanager.{args[1]}:8080", + "name": "FUND_DATAMANAGER_BASE_URL", + "value": f"http://datamanager.{args[0]}:8080", }, { - "name": "OSCM_EQUITYPRICEMODEL_BASE_URL", - "value": f"http://equitypricemodel.{args[1]}:8080", + "name": "FUND_EQUITYPRICEMODEL_BASE_URL", + "value": f"http://equitypricemodel.{args[0]}:8080", }, { "name": "ENVIRONMENT", "value": "production", }, { - "name": "OSCM_UNCERTAINTY_THRESHOLD", - "value": args[4], + "name": "FUND_UNCERTAINTY_THRESHOLD", + "value": args[5], }, ], "secrets": [ { "name": "ALPACA_API_KEY_ID", - "valueFrom": f"{args[3]}:ALPACA_API_KEY_ID::", + "valueFrom": f"{args[2]}:ALPACA_API_KEY_ID::", }, { "name": "ALPACA_API_SECRET", - "valueFrom": f"{args[3]}:ALPACA_API_SECRET::", + "valueFrom": f"{args[2]}:ALPACA_API_SECRET::", }, { "name": "ALPACA_IS_PAPER", - "valueFrom": f"{args[3]}:ALPACA_IS_PAPER::", + "valueFrom": f"{args[2]}:ALPACA_IS_PAPER::", }, { "name": "SENTRY_DSN", - "valueFrom": f"{args[3]}:SENTRY_DSN::", + "valueFrom": f"{args[4]}:SENTRY_DSN::", }, ], "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": args[0], + "awslogs-group": account_id, "awslogs-region": region, "awslogs-stream-prefix": "portfoliomanager", }, }, "essential": True, } - ] + ], + sort_keys=True, ) ), tags=tags, @@ -945,22 +1611,22 @@ service_discovery_namespace.name, equitypricemodel_image_uri, model_artifacts_bucket.bucket, - secret.arn, + shared_secret.arn, ).apply( lambda args: json.dumps( [ { "name": "equitypricemodel", - "image": args[2], + "image": args[1], "portMappings": [{"containerPort": 8080, "protocol": "tcp"}], "environment": [ { - "name": "OSCM_DATAMANAGER_BASE_URL", - "value": f"http://datamanager.{args[1]}:8080", + "name": "FUND_DATAMANAGER_BASE_URL", + "value": f"http://datamanager.{args[0]}:8080", }, { "name": "AWS_S3_MODEL_ARTIFACTS_BUCKET", - "value": args[3], + "value": args[2], }, { "name": "ENVIRONMENT", @@ -980,14 +1646,15 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": args[0], + "awslogs-group": account_id, "awslogs-region": region, "awslogs-stream-prefix": "equitypricemodel", }, }, "essential": True, } - ] + ], + sort_keys=True, ) ), tags=tags, @@ -1031,7 +1698,7 @@ datamanager_service = aws.ecs.Service( "datamanager_service", - name="oscm-datamanager", + name="fund-datamanager", cluster=cluster.arn, task_definition=datamanager_task_definition.arn, desired_count=1, @@ -1057,7 +1724,7 @@ portfoliomanager_service = aws.ecs.Service( "portfoliomanager_service", - name="oscm-portfoliomanager", + name="fund-portfoliomanager", cluster=cluster.arn, task_definition=portfoliomanager_task_definition.arn, desired_count=1, @@ -1083,7 +1750,7 @@ equitypricemodel_service = aws.ecs.Service( "equitypricemodel_service", - name="oscm-equitypricemodel", + name="fund-equitypricemodel", cluster=cluster.arn, task_definition=equitypricemodel_task_definition.arn, desired_count=1, @@ -1109,7 +1776,7 @@ protocol = "https://" if acm_certificate_arn else "http://" -oscm_base_url = pulumi.Output.concat(protocol, alb.dns_name) +fund_base_url = pulumi.Output.concat(protocol, alb.dns_name) readme_content = """ # infrastructure @@ -1147,5 +1814,13 @@ "aws_ecr_equitypricemodel_trainer_image", equitypricemodel_trainer_image_uri ) pulumi.export("aws_iam_sagemaker_role_arn", sagemaker_execution_role.arn) -pulumi.export("oscm_base_url", oscm_base_url) -pulumi.export("readme", pulumi.Output.format(readme_content, oscm_base_url)) +pulumi.export( + "aws_iam_github_actions_infrastructure_role_arn", + github_actions_infrastructure_role.arn, +) +pulumi.export( + "aws_iam_github_actions_oidc_provider_arn", + github_actions_oidc_provider.arn, +) +pulumi.export("fund_base_url", fund_base_url) +pulumi.export("readme", pulumi.Output.format(readme_content, fund_base_url)) diff --git a/infrastructure/github_environment_runbook.md b/infrastructure/github_environment_runbook.md new file mode 100644 index 000000000..11e14d88d --- /dev/null +++ b/infrastructure/github_environment_runbook.md @@ -0,0 +1,51 @@ +# GitHub Pulumi Environment Runbook + +Required GitHub Environment: `pulumi` + +Required environment secrets for operations with Pulumi: + +- `AWS_IAM_INFRASTRUCTURE_ROLE_ARN` +- `AWS_REGION` +- `PULUMI_ACCESS_TOKEN` + +## Update `AWS_IAM_INFRASTRUCTURE_ROLE_ARN` from Pulumi output + +Run from repository root: + +```bash +cd infrastructure +pulumi stack select "$(pulumi org get-default)/fund/production" +role_arn="$(pulumi stack output aws_iam_github_actions_infrastructure_role_arn --stack production)" +cd .. +gh secret set AWS_IAM_INFRASTRUCTURE_ROLE_ARN --env pulumi --body "$role_arn" +``` + +## Update `AWS_REGION` from Pulumi stack config + +Run from repository root: + +```bash +cd infrastructure +pulumi stack select "$(pulumi org get-default)/fund/production" +region="$(pulumi config get aws:region --stack production --show-secrets)" +cd .. +gh secret set AWS_REGION --env pulumi --body "$region" +``` + +## Update `PULUMI_ACCESS_TOKEN` from Pulumi account + +Generate a new access token from your Pulumi account at + +Run from repository root: + +```bash +gh secret set PULUMI_ACCESS_TOKEN --env pulumi --body "" +``` + +## Verify all secrets + +Optional verification: + +```bash +gh secret list --env pulumi +``` diff --git a/infrastructure/parameters.py b/infrastructure/parameters.py index 1b8f4987f..69011ac83 100644 --- a/infrastructure/parameters.py +++ b/infrastructure/parameters.py @@ -1,4 +1,4 @@ -"""SSM Parameter Store configuration for oscm services. +"""SSM Parameter Store configuration for fund services. This module defines all SSM parameters used across the application. Parameters can be updated in AWS without redeploying services. @@ -8,7 +8,7 @@ import pulumi_aws as aws tags = { - "project": "oscm", + "project": "fund", "stack": pulumi.get_stack(), "manager": "pulumi", } @@ -16,7 +16,7 @@ # Portfolio Manager Configuration uncertainty_threshold = aws.ssm.Parameter( "ssm_uncertainty_threshold", - name="/oscm/portfoliomanager/uncertainty_threshold", + name="/fund/portfoliomanager/uncertainty_threshold", type="String", value="1.5", description="Maximum inter-quartile range for predictions to be considered valid", diff --git a/libraries/python/tests/test_infrastructure_configuration.py b/libraries/python/tests/test_infrastructure_configuration.py new file mode 100644 index 000000000..4719719c4 --- /dev/null +++ b/libraries/python/tests/test_infrastructure_configuration.py @@ -0,0 +1,102 @@ +import re +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +INFRASTRUCTURE_ENTRYPOINT_PATH = REPOSITORY_ROOT / "infrastructure" / "__main__.py" +PRODUCTION_STACK_CONFIG_PATH = ( + REPOSITORY_ROOT / "infrastructure" / "Pulumi.production.yaml" +) + + +def load_infrastructure_entrypoint() -> str: + return INFRASTRUCTURE_ENTRYPOINT_PATH.read_text(encoding="utf-8") + + +def load_production_stack_config() -> str: + return PRODUCTION_STACK_CONFIG_PATH.read_text(encoding="utf-8") + + +def test_production_stack_config_stores_region_as_plaintext() -> None: + production_stack_config = load_production_stack_config() + + assert "aws:region:" in production_stack_config + assert re.search(r"aws:region:\s+[a-z]+-[a-z]+-\d+", production_stack_config) + assert not re.search(r"aws:region:\s+secure:", production_stack_config) + + +def test_production_stack_config_stores_budget_alert_emails_as_secret() -> None: + production_stack_config = load_production_stack_config() + + assert "fund:budgetAlertEmailAddresses:" in production_stack_config + assert re.search( + r"fund:budgetAlertEmailAddresses:\s+secure:", production_stack_config + ) + + +def test_infrastructure_entrypoint_contains_oidc_claim_constraints() -> None: + infrastructure_entrypoint = load_infrastructure_entrypoint() + + assert ( + 'github_oidc_repository_claim = "token.actions.githubusercontent.com:' + 'repository"' in infrastructure_entrypoint + ) + assert ( + 'github_oidc_ref_claim = "token.actions.githubusercontent.com:ref"' + in infrastructure_entrypoint + ) + assert ( + 'github_oidc_workflow_ref_claim = "token.actions.githubusercontent.com:' + 'job_workflow_ref"' in infrastructure_entrypoint + ) + assert ( + 'github_oidc_sub_claim = "token.actions.githubusercontent.com:sub"' + in infrastructure_entrypoint + ) + + +def test_infrastructure_entrypoint_does_not_use_administrator_access() -> None: + infrastructure_entrypoint = load_infrastructure_entrypoint() + + assert "AdministratorAccess" not in infrastructure_entrypoint + + +def test_infrastructure_entrypoint_contains_nat_gateway_baseline_alarm() -> None: + infrastructure_entrypoint = load_infrastructure_entrypoint() + + assert '"nat_gateway_bytes_out_to_destination_alarm"' in infrastructure_entrypoint + assert 'metric_name="BytesOutToDestination"' in infrastructure_entrypoint + assert "threshold=500_000_000" in infrastructure_entrypoint + assert "period=3600" in infrastructure_entrypoint + assert "evaluation_periods=2" in infrastructure_entrypoint + + +def test_infrastructure_entrypoint_contains_s3_bucket_encryption_resources() -> None: + infrastructure_entrypoint = load_infrastructure_entrypoint() + + assert '"data_bucket_encryption"' in infrastructure_entrypoint + assert '"model_artifacts_bucket_encryption"' in infrastructure_entrypoint + + +def test_infrastructure_entrypoint_attaches_custom_github_actions_policy() -> None: + infrastructure_entrypoint = load_infrastructure_entrypoint() + + assert '"github_actions_infrastructure_policy"' in infrastructure_entrypoint + assert ( + '"github_actions_infrastructure_role_custom_policy"' + in infrastructure_entrypoint + ) + + +def test_infrastructure_entrypoint_contains_s3_public_access_block_resources() -> None: + infrastructure_entrypoint = load_infrastructure_entrypoint() + + assert '"data_bucket_public_access_block"' in infrastructure_entrypoint + assert '"model_artifacts_bucket_public_access_block"' in infrastructure_entrypoint + + +def test_infrastructure_entrypoint_scopes_oidc_provider_creation_statement() -> None: + infrastructure_entrypoint = load_infrastructure_entrypoint() + + assert '"CreateGithubActionsOIDCProvider"' in infrastructure_entrypoint + assert '"Resource": github_oidc_provider_arn' in infrastructure_entrypoint + assert '"CreateIamResourcesForOscmStack"' not in infrastructure_entrypoint diff --git a/maskfile.md b/maskfile.md index 3d38cbb59..c35bd5af9 100644 --- a/maskfile.md +++ b/maskfile.md @@ -68,7 +68,7 @@ if [ -z "$aws_region" ]; then exit 1 fi -image_reference="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com/oscmcompany/${application_name}-${stage_name}" +image_reference="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com/fund/${application_name}-${stage_name}" cache_reference="${image_reference}:buildcache" # Use GHA backend for caching when running in GitHub Actions @@ -87,7 +87,7 @@ echo "Setting up Docker Buildx" if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "Using buildx builder configured by docker/setup-buildx-action" else - docker buildx create --use --name oscm-builder 2>/dev/null || docker buildx use oscm-builder || (echo "Using default buildx builder" && docker buildx use default) + docker buildx create --use --name fund-builder 2>/dev/null || docker buildx use fund-builder || (echo "Using default buildx builder" && docker buildx use default) fi echo "Logging into ECR (to pull cache if available)" @@ -125,7 +125,7 @@ if [ -z "$aws_region" ]; then exit 1 fi -repository_name="oscm/${application_name}-${stage_name}" +repository_name="fund/${application_name}-${stage_name}" image_reference="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com/${repository_name}" commit_hash=$(git rev-parse --short HEAD) @@ -173,14 +173,14 @@ cd infrastructure/ echo "Launching infrastructure" -organization_name=$(pulumi whoami) +organization_name=$(pulumi org get-default) if [ -z "${organization_name}" ]; then echo "Unable to determine Pulumi organization name - ensure you are logged in" exit 1 fi -pulumi stack select ${organization_name}/oscmcompany/production --create +pulumi stack select ${organization_name}/fund/production --create pulumi up --diff --yes @@ -191,7 +191,10 @@ cluster=$(pulumi stack output aws_ecs_cluster_name --stack production 2>/dev/nul if [ -z "$cluster" ]; then echo "Cluster not found - skipping service deployments (initial setup)" else - for service in oscmcompany-datamanager oscmcompany-portfoliomanager oscmcompany-equitypricemodel; do + # Note: Service names use 'fund' prefix matching the Pulumi project name. + # These must exactly match the ECS service names created by the infrastructure code. + # The AWS account provides environment context (one account = one environment). + for service in fund-datamanager fund-portfoliomanager fund-equitypricemodel; do echo "Checking if $service exists and is ready" # Wait up to 60 seconds for service to be active @@ -235,7 +238,7 @@ else done echo "Stack update complete - ECS is performing rolling deployments" - echo "Monitor progress: aws ecs describe-services --cluster $cluster --services oscmcompany-portfoliomanager" + echo "Monitor progress: aws ecs describe-services --cluster $cluster --services fund-portfoliomanager" fi echo "Infrastructure launched successfully" @@ -272,10 +275,10 @@ echo "Invoking ${application_name} service" cd infrastructure/ -base_url=$(pulumi stack output psf_base_url --stack production 2>/dev/null || echo "") +base_url=$(pulumi stack output fund_base_url --stack production 2>/dev/null || echo "") if [ -z "$base_url" ]; then - echo "psf_base_url not found - infrastructure might not be deployed" + echo "fund_base_url not found - infrastructure might not be deployed" exit 1 fi @@ -644,7 +647,7 @@ if [ "${data_type}" = "categories" ]; then echo "Syncing equity categories" export MASSIVE_API_KEY=$(aws secretsmanager get-secret-value \ - --secret-id oscmcompany/production/environment_variables \ + --secret-id fund/production/datamanager \ --query 'SecretString' \ --output text | jq -r '.MASSIVE_API_KEY')