diff --git a/.mise.toml b/.mise.toml index 9cb7bd0a0..e3d4fdc16 100644 --- a/.mise.toml +++ b/.mise.toml @@ -104,3 +104,9 @@ set -e cd infrastructure uv run pulumi down --yes --stack pocketsizefund/pocketsizefund/production """ + +[tasks."cli:datamanager:authorize"] +description = "Authorize the CLI with AWS credentials" +run = """ +aws iam attach-user-policy --user-name {{arg(user-name="user-name")}} --policy-arn ${{pulumi stack output DATAMANAGER_API_ACCESS_POLICY_ARN}} +""" diff --git a/application/datamanager/pyproject.toml b/application/datamanager/pyproject.toml index b9e1f5df8..8c877b256 100644 --- a/application/datamanager/pyproject.toml +++ b/application/datamanager/pyproject.toml @@ -25,6 +25,4 @@ requires = ["hatchling"] build-backend = "hatchling.build" [dependency-groups] -dev = [ - "behave>=1.2.6", -] +dev = ["behave>=1.2.6"] diff --git a/cli/datamanager.py b/cli/datamanager.py new file mode 100644 index 000000000..9664e4ea3 --- /dev/null +++ b/cli/datamanager.py @@ -0,0 +1,138 @@ +import argparse +import json +from datetime import datetime, timedelta +from urllib.parse import urlparse +from zoneinfo import ZoneInfo + +import boto3 +import requests +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from loguru import logger + + +def sign_request( + method: str, + url: str, + data: dict | None = None, + region: str = "us-east-1", +) -> dict: + session = boto3.Session() + credentials = session.get_credentials() + + request_payload = { + "method": method, + "url": url, + "headers": { + "Content-Type": "application/json", + "Host": urlparse(url).netloc, + }, + } + + if data: + request_payload["data"] = json.dumps(data) + request_payload["headers"]["Content-Type"] = "application/json" + + request = AWSRequest(**request_payload) + SigV4Auth(credentials, "execute-api", region).add_auth(request) + + return { + "method": method, + "url": url, + "headers": dict(request.headers), + "data": request.body, + } + + +def get_health(api_url: str, region: str) -> dict: + signed_request = sign_request(method="GET", url=f"{api_url}/health", region=region) + response = requests.request(**signed_request, timeout=10) + response.raise_for_status() + return response.json() + + +def get_equity_bars(api_url: str, start_date: str, end_date: str, region: str) -> dict: + url = f"{api_url}/equity-bars?start_date={start_date}&end_date={end_date}" + signed_request = sign_request(method="GET", url=url, region=region) + response = requests.request(**signed_request, timeout=30) + response.raise_for_status() + return response.json() + + +def get_metrics(api_url: str, region: str) -> dict: + signed_request = sign_request(method="GET", url=f"{api_url}/metrics", region=region) + response = requests.request(**signed_request, timeout=30) + response.raise_for_status() + return response.json() + + +def fetch_equity_bars(api_url: str, fetch_date: str, region: str) -> dict: + data = {"date": fetch_date} + signed_request = sign_request( + method="POST", + url=f"{api_url}/equity-bars/fetch", + data=data, + region=region, + ) + response = requests.request(**signed_request, timeout=60) + response.raise_for_status() + return response.json() + + +def main() -> None: + parser = argparse.ArgumentParser(description="PocketSizeFund CLI Example") + parser.add_argument("--api-url", required=True, help="API Gateway URL") + parser.add_argument("--region", default="us-east-1", help="AWS region") + parser.add_argument( + "--command", choices=["health", "bars", "metrics", "fetch"], default="health" + ) + parser.add_argument("--start-date", help="Start date for bars (YYYY-MM-DD)") + parser.add_argument("--end-date", help="End date for bars (YYYY-MM-DD)") + parser.add_argument("--fetch-date", help="Date to fetch bars for (YYYY-MM-DD)") + + args = parser.parse_args() + + eastern_timezone = ZoneInfo("America/New_York") + + today = datetime.now(tz=eastern_timezone).date() + try: + if args.command == "health": + result = get_health(args.api_url, args.region) + logger.info(json.dumps(result, indent=2)) + + elif args.command == "metrics": + result = get_metrics(args.api_url, args.region) + logger.info(json.dumps(result, indent=2)) + + elif args.command == "bars": + if not args.start_date or not args.end_date: + # Default to last 7 days + end_date = today + start_date = end_date - timedelta(days=7) + start_date_str = start_date.isoformat() + end_date_str = end_date.isoformat() + else: + start_date_str = args.start_date + end_date_str = args.end_date + + result = get_equity_bars( + args.api_url, start_date_str, end_date_str, args.region + ) + records = result.get("records", []) + logger.info(f"found {len(records)} records") + if records: + logger.info(json.dumps(records[0], indent=2)) + + elif args.command == "fetch": + fetch_date = args.fetch_date or today.isoformat() + result = fetch_equity_bars(args.api_url, fetch_date, args.region) + logger.info(json.dumps(result, indent=2)) + + except requests.exceptions.HTTPError as e: + logger.error(f"http error response: {e.response.text}") + except Exception as e: # noqa: BLE001 + logger.error(f"error: {e}") + + +if __name__ == "__main__": + main() diff --git a/cli/pyproject.toml b/cli/pyproject.toml new file mode 100644 index 000000000..3aac42fa3 --- /dev/null +++ b/cli/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "cli" +version = "0.1.0" +requires-python = "==3.12.10" +dependencies = [ + "requests>=2.31.0", + "loguru>=0.7.3", + "boto3>=1.38.23", + "botocore>=1.38.23", +] diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index fc06a120b..900514fa3 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -1,6 +1,7 @@ import tomllib from pathlib import Path +import pulumi from cluster import ( create_kubernetes_cluster, create_kubernetes_provider, @@ -8,6 +9,13 @@ ) from environment_variables import create_environment_variables from images import build_image +from ingress import ( + create_alb_controller, + create_alb_controller_role, + create_api_access_policy, + create_api_gateway_with_auth, + create_service_ingress, +) from keys import create_duckdb_user_access_key from monitors import create_prometheus_scraper from publishers_subscribers import ( @@ -42,6 +50,14 @@ root_user_arn=configuration.require_secret("AWS_EKS_IAM_ROOT_USER_ARN"), ) +alb_controller_role = create_alb_controller_role(kubernetes_cluster) + +alb_controller = create_alb_controller( + kubernetes_provider=kubernetes_provider, + cluster=kubernetes_cluster, + alb_controller_role=alb_controller_role, +) + knative_serving_core = create_knative_serving_core(kubernetes_provider) knative_eventing_core = create_knative_eventing_core(kubernetes_provider) @@ -140,3 +156,28 @@ workspace_arn=configuration.require_secret("AWS_PROMETHEUS_WORKSPACE_ARN"), cluster=kubernetes_cluster, ) + +datamanager_ingress = create_service_ingress( + kubernetes_provider=kubernetes_provider, + service_name="datamanager", + cluster=kubernetes_cluster, + depends_on=[alb_controller, datamanager_service], +) + +datamanager_alb_url = datamanager_ingress.status.load_balancer.ingress[ + 0 +].hostname.apply(lambda hostname: f"http://{hostname}") + +datamanager_api = create_api_gateway_with_auth( + service_name="datamanager", + target_url=datamanager_alb_url, +) + +datamanager_api_access_policy = create_api_access_policy( + api_gateway=datamanager_api, + service_name="datamanager", +) + +pulumi.export("DATAMANAGER_ALB_URL", datamanager_alb_url) +pulumi.export("DATAMANAGER_API_GATEWAY_URL", datamanager_api.api_endpoint) +pulumi.export("DATAMANAGER_API_ACCESS_POLICY_ARN", datamanager_api_access_policy.arn) diff --git a/infrastructure/ingress.py b/infrastructure/ingress.py new file mode 100644 index 000000000..9ea08ab39 --- /dev/null +++ b/infrastructure/ingress.py @@ -0,0 +1,247 @@ +import json + +import pulumi +import pulumi_aws as aws +import pulumi_eks as eks +import pulumi_kubernetes as k8s +from pulumi.config import Config +from tags import common_tags + +configuration = Config() + + +def create_alb_controller_role(cluster: eks.Cluster) -> aws.iam.Role: + policy_document = pulumi.Output.all( + oidc_provider_arn=cluster.core.oidc_provider.arn, + oidc_provider_url=cluster.core.oidc_provider.url, + ).apply( + lambda args: json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Federated": args["oidc_provider_arn"]}, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + f"{args['oidc_provider_url'].replace('https://', '')}:sub": "system:serviceaccount:kube-system:aws-load-balancer-controller", # noqa: E501 + f"{args['oidc_provider_url'].replace('https://', '')}:aud": "sts.amazonaws.com", # noqa: E501 + } + }, + } + ], + } + ) + ) + + alb_controller_role = aws.iam.Role( + resource_name="pocketsizefund-alb-controller-role", + name="pocketsizefund-alb-controller-role", + assume_role_policy=policy_document, + tags=common_tags, + ) + + aws.iam.RolePolicyAttachment( + resource_name="pocketsizefund-alb-controller-policy", + role=alb_controller_role.name, + policy_arn="arn:aws:iam::aws:policy/ElasticLoadBalancingFullAccess", + ) + + return alb_controller_role + + +def create_alb_controller( + kubernetes_provider: k8s.Provider, + cluster: eks.Cluster, + alb_controller_role: aws.iam.Role, +) -> k8s.helm.v3.Release: + alb_controller_service_account = k8s.core.v1.ServiceAccount( + resource_name="pocketsizefund-alb-controller-service-account", + metadata=k8s.meta.v1.ObjectMetaArgs( + name="aws-load-balancer-controller", + namespace="kube-system", + annotations={ + "eks.amazonaws.com/role-arn": alb_controller_role.arn, + }, + ), + opts=pulumi.ResourceOptions(provider=kubernetes_provider), + ) + + return k8s.helm.v3.Release( + resource_name="pocketsizefund-alb-controller", + name="aws-load-balancer-controller", + chart="aws-load-balancer-controller", + namespace="kube-system", + repository_opts=k8s.helm.v3.RepositoryOptsArgs( + repo="https://aws.github.io/eks-charts" + ), + values={ + "clusterName": cluster.name, # type: ignore + "serviceAccount": { + "create": False, + "name": "aws-load-balancer-controller", + }, + "region": configuration.get("aws:region") or "us-east-1", + "vpcId": cluster.eks_cluster.vpc_config.vpc_id, + }, + opts=pulumi.ResourceOptions( + provider=kubernetes_provider, + depends_on=[alb_controller_service_account], + ), + ) + + +def create_service_ingress( + kubernetes_provider: k8s.Provider, + service_name: str, + cluster: eks.Cluster, + certificate_arn: pulumi.Output[str] | None = None, + depends_on: list[pulumi.Resource] | None = None, +) -> k8s.networking.v1.Ingress: + annotations = { + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/target-type": "pod", + "alb.ingress.kubernetes.io/load-balancer-name": f"pocketsizefund-{service_name}", # noqa: E501 + "alb.ingress.kubernetes.io/subnets": cluster.public_subnet_ids.apply( # type: ignore + lambda subnets: ",".join(subnets) + ), + } + + if certificate_arn: + annotations.update( + { + "alb.ingress.kubernetes.io/listen-ports": '[{"HTTP": 80}, {"HTTPS": 443}]', # noqa: E501 + "alb.ingress.kubernetes.io/certificate-arn": certificate_arn, + "alb.ingress.kubernetes.io/ssl-redirect": "443", + } + ) + else: + annotations["alb.ingress.kubernetes.io/listen-ports"] = '[{"HTTP": 80}]' + + return k8s.networking.v1.Ingress( + resource_name=f"pocketsizefund-{service_name}-ingress", + metadata=k8s.meta.v1.ObjectMetaArgs( + name=f"{service_name}-ingress", + namespace="default", + annotations=annotations, + ), + spec=k8s.networking.v1.IngressSpecArgs( + rules=[ + k8s.networking.v1.IngressRuleArgs( + http=k8s.networking.v1.HTTPIngressRuleValueArgs( + paths=[ + k8s.networking.v1.HTTPIngressPathArgs( + path="/", + path_type="Prefix", + backend=k8s.networking.v1.IngressBackendArgs( + service=k8s.networking.v1.IngressServiceBackendArgs( + name=service_name, + port=k8s.networking.v1.ServiceBackendPortArgs( + number=80 + ), + ) + ), + ) + ] + ) + ) + ] + ), + opts=pulumi.ResourceOptions( + provider=kubernetes_provider, + depends_on=depends_on or [], + ), + ) + + +def create_self_signed_certificate() -> aws.acm.Certificate: + return aws.acm.Certificate( + resource_name="pocketsizefund-self-signed-cert", + domain_name="*.amazonaws.com", + validation_method="DNS", + subject_alternative_names=["*.elb.amazonaws.com"], + tags=common_tags, + ) + + +def create_api_gateway_with_auth( + service_name: str, + target_url: pulumi.Output[str], +) -> aws.apigatewayv2.Api: + api = aws.apigatewayv2.Api( + resource_name=f"pocketsizefund-{service_name}-api", + name=f"pocketsizefund-{service_name}", + protocol_type="HTTP", + cors_configuration=aws.apigatewayv2.ApiCorsConfigurationArgs( + allow_origins=["*"], # reduce allowed origins in production + allow_methods=["GET", "POST", "DELETE"], + allow_headers=["Content-Type", "Authorization", "Host"], + max_age=86400, + ), + tags=common_tags, + ) + + integration = aws.apigatewayv2.Integration( + resource_name=f"pocketsizefund-{service_name}-integration", + api_id=api.id, + integration_type="HTTP_PROXY", + integration_method="ANY", + integration_uri=target_url, + connection_type="INTERNET", + ) + + aws.apigatewayv2.Route( + resource_name=f"pocketsizefund-{service_name}-route", + api_id=api.id, + route_key="ANY /{proxy+}", + target=integration.id.apply( + lambda integration_id: f"integrations/{integration_id}" + ), + authorization_type="AWS_IAM", + ) + + aws.apigatewayv2.Stage( + resource_name=f"pocketsizefund-{service_name}-stage", + api_id=api.id, + name="$default", + auto_deploy=True, + default_route_settings=aws.apigatewayv2.StageDefaultRouteSettingsArgs( + throttling_burst_limit=100, + throttling_rate_limit=50, + ), + tags=common_tags, + ) + + pulumi.export(f"{service_name.upper()}_API_GATEWAY_URL", api.api_endpoint) + + return api + + +def create_api_access_policy( + api_gateway: aws.apigatewayv2.Api, + service_name: str, +) -> aws.iam.Policy: + policy_document = api_gateway.arn.apply( + lambda arn: json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["execute-api:Invoke"], + "Resource": f"{arn}/$default/*", + } + ], + } + ) + ) + + return aws.iam.Policy( + resource_name=f"pocketsizefund-{service_name}-api-access", + name=f"pocketsizefund-{service_name}-api-access", + description=f"Policy for accessing {service_name} API Gateway", + policy=policy_document, + tags=common_tags, + ) diff --git a/infrastructure/pyproject.toml b/infrastructure/pyproject.toml index 1dfecf61f..579afc09e 100644 --- a/infrastructure/pyproject.toml +++ b/infrastructure/pyproject.toml @@ -12,4 +12,7 @@ dependencies = [ "pulumi-docker-build>=0.0.12", "pulumi-kubernetes>=4.23.0", "loguru>=0.7.3", + "boto3>=1.38.23", + "botocore>=1.38.23", + "requests>=2.32.0", # fixes CVE-2024-35195 ] diff --git a/pyproject.toml b/pyproject.toml index 370d3ec8a..cb47eb985 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ members = [ "application/positionmanager", "application/predictionengine", "workflows", + "cli", ] [tool.uv.sources] diff --git a/uv.lock b/uv.lock index 349157a9c..56e9ca0f7 100644 --- a/uv.lock +++ b/uv.lock @@ -4,6 +4,7 @@ requires-python = "==3.12.10" [manifest] members = [ + "cli", "datamanager", "infrastructure", "pocketsizefund", @@ -359,6 +360,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/89/d4caad4f3851e317e423359d31d9862b08321fd61db6281edb80455ca562/clarabel-0.11.0-cp39-abi3-win_amd64.whl", hash = "sha256:6b804f99741719531b7a8ce7e44c8162b4cac7782334ea48dad0c48d6904f7d7", size = 892575, upload-time = "2025-05-23T17:59:28.389Z" }, ] +[[package]] +name = "cli" +version = "0.1.0" +source = { virtual = "cli" } +dependencies = [ + { name = "boto3" }, + { name = "botocore" }, + { name = "loguru" }, + { name = "requests" }, +] + +[package.metadata] +requires-dist = [ + { name = "boto3", specifier = ">=1.34.0" }, + { name = "botocore", specifier = ">=1.34.0" }, + { name = "loguru", specifier = ">=0.7.3" }, + { name = "requests", specifier = ">=2.31.0" }, +] + [[package]] name = "click" version = "8.2.1" @@ -1021,6 +1041,8 @@ name = "infrastructure" version = "20250709.1" source = { virtual = "infrastructure" } dependencies = [ + { name = "boto3" }, + { name = "botocore" }, { name = "loguru" }, { name = "pulumi" }, { name = "pulumi-aws" }, @@ -1030,10 +1052,13 @@ dependencies = [ { name = "pulumi-eks" }, { name = "pulumi-kubernetes" }, { name = "pulumi-std" }, + { name = "requests" }, ] [package.metadata] requires-dist = [ + { name = "boto3", specifier = ">=1.34.0" }, + { name = "botocore", specifier = ">=1.34.0" }, { name = "loguru", specifier = ">=0.7.3" }, { name = "pulumi", specifier = ">=3.169.0" }, { name = "pulumi-aws", specifier = ">=6.0.0" }, @@ -1043,6 +1068,7 @@ requires-dist = [ { name = "pulumi-eks", specifier = ">=3.9.1" }, { name = "pulumi-kubernetes", specifier = ">=4.23.0" }, { name = "pulumi-std", specifier = ">=2.2.0" }, + { name = "requests", specifier = ">=2.31.0" }, ] [[package]]