From 2e9be9256c8df1ef7fb2c5060f2b7be285f750f6 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Mon, 2 Jun 2025 13:09:50 -0400 Subject: [PATCH 01/41] update infra infra updates --- application/datamanager/pyproject.toml | 5 + .../datamanager/src/datamanager/main.py | 2 +- application/positionmanager/pyproject.toml | 1 + application/predictionengine/compose.yaml | 20 +++ ...ure_temporal_fusion_transformer.safetensor | 0 application/predictionengine/pyproject.toml | 2 + .../src/predictionengine/main.py | 2 +- infrastructure/__main__.py | 73 ++++++++- infrastructure/cloud_run.py | 148 ------------------ infrastructure/environment_variables.py | 38 +++++ infrastructure/pyproject.toml | 2 +- infrastructure/services.py | 72 +++++++++ pyproject.toml | 2 +- uv.lock | 18 ++- 14 files changed, 228 insertions(+), 157 deletions(-) create mode 100644 application/predictionengine/compose.yaml create mode 100644 application/predictionengine/miniature_temporal_fusion_transformer.safetensor delete mode 100644 infrastructure/cloud_run.py create mode 100644 infrastructure/environment_variables.py create mode 100644 infrastructure/services.py diff --git a/application/datamanager/pyproject.toml b/application/datamanager/pyproject.toml index a18dd7b95..64ecf54dd 100644 --- a/application/datamanager/pyproject.toml +++ b/application/datamanager/pyproject.toml @@ -22,3 +22,8 @@ packages = ["datamanager"] [build-system] requires = ["hatchling"] build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "behave>=1.2.6", +] diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index d9af43d85..dceaa9c88 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -39,7 +39,7 @@ def bars_query(*, bucket: str, start_date: date, end_date: date) -> str: @asynccontextmanager -async def lifespan(app: FastAPI) -> AsyncGenerator[None]: +async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: app.state.settings = Settings() app.state.bucket = storage.Client(os.getenv("GCP_PROJECT")).bucket( app.state.settings.gcp.bucket.name diff --git a/application/positionmanager/pyproject.toml b/application/positionmanager/pyproject.toml index c2d2de6f1..0dac627de 100644 --- a/application/positionmanager/pyproject.toml +++ b/application/positionmanager/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "pandas>=2.1.0", "pyportfolioopt>=1.5.6", "ecos>=2.0.14", + "prometheus-fastapi-instrumentator>=7.1.0", ] [tool.hatch.build.targets.wheel] diff --git a/application/predictionengine/compose.yaml b/application/predictionengine/compose.yaml new file mode 100644 index 000000000..a88c6b622 --- /dev/null +++ b/application/predictionengine/compose.yaml @@ -0,0 +1,20 @@ +name: predictionengine integration tests + +services: + predictionengine: + build: + context: . + dockerfile: Dockerfile + ports: + - 8080:8080 + environment: + - DATAMANAGER_BASE_URL=${DATAMANAGER_BASE_URL} + volumes: + - ./:/app/datamanager + - ~/.config/gcloud/application_default_credentials.json:/root/.config/gcloud/application_default_credentials.json:ro + healthcheck: + test: ["CMD", "curl", "-f", "http://0.0.0.0:8080/health"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 1s diff --git a/application/predictionengine/miniature_temporal_fusion_transformer.safetensor b/application/predictionengine/miniature_temporal_fusion_transformer.safetensor new file mode 100644 index 000000000..e69de29bb diff --git a/application/predictionengine/pyproject.toml b/application/predictionengine/pyproject.toml index 930c98b58..ed10cef5d 100644 --- a/application/predictionengine/pyproject.toml +++ b/application/predictionengine/pyproject.toml @@ -10,6 +10,8 @@ dependencies = [ "polars>=1.29.0", "category-encoders>=2.8.1", "requests>=2.31.0", + "prometheus-fastapi-instrumentator>=7.1.0", + "loguru>=0.7.3", ] [tool.hatch.build.targets.wheel] diff --git a/application/predictionengine/src/predictionengine/main.py b/application/predictionengine/src/predictionengine/main.py index 20075aee5..3c1c8d4b8 100644 --- a/application/predictionengine/src/predictionengine/main.py +++ b/application/predictionengine/src/predictionengine/main.py @@ -14,7 +14,7 @@ @asynccontextmanager -async def lifespan(app: FastAPI) -> AsyncGenerator[None]: +async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: datamanager_base_url = os.getenv("DATAMANAGER_BASE_URL", "") app.state.datamanager_base_url = datamanager_base_url diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index 27f821ae4..c93d24409 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -1,5 +1,72 @@ +import base64 +from pulumi_gcp import pubsub, cloudscheduler +from project import platform_service_account +from environment_variables import ( + create_environment_variable, + ALPACA_API_KEY_ID, + ALPACA_API_SECRET_KEY, + GCP_PROJECT, + DATA_BUCKET, + DUCKDB_ACCESS_KEY, + DUCKDB_SECRET, + POLYGON_API_KEY, +) +from services import create_service import topics # noqa: F401 import buckets # noqa: F401 -import images # noqa: F401 -import cloud_run # noqa: F401 -import monitoring # noqa: F401 + + +datamanager_service = create_service( + name="datamanager", + envs=[ + ALPACA_API_KEY_ID, + ALPACA_API_SECRET_KEY, + GCP_PROJECT, + DATA_BUCKET, + DUCKDB_ACCESS_KEY, + DUCKDB_SECRET, + POLYGON_API_KEY, + ], +) + +DATAMANAGER_BASE_URL = create_environment_variable( + "DATAMANAGER_BASE_URL", datamanager_service.statuses[0].url +) + +predicitonengine_service = create_service( + "predictionengine", envs=[DATAMANAGER_BASE_URL] +) + + +positionmanager_service = create_service( + "positionmanager", + envs=[ + ALPACA_API_KEY_ID, + ALPACA_API_SECRET_KEY, + DATAMANAGER_BASE_URL, + # MINIMUM_PORTFOLIO_TICKERS, # 20 + # MAXIMUM_PORTFOLIO_TICKERS, # 20 + ], +) + + +datamanager_subscription = pubsub.Subscription( + "datamanager-subscription", + topic=topics.datamanager_ping.id, + push_config=pubsub.SubscriptionPushConfigArgs( + push_endpoint=datamanager_service.statuses[0].url, + oidc_token=pubsub.SubscriptionPushConfigOidcTokenArgs( + service_account_email=platform_service_account.email + ), + ), +) + +datamanager_job = cloudscheduler.Job( + "datamanager-job", + schedule="0 0 * * *", + time_zone="UTC", + pubsub_target=cloudscheduler.JobPubsubTargetArgs( + topic_name=topics.datamanager_ping.id, + data=base64.b64encode(b"{}").decode("utf-8"), + ), +) diff --git a/infrastructure/cloud_run.py b/infrastructure/cloud_run.py deleted file mode 100644 index 5daecd9a6..000000000 --- a/infrastructure/cloud_run.py +++ /dev/null @@ -1,148 +0,0 @@ -from pulumi_gcp import cloudrun, pubsub, cloudscheduler -import base64 -from pulumi import Config -import project -import topics -import buckets - -config = Config() - -alpaca_api_key = config.require_secret("ALPACA_API_KEY_ID") -alpaca_api_secret = config.require_secret("ALPACA_API_SECRET_KEY") -duckdb_access_key = config.require_secret("DUCKDB_ACCESS_KEY") -duckdb_secret = config.require_secret("DUCKDB_SECRET") - - -datamanager_service = cloudrun.Service( - "datamanager", - location=project.REGION, - template=cloudrun.ServiceTemplateArgs( - spec=cloudrun.ServiceTemplateSpecArgs( - service_account_name=project.platform_service_account.email, - containers=[ - cloudrun.ServiceTemplateSpecContainerArgs( - image="pocketsizefund/datamanager:latest", - envs=[ - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="ALPACA_API_KEY_ID", - value=alpaca_api_key, - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="ALPACA_API_SECRET_KEY", - value=alpaca_api_secret, - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="GCP_PROJECT", - value=project.PROJECT, - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="DATA_BUCKET", - value=buckets.production_data_bucket.name, - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="DUCKDB_ACCESS_KEY", value=duckdb_access_key - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="DUCKDB_SECRET", - value=duckdb_secret, - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="DATA_BUCKET", - value=buckets.production_data_bucket.name, - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="DUCKDB_ACCESS_KEY", value=duckdb_access_key - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="DUCKDB_SECRET", - value=duckdb_secret, - ), - ], - ) - ], - ), - ), -) - -subscription = pubsub.Subscription( - "datamanager-subscription", - topic=topics.datamanager_ping.id, - push_config=pubsub.SubscriptionPushConfigArgs( - push_endpoint=datamanager_service.statuses[0].url, - oidc_token=pubsub.SubscriptionPushConfigOidcTokenArgs( - service_account_email=project.platform_service_account.email - ), - ), -) - -job = cloudscheduler.Job( - "datamanager-job", - schedule="0 0 * * *", - time_zone="UTC", - pubsub_target=cloudscheduler.JobPubsubTargetArgs( - topic_name=topics.datamanager_ping.id, - data=base64.b64encode(b"{}").decode("utf-8"), - ), -) - -positionmanager_service = cloudrun.Service( - "positionmanager", - location=project.REGION, - template=cloudrun.ServiceTemplateArgs( - spec=cloudrun.ServiceTemplateSpecArgs( - service_account_name=project.platform_service_account.email, - containers=[ - cloudrun.ServiceTemplateSpecContainerArgs( - image="pocketsizefund/positionmanager:latest", - envs=[ - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="ALPACA_API_KEY", - value=config.require_secret("ALPACA_API_KEY"), - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="ALPACA_API_SECRET", - value=config.require_secret("ALPACA_API_SECRET"), - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="ALPACA_PAPER", - value=config.get("ALPACA_PAPER") or "true", - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="DATAMANAGER_BASE_URL", - value=datamanager_service.statuses[0].url, - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="MINIMUM_PORTFOLIO_TICKERS", - value=config.get("MINIMUM_PORTFOLIO_TICKERS") or "5", - ), - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="MAXIMUM_PORTFOLIO_TICKERS", - value=config.get("MAXIMUM_PORTFOLIO_TICKERS") or "20", - ), - ], - ) - ], - ) - ), -) - -predictionengine_service = cloudrun.Service( - "predictionengine", - location=project.REGION, - template=cloudrun.ServiceTemplateArgs( - spec=cloudrun.ServiceTemplateSpecArgs( - service_account_name=project.platform_service_account.email, - containers=[ - cloudrun.ServiceTemplateSpecContainerArgs( - image="pocketsizefund/predictionengine:latest", - envs=[ - cloudrun.ServiceTemplateSpecContainerEnvArgs( - name="DATAMANAGER_BASE_URL", - value=datamanager_service.statuses[0].url, - ) - ], - ) - ], - ) - ), -) diff --git a/infrastructure/environment_variables.py b/infrastructure/environment_variables.py new file mode 100644 index 000000000..22f6f4353 --- /dev/null +++ b/infrastructure/environment_variables.py @@ -0,0 +1,38 @@ +from pulumi import Output +from pulumi.config import Config +from pulumi_gcp.cloudrun import ServiceTemplateSpecContainerEnvArgs + +config = Config() + +ENVIRONMENT_VARIABLE = ServiceTemplateSpecContainerEnvArgs + + +def create_environment_variable( + name: str, value: str | Output[str] +) -> ENVIRONMENT_VARIABLE: + return ServiceTemplateSpecContainerEnvArgs(name=name, value=value) + + +GCP_PROJECT = create_environment_variable( + name="GCP_PROJECT", value=config.require_secret("GCP_PROJECT") +) + +ALPACA_API_KEY_ID = create_environment_variable( + name="ALPACA_API_KEY_ID", value=config.require_secret("ALPACA_API_KEY_ID") +) +ALPACA_API_SECRET_KEY = create_environment_variable( + name="ALPACA_API_SECRET_KEY", value=config.require_secret("ALPACA_API_SECRET_KEY") +) + +DATA_BUCKET = create_environment_variable( + name="DATA_BUCKET", value=config.require_secret("DATA_BUCKET") +) +DUCKDB_ACCESS_KEY = create_environment_variable( + name="DUCKDB_ACCESS_KEY", value=config.require_secret("DUCKDB_ACCESS_KEY") +) +DUCKDB_SECRET = create_environment_variable( + name="DUCKDB_SECRET", value=config.require_secret("DUCKDB_SECRET") +) +POLYGON_API_KEY = create_environment_variable( + name="POLYGON_API_KEY", value=config.require_secret("POLYGON_API_KEY") +) diff --git a/infrastructure/pyproject.toml b/infrastructure/pyproject.toml index df7831b2d..d26bf50ef 100644 --- a/infrastructure/pyproject.toml +++ b/infrastructure/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "infrastructure" -version = "0.1.0" +version = "20250602.4" requires-python = ">=3.13" dependencies = [ "pulumi>=3.169.0", diff --git a/infrastructure/services.py b/infrastructure/services.py new file mode 100644 index 000000000..9f4f99a07 --- /dev/null +++ b/infrastructure/services.py @@ -0,0 +1,72 @@ +from pathlib import Path +import tomllib + +import project +import pulumi_docker_build as docker_build +from environment_variables import ENVIRONMENT_VARIABLE +from pulumi.config import Config +from pulumi_gcp.cloudrun import ( + Service, + ServiceTemplateArgs, + ServiceTemplateSpecArgs, + ServiceTemplateSpecContainerArgs, + ServiceTemplateSpecContainerStartupProbeArgs, + ServiceTemplateSpecContainerStartupProbeHttpGetArgs, +) + +config = Config() + + +def create_service( + name: str, envs: list[ENVIRONMENT_VARIABLE] | None = None +) -> Service: + if envs is None: + envs = [] + + with Path("pyproject.toml").open("rb") as f: + version = tomllib.load(f).get("project", {}).get("version") + + service_dir = Path("../application") / name + + image = docker_build.Image( + f"{name}-image", + tags=[f"pocketsizefund/{name}:{version}"], + context=docker_build.BuildContextArgs(location=str(service_dir)), + platforms=[ + docker_build.Platform.LINUX_AMD64, + docker_build.Platform.LINUX_ARM64, + ], + push=True, + registries=[ + docker_build.RegistryArgs( + address="docker.io", + username=config.require_secret("dockerhub_username"), + password=config.require_secret("dockerhub_password"), + ) + ], + ) + + return Service( + name, + location=project.REGION, + template=ServiceTemplateArgs( + spec=ServiceTemplateSpecArgs( + service_account_name=project.platform_service_account.email, + containers=[ + ServiceTemplateSpecContainerArgs( + image=f"pocketsizefund/{name}:{version}", + envs=envs, + startup_probe=ServiceTemplateSpecContainerStartupProbeArgs( + initial_delay_seconds=60, + period_seconds=60, + failure_threshold=50, + http_get=ServiceTemplateSpecContainerStartupProbeHttpGetArgs( + path="/health", + port=8080, + ), + ), + ) + ], + ), + ), + ) diff --git a/pyproject.toml b/pyproject.toml index b7a05e1fb..b1e6e0780 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pocketsizefund" -version = "0.1.0" +version = "20250602.4" description = "Open source quantitative hedge fund 🍊" requires-python = ">=3.12" diff --git a/uv.lock b/uv.lock index f786b652b..872c15298 100644 --- a/uv.lock +++ b/uv.lock @@ -511,6 +511,11 @@ dependencies = [ { name = "uvicorn" }, ] +[package.dev-dependencies] +dev = [ + { name = "behave" }, +] + [package.metadata] requires-dist = [ { name = "duckdb", specifier = ">=1.2.2" }, @@ -524,6 +529,9 @@ requires-dist = [ { name = "uvicorn", specifier = ">=0.34.2" }, ] +[package.metadata.requires-dev] +dev = [{ name = "behave", specifier = ">=1.2.6" }] + [[package]] name = "debugpy" version = "1.8.14" @@ -960,7 +968,7 @@ wheels = [ [[package]] name = "infrastructure" -version = "0.1.0" +version = "20250602.4" source = { virtual = "infrastructure" } dependencies = [ { name = "pulumi" }, @@ -1493,7 +1501,7 @@ wheels = [ [[package]] name = "pocketsizefund" -version = "0.1.0" +version = "20250602.4" source = { editable = "." } [package.dev-dependencies] @@ -1536,6 +1544,7 @@ dependencies = [ { name = "fastapi" }, { name = "pandas" }, { name = "polars" }, + { name = "prometheus-fastapi-instrumentator" }, { name = "pydantic" }, { name = "pyportfolioopt" }, { name = "requests" }, @@ -1549,6 +1558,7 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.115.12" }, { name = "pandas", specifier = ">=2.1.0" }, { name = "polars", specifier = ">=1.29.0" }, + { name = "prometheus-fastapi-instrumentator", specifier = ">=7.1.0" }, { name = "pydantic", specifier = ">=2.8.2" }, { name = "pyportfolioopt", specifier = ">=1.5.6" }, { name = "requests", specifier = ">=2.31.0" }, @@ -1562,7 +1572,9 @@ source = { editable = "application/predictionengine" } dependencies = [ { name = "category-encoders" }, { name = "fastapi" }, + { name = "loguru" }, { name = "polars" }, + { name = "prometheus-fastapi-instrumentator" }, { name = "requests" }, { name = "tinygrad" }, { name = "uvicorn" }, @@ -1572,7 +1584,9 @@ dependencies = [ requires-dist = [ { name = "category-encoders", specifier = ">=2.8.1" }, { name = "fastapi", specifier = ">=0.115.12" }, + { name = "loguru", specifier = ">=0.7.3" }, { name = "polars", specifier = ">=1.29.0" }, + { name = "prometheus-fastapi-instrumentator", specifier = ">=7.1.0" }, { name = "requests", specifier = ">=2.31.0" }, { name = "tinygrad", specifier = ">=0.10.3" }, { name = "uvicorn", specifier = ">=0.34.2" }, From 76aeba8f0571e0bbf09a0a238d9a8046e713f833 Mon Sep 17 00:00:00 2001 From: Chris Addy Date: Mon, 2 Jun 2025 19:45:23 -0400 Subject: [PATCH 02/41] Update infrastructure/services.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- infrastructure/services.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infrastructure/services.py b/infrastructure/services.py index 9f4f99a07..320b41375 100644 --- a/infrastructure/services.py +++ b/infrastructure/services.py @@ -27,6 +27,8 @@ def create_service( version = tomllib.load(f).get("project", {}).get("version") service_dir = Path("../application") / name + if not service_dir.exists(): + raise FileNotFoundError(f"Service directory not found: {service_dir}") image = docker_build.Image( f"{name}-image", From cb2679a6d012baf4de0490e17bc5c47d90285320 Mon Sep 17 00:00:00 2001 From: Chris Addy Date: Mon, 2 Jun 2025 19:45:38 -0400 Subject: [PATCH 03/41] Update infrastructure/services.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- infrastructure/services.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/infrastructure/services.py b/infrastructure/services.py index 320b41375..0f7ca6772 100644 --- a/infrastructure/services.py +++ b/infrastructure/services.py @@ -23,9 +23,14 @@ def create_service( if envs is None: envs = [] - with Path("pyproject.toml").open("rb") as f: - version = tomllib.load(f).get("project", {}).get("version") - + try: + with Path("pyproject.toml").open("rb") as f: + project_data = tomllib.load(f) + version = project_data.get("project", {}).get("version") + if not version: + raise ValueError("Version not found in pyproject.toml") + except (FileNotFoundError, tomllib.TOMLDecodeError, ValueError) as e: + raise RuntimeError(f"Failed to read version from pyproject.toml: {e}") from e service_dir = Path("../application") / name if not service_dir.exists(): raise FileNotFoundError(f"Service directory not found: {service_dir}") From ae44591305812fe71e58b3c4b3e92018479e0d4b Mon Sep 17 00:00:00 2001 From: Chris Addy Date: Mon, 2 Jun 2025 19:45:52 -0400 Subject: [PATCH 04/41] Update application/predictionengine/compose.yaml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- application/predictionengine/compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/predictionengine/compose.yaml b/application/predictionengine/compose.yaml index a88c6b622..b05558070 100644 --- a/application/predictionengine/compose.yaml +++ b/application/predictionengine/compose.yaml @@ -10,7 +10,7 @@ services: environment: - DATAMANAGER_BASE_URL=${DATAMANAGER_BASE_URL} volumes: - - ./:/app/datamanager + - ./:/app/predictionengine - ~/.config/gcloud/application_default_credentials.json:/root/.config/gcloud/application_default_credentials.json:ro healthcheck: test: ["CMD", "curl", "-f", "http://0.0.0.0:8080/health"] From 3ed04441d60cc9b4a1a4bc0d33d9c8db392e5140 Mon Sep 17 00:00:00 2001 From: Chris Addy Date: Mon, 2 Jun 2025 19:46:15 -0400 Subject: [PATCH 05/41] Update infrastructure/__main__.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- infrastructure/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index c93d24409..e664ae5be 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -13,7 +13,7 @@ ) from services import create_service import topics # noqa: F401 -import buckets # noqa: F401 +import buckets # noqa: F401 # registers Pulumi `production_data_bucket` resource datamanager_service = create_service( From 14c55efe54d097d32a6cde37ef0d6349a1d16ffe Mon Sep 17 00:00:00 2001 From: Chris Addy Date: Mon, 2 Jun 2025 19:46:39 -0400 Subject: [PATCH 06/41] Update infrastructure/__main__.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- infrastructure/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index e664ae5be..aea0e50c7 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -33,7 +33,9 @@ "DATAMANAGER_BASE_URL", datamanager_service.statuses[0].url ) -predicitonengine_service = create_service( +predictionengine_service = create_service( + "predictionengine", envs=[DATAMANAGER_BASE_URL] +) "predictionengine", envs=[DATAMANAGER_BASE_URL] ) From 371d94a248d88695c328d53ae9c23e9b1e02db62 Mon Sep 17 00:00:00 2001 From: Chris Addy Date: Mon, 2 Jun 2025 21:41:24 -0400 Subject: [PATCH 07/41] Grant storage access --- infrastructure/__main__.py | 2 -- infrastructure/buckets.py | 9 +++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index aea0e50c7..e55c9298c 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -34,8 +34,6 @@ ) predictionengine_service = create_service( - "predictionengine", envs=[DATAMANAGER_BASE_URL] -) "predictionengine", envs=[DATAMANAGER_BASE_URL] ) diff --git a/infrastructure/buckets.py b/infrastructure/buckets.py index 1be03e0dd..e48be9cb0 100644 --- a/infrastructure/buckets.py +++ b/infrastructure/buckets.py @@ -11,3 +11,12 @@ location=project.REGION, uniform_bucket_level_access=True, ) + +storage.BucketIAMMember( + "platform-write-access", + bucket=production_data_bucket.name, + role="roles/storage.objectCreator", + member=project.platform_service_account.email.apply( + lambda e: f"serviceAccount:{e}" + ), +) From c8991d48a752cb39273f5d4914c7c1907f11e33e Mon Sep 17 00:00:00 2001 From: Chris Addy Date: Mon, 2 Jun 2025 19:46:39 -0400 Subject: [PATCH 08/41] Update infrastructure/__main__.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Grant storage access PR commit fixes --- infrastructure/__main__.py | 4 +--- infrastructure/buckets.py | 9 +++++++++ infrastructure/ping.nu | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 infrastructure/ping.nu diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index e664ae5be..ddfd0ddc8 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -33,7 +33,7 @@ "DATAMANAGER_BASE_URL", datamanager_service.statuses[0].url ) -predicitonengine_service = create_service( +predictionengine_service = create_service( "predictionengine", envs=[DATAMANAGER_BASE_URL] ) @@ -44,8 +44,6 @@ ALPACA_API_KEY_ID, ALPACA_API_SECRET_KEY, DATAMANAGER_BASE_URL, - # MINIMUM_PORTFOLIO_TICKERS, # 20 - # MAXIMUM_PORTFOLIO_TICKERS, # 20 ], ) diff --git a/infrastructure/buckets.py b/infrastructure/buckets.py index 1be03e0dd..e48be9cb0 100644 --- a/infrastructure/buckets.py +++ b/infrastructure/buckets.py @@ -11,3 +11,12 @@ location=project.REGION, uniform_bucket_level_access=True, ) + +storage.BucketIAMMember( + "platform-write-access", + bucket=production_data_bucket.name, + role="roles/storage.objectCreator", + member=project.platform_service_account.email.apply( + lambda e: f"serviceAccount:{e}" + ), +) diff --git a/infrastructure/ping.nu b/infrastructure/ping.nu new file mode 100644 index 000000000..0f03ebe88 --- /dev/null +++ b/infrastructure/ping.nu @@ -0,0 +1,14 @@ +let headers = [Authorization $"Bearer (gcloud auth print-identity-token)"] +let services = gcloud run services list --format=json +| from json +| get status.address.url +| each {|url| + { + service: ($url | split row "https://" | get 1 | split row "-" | get 0) + url: $url + } +} + +let datamanager_url = ($services | where service == "datamanager" | get url.0) + +http get --full --headers $headers $"($datamanager_url)/health" From 793af487ea8ac01a11a711000592067c8d099b97 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 15:29:47 -0400 Subject: [PATCH 09/41] add type annotations where missing --- application/datamanager/features/environment.py | 1 - pyproject.toml | 15 +++++++++++++++ uv.lock | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/application/datamanager/features/environment.py b/application/datamanager/features/environment.py index 5ea6dc85b..8c84c4cef 100644 --- a/application/datamanager/features/environment.py +++ b/application/datamanager/features/environment.py @@ -3,5 +3,4 @@ def before_all(context: Context) -> None: - """Set up test environment.""" context.base_url = os.environ.get("BASE_URL", "http://datamanager:8080") diff --git a/pyproject.toml b/pyproject.toml index b1e6e0780..9455f7362 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,13 @@ name = "pocketsizefund" version = "20250602.4" description = "Open source quantitative hedge fund 🍊" requires-python = ">=3.12" +dependencies = [ + "flytekit>=1.15.4", + "httpx>=0.28.1", + "pulumi-docker-build>=0.0.12", + "pulumi-gcp>=8.32.0", + "requests>=2.32.3", +] [tool.uv.workspace] members = [ @@ -71,8 +78,16 @@ select = [ "ERA" ] +[tool.ruff] +select = [ + "ANN" +] + [tool.ty.rules] unresolved-import = "ignore" invalid-return-type = "error" invalid-argument-type = "error" unresolved-reference = "error" + +[tool.pyright] +reportMissingImports = "none" diff --git a/uv.lock b/uv.lock index 872c15298..3a37ffde2 100644 --- a/uv.lock +++ b/uv.lock @@ -1503,6 +1503,13 @@ wheels = [ name = "pocketsizefund" version = "20250602.4" source = { editable = "." } +dependencies = [ + { name = "flytekit" }, + { name = "httpx" }, + { name = "pulumi-docker-build" }, + { name = "pulumi-gcp" }, + { name = "requests" }, +] [package.dev-dependencies] dev = [ @@ -1512,6 +1519,13 @@ dev = [ ] [package.metadata] +requires-dist = [ + { name = "flytekit", specifier = ">=1.15.4" }, + { name = "httpx", specifier = ">=0.28.1" }, + { name = "pulumi-docker-build", specifier = ">=0.0.12" }, + { name = "pulumi-gcp", specifier = ">=8.32.0" }, + { name = "requests", specifier = ">=2.32.3" }, +] [package.metadata.requires-dev] dev = [ From 7119965abd110e05a2a33c48c0944f4a9658f2cd Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 15:30:33 -0400 Subject: [PATCH 10/41] add ANN type annotations and fix for ruff --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9455f7362..90b5a4de6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,7 @@ select = [ [tool.ruff] select = [ "ANN" + "ERA" ] [tool.ty.rules] @@ -88,6 +89,3 @@ unresolved-import = "ignore" invalid-return-type = "error" invalid-argument-type = "error" unresolved-reference = "error" - -[tool.pyright] -reportMissingImports = "none" From 0299c66b51ed42094ac5086e2b43a4a6d07c7a02 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 15:45:57 -0400 Subject: [PATCH 11/41] ruff fixes From 5e446fcd8bcdc6fd657e0459dc0dce195c8ebdb3 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 15:33:27 -0400 Subject: [PATCH 12/41] add fastapi linting fix bandit issues fixing ruff issues --- .../datamanager/features/steps/equity_bars_steps.py | 5 +++-- .../datamanager/features/steps/health_steps.py | 2 +- application/datamanager/src/datamanager/main.py | 6 +++--- pyproject.toml | 13 ++++++++++--- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/application/datamanager/features/steps/equity_bars_steps.py b/application/datamanager/features/steps/equity_bars_steps.py index 2fff7bc33..55b4f8acb 100644 --- a/application/datamanager/features/steps/equity_bars_steps.py +++ b/application/datamanager/features/steps/equity_bars_steps.py @@ -24,7 +24,7 @@ def step_impl_api_url(context: Context) -> None: @when('I send a POST request to "{endpoint}" for date range') def step_impl_post_request(context: Context, endpoint: str) -> None: url = f"{context.api_url}{endpoint}" - response = requests.post(url, json={"date": context.start_date}) + response = requests.post(url, json={"date": context.start_date}, timeout=30) context.response = response @@ -34,6 +34,7 @@ def step_imp_get_request(context: Context, endpoint: str) -> None: response = requests.get( url, params={"start_date": context.start_date, "end_date": context.end_date}, + timeout=30, ) context.response = response @@ -48,7 +49,7 @@ def step_impl_response_status_code(context: Context, status_code: str) -> None: @when('I send a DELETE request to "{endpoint}" for date "{date_str}"') def step_impl(context: Context, endpoint: str, date_str: str) -> None: url = f"{context.api_url}{endpoint}" - response = requests.delete(url, json={"date": date_str}) + response = requests.delete(url, json={"date": date_str}, timeout=30) context.response = response context.test_date = date_str diff --git a/application/datamanager/features/steps/health_steps.py b/application/datamanager/features/steps/health_steps.py index 1f1660343..446b07522 100644 --- a/application/datamanager/features/steps/health_steps.py +++ b/application/datamanager/features/steps/health_steps.py @@ -7,4 +7,4 @@ @when('I send a GET request to "{endpoint}"') def step_impl(context: Context, endpoint: str) -> None: url = f"{context.api_url}{endpoint}" - context.response = requests.get(url) + context.response = requests.get(url, timeout=30) diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index dceaa9c88..3127086f8 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -21,7 +21,7 @@ def bars_query(*, bucket: str, start_date: date, end_date: date) -> str: path_pattern = f"gs://{bucket}/equity/bars/*/*/*/*" - return f""" + return f""" SELECT * FROM read_parquet( '{path_pattern}', @@ -35,7 +35,7 @@ def bars_query(*, bucket: str, start_date: date, end_date: date) -> str: (year < {end_date.year} OR (year = {end_date.year} AND month < {end_date.month}) OR (year = {end_date.year} AND month = {end_date.month} AND day <= {end_date.day})) - """ + """ # noqa: S608 @asynccontextmanager @@ -111,7 +111,7 @@ async def get_equity_bars( return Response(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) -@application.post("/equity-bars", response_model=BarsSummary) +@application.post("/equity-bars") async def fetch_equity_bars(request: Request, summary_date: SummaryDate) -> BarsSummary: polygon = request.app.state.settings.polygon bucket = request.app.state.settings.gcp.bucket diff --git a/pyproject.toml b/pyproject.toml index 90b5a4de6..540e726da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,11 +78,18 @@ select = [ "ERA" ] -[tool.ruff] +[tool.ruff.lint] select = [ - "ANN" - "ERA" + "ANN", # type annotations + "ASYNC", + "ERA", # dead code + "FAST", # fastapi + "S", # bandit (security) + "YTT" # flake8 ] +[tool.ruff.lint.per-file-ignores] +"**/tests/**/*.py" = ["S101"] +"**/features/steps/**/*.py" = ["S101"] [tool.ty.rules] unresolved-import = "ignore" From 72d0ed9ce72189db7ee6f5c3a75ba422ef451c68 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 16:41:39 -0400 Subject: [PATCH 13/41] no blind exceptions update to fix blind exceptions --- .../datamanager/src/datamanager/main.py | 22 +++++++++++++++++-- .../src/positionmanager/main.py | 14 +++++++----- .../tests/test_positionmanager_main.py | 12 +++++++--- pyproject.toml | 1 + 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index 3127086f8..a78655dbd 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -8,10 +8,16 @@ import httpx import polars as pl import pyarrow +import pyarrow.lib # for ArrowIOError if using Arrow internally +from duckdb import IOException +import requests + from fastapi import FastAPI, HTTPException, Request, Response, status from google.api_core import exceptions +from google.api_core.exceptions import GoogleAPIError from google.cloud import storage # type: ignore from loguru import logger +from polars.exceptions import ComputeError from prometheus_fastapi_instrumentator import Instrumentator from .config import Settings @@ -105,7 +111,13 @@ async def get_equity_bars( }, ) - except Exception as e: + except ( + requests.RequestsException, + ComputeError, + IOException, + GoogleAPIError, + pyarrow.lib.ArrowIOError, + ) as e: logger.error(f"Error querying data: {e}") logger.error(traceback.format_exc()) return Response(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) @@ -142,7 +154,13 @@ async def fetch_equity_bars(request: Request, summary_date: SummaryDate) -> Bars ).write_parquet( bucket.daily_bars_path, partition_by=["year", "month", "day"] ) - except Exception as e: + except ( + requests.RequestsException, + ComputeError, + IOException, + GoogleAPIError, + pyarrow.lib.ArrowIOError, + ) as e: logger.error(f"Error writing parquet file: {e}") logger.error(traceback.format_exc()) raise HTTPException( diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index 263f60a28..a3f0ea523 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -1,4 +1,5 @@ from fastapi import FastAPI, HTTPException +import requests import os from datetime import datetime, timedelta import polars as pl @@ -8,6 +9,9 @@ from .portfolio import PortfolioOptimizer from prometheus_fastapi_instrumentator import Instrumentator +from alpaca.common.rest import APIError +from pydantic import ValidationError + trading_days_per_year = 252 @@ -38,7 +42,7 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: try: cash_balance = alpaca_client.get_cash_balance() - except Exception as e: + except (requests.RequestException, APIError, ValidationError) as e: raise HTTPException( status_code=500, detail=f"Error getting cash balance: {str(e)}", @@ -52,7 +56,7 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: try: historical_data = data_client.get_data(date_range=date_range) - except Exception as e: + except (requests.RequestException, APIError, ValidationError) as e: raise HTTPException( status_code=500, detail=f"Error getting historical data: {str(e)}", @@ -65,7 +69,7 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: predictions=payload.predictions, ) - except Exception as e: + except (requests.RequestException, APIError, ValidationError) as e: raise HTTPException( status_code=500, detail=f"Error optimizing portfolio: {str(e)}", @@ -106,7 +110,7 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: } ) - except Exception as e: + except (requests.RequestException, APIError, ValidationError) as e: executed_trades.append( { "ticker": ticker, @@ -140,7 +144,7 @@ def delete_positions() -> Dict[str, Any]: try: result = alpaca_client.clear_positions() - except Exception as e: + except (requests.RequestException, APIError, ValidationError) as e: raise HTTPException(status_code=500, detail=str(e)) from e cash_balance = alpaca_client.get_cash_balance() diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index 1db0b3715..9de8f0991 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -1,4 +1,6 @@ from fastapi.testclient import TestClient +from fastapi import HTTPException + import unittest from unittest.mock import patch, MagicMock import polars as pl @@ -81,7 +83,9 @@ def test_create_position_success( @patch("application.positionmanager.src.positionmanager.main.AlpacaClient") def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) - mock_alpaca_instance.get_cash_balance.side_effect = Exception("API error") + mock_alpaca_instance.get_cash_balance.side_effect = HTTPException( + status_code=500, detail="Error getting cash balance" + ) MockAlpacaClient.return_value = mock_alpaca_instance payload = {"predictions": {"AAPL": 0.8}} @@ -121,13 +125,15 @@ def test_delete_positions_error( MockAlpacaClient: MagicMock, ) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) - mock_alpaca_instance.clear_positions.side_effect = Exception("API error") + mock_alpaca_instance.clear_positions.side_effect = HTTPException( + status_code=500, detail="Error getting cash balance" + ) MockAlpacaClient.return_value = mock_alpaca_instance response = client.delete("/positions") assert response.status_code == 500 - assert "API error" in response.json()["detail"] + assert "Error" in response.json()["detail"] MockAlpacaClient.assert_called_once() mock_alpaca_instance.clear_positions.assert_called_once() diff --git a/pyproject.toml b/pyproject.toml index 540e726da..7d1154d35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ select = [ select = [ "ANN", # type annotations "ASYNC", + "BLE", # no blind exceptions "ERA", # dead code "FAST", # fastapi "S", # bandit (security) From 40d4c92b5bb1618529be0288b8730d0183c2315f Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 17:11:44 -0400 Subject: [PATCH 14/41] fix boolean traps --- application/datamanager/src/datamanager/main.py | 4 ++-- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index a78655dbd..e237444bd 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -112,7 +112,7 @@ async def get_equity_bars( ) except ( - requests.RequestsException, + requests.RequestException, ComputeError, IOException, GoogleAPIError, @@ -155,7 +155,7 @@ async def fetch_equity_bars(request: Request, summary_date: SummaryDate) -> Bars bucket.daily_bars_path, partition_by=["year", "month", "day"] ) except ( - requests.RequestsException, + requests.RequestException, ComputeError, IOException, GoogleAPIError, diff --git a/pyproject.toml b/pyproject.toml index 7d1154d35..0c5bd57ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ select = [ "BLE", # no blind exceptions "ERA", # dead code "FAST", # fastapi + "FBT", # boolean traps "S", # bandit (security) "YTT" # flake8 ] From 2c5ba7e2a430ade29ea5579ab473bc306e0c674d Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 17:12:24 -0400 Subject: [PATCH 15/41] fix boolean traps add bugbear add comma linting fix bugbear and commas add timezones --- .../datamanager/src/datamanager/main.py | 17 +++++++++----- .../datamanager/src/datamanager/models.py | 12 +++++++--- .../src/positionmanager/clients.py | 2 +- .../src/positionmanager/main.py | 16 +++++++------- .../src/positionmanager/models.py | 7 ++++-- .../src/positionmanager/portfolio.py | 4 +++- .../tests/test_positionmanager_main.py | 8 ++++--- infrastructure/images.py | 9 +++++--- infrastructure/monitoring.py | 22 +++++++++---------- infrastructure/project.py | 2 +- pyproject.toml | 9 ++++++++ 11 files changed, 69 insertions(+), 39 deletions(-) diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index e237444bd..ee35add1d 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -48,7 +48,7 @@ def bars_query(*, bucket: str, start_date: date, end_date: date) -> str: async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: app.state.settings = Settings() app.state.bucket = storage.Client(os.getenv("GCP_PROJECT")).bucket( - app.state.settings.gcp.bucket.name + app.state.settings.gcp.bucket.name, ) DUCKDB_ACCESS_KEY = os.getenv("DUCKDB_ACCESS_KEY") @@ -81,12 +81,16 @@ async def health_check() -> Response: @application.get("/equity-bars") async def get_equity_bars( - request: Request, start_date: date, end_date: date + request: Request, + start_date: date, + end_date: date, ) -> Response: settings: Settings = request.app.state.settings query = bars_query( - bucket=settings.gcp.bucket.name, start_date=start_date, end_date=end_date + bucket=settings.gcp.bucket.name, + start_date=start_date, + end_date=end_date, ) try: @@ -150,9 +154,10 @@ async def fetch_equity_bars(request: Request, summary_date: SummaryDate) -> Bars pl.from_epoch("t", time_unit="ms").dt.year().alias("year"), pl.from_epoch("t", time_unit="ms").dt.month().alias("month"), pl.from_epoch("t", time_unit="ms").dt.day().alias("day"), - ] + ], ).write_parquet( - bucket.daily_bars_path, partition_by=["year", "month", "day"] + bucket.daily_bars_path, + partition_by=["year", "month", "day"], ) except ( requests.RequestException, @@ -166,7 +171,7 @@ async def fetch_equity_bars(request: Request, summary_date: SummaryDate) -> Bars raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to write data", - ) + ) from e return BarsSummary(date=summary_date.date.strftime("%Y-%m-%d"), count=count) diff --git a/application/datamanager/src/datamanager/models.py b/application/datamanager/src/datamanager/models.py index efd255525..36f1261ed 100644 --- a/application/datamanager/src/datamanager/models.py +++ b/application/datamanager/src/datamanager/models.py @@ -5,7 +5,7 @@ class SummaryDate(BaseModel): date: datetime.date = Field( - default_factory=lambda: datetime.datetime.utcnow().date() + default_factory=lambda: datetime.datetime.now(tz=datetime.timezone.utc).date(), ) @field_validator("date", mode="before") @@ -14,7 +14,11 @@ def parse_date(cls, value: datetime.date | str) -> datetime.date: return value for fmt in ("%Y-%m-%d", "%Y/%m/%d"): try: - return datetime.datetime.strptime(value, fmt).date() + return ( + datetime.datetime.strptime(value, fmt) + .replace(tzinfo=datetime.timezone.utc) + .date() + ) except ValueError: continue raise ValueError("Invalid date format: expected YYYY-MM-DD or YYYY/MM/DD") @@ -29,7 +33,9 @@ class DateRange(BaseModel): @field_validator("end") @classmethod def check_end_after_start( - cls, end_value: datetime.datetime, info: core_schema.ValidationInfo + cls, + end_value: datetime.datetime, + info: core_schema.ValidationInfo, ) -> datetime.datetime: start_value = info.data.get("start") if start_value and end_value <= start_value: diff --git a/application/positionmanager/src/positionmanager/clients.py b/application/positionmanager/src/positionmanager/clients.py index cf4ff79f8..da3919949 100644 --- a/application/positionmanager/src/positionmanager/clients.py +++ b/application/positionmanager/src/positionmanager/clients.py @@ -89,7 +89,7 @@ def get_data( pl.col("timestamp") .str.slice(0, 10) .str.strptime(pl.Date, "%Y-%m-%d") - .alias("date") + .alias("date"), ) data = ( diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index a3f0ea523..22f6bd1c6 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -1,7 +1,7 @@ from fastapi import FastAPI, HTTPException import requests import os -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import polars as pl from typing import Dict, Any from .models import Money, DateRange, PredictionPayload @@ -49,8 +49,8 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: ) from e date_range = DateRange( - start=datetime.now() - timedelta(days=trading_days_per_year), - end=datetime.now(), + start=datetime.now(tz=timezone.utc) - timedelta(days=trading_days_per_year), + end=datetime.now(tz=timezone.utc), ) try: @@ -81,7 +81,7 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: continue latest_prices = historical_data.filter(pl.col(ticker).is_not_null()).select( - ticker + ticker, ) if latest_prices.is_empty(): executed_trades.append( @@ -89,13 +89,13 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: "ticker": ticker, "status": "error", "error": "No recent price available", - } + }, ) continue latest_price = latest_prices.tail(1)[0, 0] notional_amount = Money.from_float( - latest_price * share_count * 0.95 + latest_price * share_count * 0.95, ) # 5% buffer try: @@ -107,7 +107,7 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: "share_count": share_count, "notional_amount": float(notional_amount), "status": "success", - } + }, ) except (requests.RequestException, APIError, ValidationError) as e: @@ -118,7 +118,7 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: "notional_amount": float(notional_amount), "status": "error", "error": str(e), - } + }, ) final_cash_balance = alpaca_client.get_cash_balance() diff --git a/application/positionmanager/src/positionmanager/models.py b/application/positionmanager/src/positionmanager/models.py index 5bfe85584..cbfbc8cb9 100644 --- a/application/positionmanager/src/positionmanager/models.py +++ b/application/positionmanager/src/positionmanager/models.py @@ -8,7 +8,8 @@ class Money(BaseModel): amount: Decimal = Field( - ..., description="Monetary value in USD with 2 decimal places" + ..., + description="Monetary value in USD with 2 decimal places", ) @field_validator("amount", check_fields=True) @@ -42,7 +43,9 @@ class DateRange(BaseModel): @field_validator("end") @classmethod def check_end_after_start( - cls, end_value: datetime, info: core_schema.ValidationInfo + cls, + end_value: datetime, + info: core_schema.ValidationInfo, ) -> datetime: start_value = info.data.get("start") if start_value and end_value <= start_value: diff --git a/application/positionmanager/src/positionmanager/portfolio.py b/application/positionmanager/src/positionmanager/portfolio.py index 6903f2e87..b0f6eadc7 100644 --- a/application/positionmanager/src/positionmanager/portfolio.py +++ b/application/positionmanager/src/positionmanager/portfolio.py @@ -40,7 +40,9 @@ def get_optimized_portfolio( long_only_weight_bounds = (0, 0.2) # 20% max weight per asset efficient_frontier = EfficientFrontier( - mu, S, weight_bounds=long_only_weight_bounds + mu, + S, + weight_bounds=long_only_weight_bounds, ) efficient_frontier.max_sharpe(risk_free_rate=0.02) # 2% risk-free rate diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index 9de8f0991..357bf4cf0 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -43,7 +43,7 @@ def test_create_position_success( mock_data_instance = MagicMock(spec=DataClient) mock_historical_data = pl.DataFrame( - {"date": ["2025-05-01"], "AAPL": [150.00], "MSFT": [250.00]} + {"date": ["2025-05-01"], "AAPL": [150.00], "MSFT": [250.00]}, ) mock_data_instance.get_data.return_value = mock_historical_data MockDataClient.return_value = mock_data_instance @@ -84,7 +84,8 @@ def test_create_position_success( def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.get_cash_balance.side_effect = HTTPException( - status_code=500, detail="Error getting cash balance" + status_code=500, + detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance @@ -126,7 +127,8 @@ def test_delete_positions_error( ) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.clear_positions.side_effect = HTTPException( - status_code=500, detail="Error getting cash balance" + status_code=500, + detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance diff --git a/infrastructure/images.py b/infrastructure/images.py index dfd2e1e2d..4c2190810 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -1,5 +1,5 @@ import os -from datetime import datetime +from datetime import datetime, timezone from glob import glob import pulumi import pulumi_docker_build as docker_build @@ -12,7 +12,10 @@ dockerfile_paths = glob(os.path.join("..", "application", "*", "Dockerfile")) dockerfile_paths = [os.path.relpath(dockerfile) for dockerfile in dockerfile_paths] -tags = ["latest", datetime.utcnow().strftime("%Y%m%d")] +tags = [ + "latest", + datetime.now(tz=timezone.utc).strftime("%Y%m%d"), +] images = {} for dockerfile in dockerfile_paths: @@ -36,7 +39,7 @@ address="docker.io", username=dockerhub_username, password=dockerhub_password, - ) + ), ], ) diff --git a/infrastructure/monitoring.py b/infrastructure/monitoring.py index bf5cbdd81..5ff0cc8b8 100644 --- a/infrastructure/monitoring.py +++ b/infrastructure/monitoring.py @@ -23,14 +23,14 @@ cloudrun.ServiceTemplateSpecContainerVolumeMountArgs( name="prometheus-config", mount_path="/etc/prometheus", - ) + ), ], ports=[ cloudrun.ServiceTemplateSpecContainerPortArgs( - container_port=9090 - ) + container_port=9090, + ), ], - ) + ), ], volumes=[ cloudrun.ServiceTemplateSpecVolumeArgs( @@ -41,12 +41,12 @@ cloudrun.ServiceTemplateSpecVolumeSecretItemArgs( path="prometheus.yaml", version=prometheus_config_version.version, - ) + ), ], ), - ) + ), ], - ) + ), ), ) @@ -61,11 +61,11 @@ image="grafana/grafana:latest", ports=[ cloudrun.ServiceTemplateSpecContainerPortArgs( - container_port=3000 - ) + container_port=3000, + ), ], - ) + ), ], - ) + ), ), ) diff --git a/infrastructure/project.py b/infrastructure/project.py index 39b9c8bf6..04ad148c5 100644 --- a/infrastructure/project.py +++ b/infrastructure/project.py @@ -55,6 +55,6 @@ project=PROJECT, role="roles/pubsub.subscriber", member=platform_service_account.email.apply( - lambda e: f"serviceAccount:{e}" + lambda e: f"serviceAccount:{e}", ), # ty: ignore[missing-argument] ) diff --git a/pyproject.toml b/pyproject.toml index 0c5bd57ea..8487fbb2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,15 +80,24 @@ select = [ [tool.ruff.lint] select = [ + "A", # flake8 builtins "ANN", # type annotations "ASYNC", + "B", # bugbear + "COM", # commas + "C4", # comprehensions "BLE", # no blind exceptions + "DTZ", # datetimes "ERA", # dead code "FAST", # fastapi "FBT", # boolean traps "S", # bandit (security) "YTT" # flake8 ] +ignore = [ + "COM812", +] + [tool.ruff.lint.per-file-ignores] "**/tests/**/*.py" = ["S101"] "**/features/steps/**/*.py" = ["S101"] From 3a32137cf9868f774f7b21c82090997f2d997b17 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 23:19:57 -0400 Subject: [PATCH 16/41] lint error messages no fixme/todos implicit string concatenation import conventions logging pathlib --- .../features/steps/equity_bars_steps.py | 2 +- .../datamanager/src/datamanager/config.py | 6 +-- .../datamanager/src/datamanager/main.py | 8 ++-- .../datamanager/src/datamanager/models.py | 6 ++- .../src/positionmanager/clients.py | 16 ++++--- .../src/positionmanager/models.py | 3 +- infrastructure/images.py | 20 ++++---- infrastructure/pyproject.toml | 1 + pyproject.toml | 47 ++++++++++++++----- uv.lock | 2 + 10 files changed, 71 insertions(+), 40 deletions(-) diff --git a/application/datamanager/features/steps/equity_bars_steps.py b/application/datamanager/features/steps/equity_bars_steps.py index 55b4f8acb..5700e7433 100644 --- a/application/datamanager/features/steps/equity_bars_steps.py +++ b/application/datamanager/features/steps/equity_bars_steps.py @@ -55,7 +55,7 @@ def step_impl(context: Context, endpoint: str, date_str: str) -> None: @then('the equity bars data for "{date_str}" should be deleted') -def step_impl_equity_bars(context: Context, date_str: str) -> None: +def step_impl_equity_bars(context: Context, date_str: str) -> None: # noqa: ARG001 if os.environ.get("GCP_GCS_BUCKET"): assert True, "GCS bucket deletion check would go here" else: diff --git a/application/datamanager/src/datamanager/config.py b/application/datamanager/src/datamanager/config.py index c3110c462..591a23507 100644 --- a/application/datamanager/src/datamanager/config.py +++ b/application/datamanager/src/datamanager/config.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import json from functools import cached_property from pydantic import BaseModel, Field, computed_field @@ -27,9 +28,8 @@ class GCP(BaseModel): @cached_property def _creds(self) -> dict: - with open(self.credentials_path) as f: - creds = json.load(f) - return creds + with Path(self.credentials_path).open("r") as f: + return json.load(f) @computed_field def key_id(self) -> str | None: diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index ee35add1d..2f3e80d62 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -7,8 +7,8 @@ import duckdb import httpx import polars as pl -import pyarrow -import pyarrow.lib # for ArrowIOError if using Arrow internally +import pyarrow as pa +import pyarrow.lib from duckdb import IOException import requests @@ -100,8 +100,8 @@ async def get_equity_bars( return Response(status_code=status.HTTP_404_NOT_FOUND) logger.info(f"Query returned {data.num_rows} rows") - sink = pyarrow.BufferOutputStream() - with pyarrow.ipc.RecordBatchStreamWriter(sink, data.schema) as writer: + sink = pa.BufferOutputStream() + with pa.ipc.RecordBatchStreamWriter(sink, data.schema) as writer: writer.write_table(data) return Response( diff --git a/application/datamanager/src/datamanager/models.py b/application/datamanager/src/datamanager/models.py index 36f1261ed..663ed0950 100644 --- a/application/datamanager/src/datamanager/models.py +++ b/application/datamanager/src/datamanager/models.py @@ -21,7 +21,8 @@ def parse_date(cls, value: datetime.date | str) -> datetime.date: ) except ValueError: continue - raise ValueError("Invalid date format: expected YYYY-MM-DD or YYYY/MM/DD") + msg = "Invalid date format: expected YYYY-MM-DD or YYYY/MM/DD" + raise ValueError(msg) model_config = {"json_encoders": {datetime.date: lambda d: d.strftime("%Y/%m/%d")}} @@ -39,7 +40,8 @@ def check_end_after_start( ) -> datetime.datetime: start_value = info.data.get("start") if start_value and end_value <= start_value: - raise ValueError("End date must be after start date.") + msg = "End date must be after start date." + raise ValueError(msg) return end_value diff --git a/application/positionmanager/src/positionmanager/clients.py b/application/positionmanager/src/positionmanager/clients.py index da3919949..2c7a16e49 100644 --- a/application/positionmanager/src/positionmanager/clients.py +++ b/application/positionmanager/src/positionmanager/clients.py @@ -17,7 +17,8 @@ def __init__( paper: bool = True, ) -> None: if not api_key or not api_secret: - raise ValueError("Alpaca API key and secret are required") + msg = "Alpaca API key and secret are required" + raise ValueError(msg) self.trading_client = TradingClient(api_key, api_secret, paper=paper) @@ -67,19 +68,22 @@ def get_data( date_range: DateRange, ) -> pl.DataFrame: if not self.datamanager_base_url: - raise ValueError("Data manager URL is not configured") + msg = "Data manager URL is not configured" + raise ValueError(msg) endpoint = f"{self.datamanager_base_url}/equity-bars" try: response = requests.post(endpoint, json=date_range.to_payload(), timeout=10) except requests.RequestException as err: - raise RuntimeError(f"Data manager service call error: {err}") from err + msg = f"Data manager service call error: {err}" + raise RuntimeError(msg) from err if response.status_code != 200: - raise Exception( + msg = ( f"Data service error: {response.text}, status code: {response.status_code}", ) + raise Exception(msg) response_data = response.json() @@ -92,10 +96,8 @@ def get_data( .alias("date"), ) - data = ( + return ( data.sort("date") .pivot(on="ticker", index="date", values="close_price") .with_columns(pl.all().exclude("date").cast(pl.Float64)) ) - - return data diff --git a/application/positionmanager/src/positionmanager/models.py b/application/positionmanager/src/positionmanager/models.py index cbfbc8cb9..c9cbacb02 100644 --- a/application/positionmanager/src/positionmanager/models.py +++ b/application/positionmanager/src/positionmanager/models.py @@ -49,7 +49,8 @@ def check_end_after_start( ) -> datetime: start_value = info.data.get("start") if start_value and end_value <= start_value: - raise ValueError("End date must be after start date.") + msg = "End date must be after start date." + raise ValueError(msg) return end_value diff --git a/infrastructure/images.py b/infrastructure/images.py index 4c2190810..0acf5cd96 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -1,16 +1,20 @@ import os +from pathlib import Path from datetime import datetime, timezone from glob import glob import pulumi import pulumi_docker_build as docker_build from pulumi import Config +from loguru import logger config = Config() dockerhub_username = config.require_secret("dockerhub_username") dockerhub_password = config.require_secret("dockerhub_password") -dockerfile_paths = glob(os.path.join("..", "application", "*", "Dockerfile")) -dockerfile_paths = [os.path.relpath(dockerfile) for dockerfile in dockerfile_paths] +application_path = Path("../application/").resolve() +dockerfile_paths = [ + app.relative_to(application_path) for app in application_path.glob("*/Dockerfile") +] tags = [ "latest", @@ -19,9 +23,9 @@ images = {} for dockerfile in dockerfile_paths: - service_dir = os.path.dirname(dockerfile) - service_name = os.path.basename(service_dir) - print(f"Creating image for service: {service_name}") + service_dir = dockerfile.parent + service_name = dockerfile.name + logger.info(f"Creating image for service: {service_name}") images[service_name] = docker_build.Image( f"{service_name}-image", @@ -45,8 +49,4 @@ pulumi.export(f"{service_name}-ref", images[service_name].ref) -datamanager_image = images.get("datamanager") -positionmanager_image = images.get("positionmanager") -predictionengine_image = images.get("predictionengine") - -print(f"Available image services: {list(images.keys())}") +logger.info(f"Available image services: {list(images.keys())}") diff --git a/infrastructure/pyproject.toml b/infrastructure/pyproject.toml index d26bf50ef..3f854f5ca 100644 --- a/infrastructure/pyproject.toml +++ b/infrastructure/pyproject.toml @@ -7,4 +7,5 @@ dependencies = [ "pulumi-gcp>=8.30.1", "pulumi-docker>=3.0.0", "pulumi-docker-build>=0.0.12", + "loguru>=0.7.3", ] diff --git a/pyproject.toml b/pyproject.toml index 8487fbb2a..87b2821f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,19 +80,42 @@ select = [ [tool.ruff.lint] select = [ - "A", # flake8 builtins - "ANN", # type annotations + "A", # flake8 builtins + "ANN", # type annotations + "ARG", # unused args "ASYNC", - "B", # bugbear - "COM", # commas - "C4", # comprehensions - "BLE", # no blind exceptions - "DTZ", # datetimes - "ERA", # dead code - "FAST", # fastapi - "FBT", # boolean traps - "S", # bandit (security) - "YTT" # flake8 + "B", # bugbear + "COM", # commas + "C4", # comprehensions + "BLE", # no blind exceptions + "DTZ", # datetimes + "EM", # error messages + "ERA", # dead code + "EXE", # executables + "FA", # future annotations + "FAST", # fastapi + "FIX", # no fixme/todo comments + "FBT", # boolean traps + "G", # logging format + "ICN", # import conventions + "ISC", # implicit string concatenation + "LOG", # logging + "Q", # quotes + "PIE", # misc lints + "PT", # pytest style + "PTH", # use pathlib + "PYI", # type hints + "RSE", # raises + "RET", # returns + "S", # bandit (security) + "SIM", # simplicity + "SLF", # self + "SLOT", # slots + "TC", # type checking + "TID", # tidy imports + "T10", # debugger + "T20", # printing + "YTT" # flake8 ] ignore = [ "COM812", diff --git a/uv.lock b/uv.lock index 3a37ffde2..12bafa67e 100644 --- a/uv.lock +++ b/uv.lock @@ -971,6 +971,7 @@ name = "infrastructure" version = "20250602.4" source = { virtual = "infrastructure" } dependencies = [ + { name = "loguru" }, { name = "pulumi" }, { name = "pulumi-docker" }, { name = "pulumi-docker-build" }, @@ -979,6 +980,7 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "loguru", specifier = ">=0.7.3" }, { name = "pulumi", specifier = ">=3.169.0" }, { name = "pulumi-docker", specifier = ">=3.0.0" }, { name = "pulumi-docker-build", specifier = ">=0.0.12" }, From e4d82f08aa962afc1c2b0127dfcab155f4497c06 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 23:20:26 -0400 Subject: [PATCH 17/41] a bunch of linting sort imports fix naming conventions --- .../datamanager/features/environment.py | 1 + .../features/steps/equity_bars_steps.py | 2 +- .../features/steps/health_steps.py | 3 +-- .../datamanager/src/datamanager/config.py | 5 ++-- .../datamanager/src/datamanager/main.py | 7 +++-- .../datamanager/src/datamanager/models.py | 3 ++- .../src/positionmanager/clients.py | 10 +++---- .../src/positionmanager/main.py | 9 +++++-- .../src/positionmanager/models.py | 2 +- .../src/positionmanager/portfolio.py | 9 ++++--- .../tests/test_positionmanager_main.py | 27 ++++++++++--------- infrastructure/buckets.py | 3 +-- infrastructure/images.py | 5 ++-- infrastructure/monitoring.py | 4 +-- infrastructure/project.py | 2 +- pyproject.toml | 8 +++++- 16 files changed, 57 insertions(+), 43 deletions(-) diff --git a/application/datamanager/features/environment.py b/application/datamanager/features/environment.py index 8c84c4cef..796fe865c 100644 --- a/application/datamanager/features/environment.py +++ b/application/datamanager/features/environment.py @@ -1,4 +1,5 @@ import os + from behave.runner import Context diff --git a/application/datamanager/features/steps/equity_bars_steps.py b/application/datamanager/features/steps/equity_bars_steps.py index 5700e7433..3e3b386f2 100644 --- a/application/datamanager/features/steps/equity_bars_steps.py +++ b/application/datamanager/features/steps/equity_bars_steps.py @@ -5,7 +5,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent)) import requests -from behave import given, when, then +from behave import given, then, when from behave.runner import Context diff --git a/application/datamanager/features/steps/health_steps.py b/application/datamanager/features/steps/health_steps.py index 446b07522..b6be9a63c 100644 --- a/application/datamanager/features/steps/health_steps.py +++ b/application/datamanager/features/steps/health_steps.py @@ -1,8 +1,7 @@ +import requests from behave import when from behave.runner import Context -import requests - @when('I send a GET request to "{endpoint}"') def step_impl(context: Context, endpoint: str) -> None: diff --git a/application/datamanager/src/datamanager/config.py b/application/datamanager/src/datamanager/config.py index 591a23507..fa234e05a 100644 --- a/application/datamanager/src/datamanager/config.py +++ b/application/datamanager/src/datamanager/config.py @@ -1,7 +1,8 @@ -import os -from pathlib import Path import json +import os from functools import cached_property +from pathlib import Path + from pydantic import BaseModel, Field, computed_field diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index 2f3e80d62..37a3924db 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -9,9 +9,8 @@ import polars as pl import pyarrow as pa import pyarrow.lib -from duckdb import IOException import requests - +from duckdb import IOException from fastapi import FastAPI, HTTPException, Request, Response, status from google.api_core import exceptions from google.api_core.exceptions import GoogleAPIError @@ -51,8 +50,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: app.state.settings.gcp.bucket.name, ) - DUCKDB_ACCESS_KEY = os.getenv("DUCKDB_ACCESS_KEY") - DUCKDB_SECRET = os.getenv("DUCKDB_SECRET") + DUCKDB_ACCESS_KEY = os.getenv("DUCKDB_ACCESS_KEY") # noqa: N806 + DUCKDB_SECRET = os.getenv("DUCKDB_SECRET") # noqa: N806 app.state.connection = duckdb.connect() app.state.connection.execute(f""" diff --git a/application/datamanager/src/datamanager/models.py b/application/datamanager/src/datamanager/models.py index 663ed0950..c2f4ed8a2 100644 --- a/application/datamanager/src/datamanager/models.py +++ b/application/datamanager/src/datamanager/models.py @@ -1,4 +1,5 @@ import datetime + from pydantic import BaseModel, Field, field_validator from pydantic_core import core_schema @@ -9,7 +10,7 @@ class SummaryDate(BaseModel): ) @field_validator("date", mode="before") - def parse_date(cls, value: datetime.date | str) -> datetime.date: + def parse_date(cls, value: datetime.date | str) -> datetime.date: # noqa: N805 if isinstance(value, datetime.date): return value for fmt in ("%Y-%m-%d", "%Y/%m/%d"): diff --git a/application/positionmanager/src/positionmanager/clients.py b/application/positionmanager/src/positionmanager/clients.py index 2c7a16e49..fe2364eb9 100644 --- a/application/positionmanager/src/positionmanager/clients.py +++ b/application/positionmanager/src/positionmanager/clients.py @@ -1,12 +1,12 @@ -import requests -import polars as pl -from typing import Dict, Any +from typing import Any, Dict +import polars as pl +import requests from alpaca.trading.client import TradingClient -from alpaca.trading.requests import MarketOrderRequest from alpaca.trading.enums import OrderSide, TimeInForce +from alpaca.trading.requests import MarketOrderRequest -from .models import Money, DateRange +from .models import DateRange, Money class AlpacaClient: diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index 22f6bd1c6..63268c7a2 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -1,7 +1,7 @@ -from fastapi import FastAPI, HTTPException -import requests import os from datetime import datetime, timedelta, timezone +from typing import Any, Dict + import polars as pl from typing import Dict, Any from .models import Money, DateRange, PredictionPayload @@ -10,8 +10,13 @@ from prometheus_fastapi_instrumentator import Instrumentator from alpaca.common.rest import APIError +from fastapi import FastAPI, HTTPException +from prometheus_fastapi_instrumentator import Instrumentator from pydantic import ValidationError +from .clients import AlpacaClient, DataClient +from .models import DateRange, Money, PredictionPayload +from .portfolio import PortfolioOptimizer trading_days_per_year = 252 diff --git a/application/positionmanager/src/positionmanager/models.py b/application/positionmanager/src/positionmanager/models.py index c9cbacb02..1797fe36e 100644 --- a/application/positionmanager/src/positionmanager/models.py +++ b/application/positionmanager/src/positionmanager/models.py @@ -13,7 +13,7 @@ class Money(BaseModel): ) @field_validator("amount", check_fields=True) - def validate_amount(cls, v: str | Decimal) -> Decimal: + def validate_amount(cls, v: str | Decimal) -> Decimal: # noqa: N805 if not isinstance(v, Decimal): v = Decimal(str(v)) diff --git a/application/positionmanager/src/positionmanager/portfolio.py b/application/positionmanager/src/positionmanager/portfolio.py index b0f6eadc7..72f7b8cca 100644 --- a/application/positionmanager/src/positionmanager/portfolio.py +++ b/application/positionmanager/src/positionmanager/portfolio.py @@ -1,7 +1,8 @@ from typing import Dict -import polars as pl + import pandas as pd -from pypfopt import EfficientFrontier, risk_models, expected_returns +import polars as pl +from pypfopt import EfficientFrontier, expected_returns, risk_models from pypfopt.discrete_allocation import DiscreteAllocation, get_latest_prices from .models import Money @@ -36,12 +37,12 @@ def get_optimized_portfolio( ticker ] + prediction_weight * prediction_series[ticker] - S = risk_models.CovarianceShrinkage(converted_data).ledoit_wolf() + covariance = risk_models.CovarianceShrinkage(converted_data).ledoit_wolf() long_only_weight_bounds = (0, 0.2) # 20% max weight per asset efficient_frontier = EfficientFrontier( mu, - S, + covariance, weight_bounds=long_only_weight_bounds, ) diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index 357bf4cf0..037623c0e 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -1,16 +1,17 @@ -from fastapi.testclient import TestClient -from fastapi import HTTPException - import unittest -from unittest.mock import patch, MagicMock -import polars as pl from decimal import Decimal -from application.positionmanager.src.positionmanager.main import application -from application.positionmanager.src.positionmanager.models import Money +from unittest.mock import MagicMock, patch + +import polars as pl +from fastapi import HTTPException +from fastapi.testclient import TestClient + from application.positionmanager.src.positionmanager.clients import ( AlpacaClient, DataClient, ) +from application.positionmanager.src.positionmanager.main import application +from application.positionmanager.src.positionmanager.models import Money from application.positionmanager.src.positionmanager.portfolio import PortfolioOptimizer client = TestClient(application) @@ -28,9 +29,9 @@ class TestPositionsEndpoint(unittest.TestCase): @patch("application.positionmanager.src.positionmanager.main.PortfolioOptimizer") def test_create_position_success( self, - MockPortfolioOptimizer: MagicMock, - MockDataClient: MagicMock, - MockAlpacaClient: MagicMock, + MockPortfolioOptimizer: MagicMock, # noqa: N803 + MockDataClient: MagicMock, # noqa: N803 + MockAlpacaClient: MagicMock, # noqa: N803 ) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_cash_balance = Money(amount=Decimal("100000.00")) @@ -81,7 +82,7 @@ def test_create_position_success( assert mock_alpaca_instance.place_notional_order.call_count == 2 @patch("application.positionmanager.src.positionmanager.main.AlpacaClient") - def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None: + def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None: # noqa: N803 mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.get_cash_balance.side_effect = HTTPException( status_code=500, @@ -99,7 +100,7 @@ def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None mock_alpaca_instance.get_cash_balance.assert_called_once() @patch("application.positionmanager.src.positionmanager.main.AlpacaClient") - def test_delete_positions_success(self, MockAlpacaClient: MagicMock) -> None: + def test_delete_positions_success(self, MockAlpacaClient: MagicMock) -> None: # noqa: N803 mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.clear_positions.return_value = { "status": "success", @@ -123,7 +124,7 @@ def test_delete_positions_success(self, MockAlpacaClient: MagicMock) -> None: @patch("application.positionmanager.src.positionmanager.main.AlpacaClient") def test_delete_positions_error( self, - MockAlpacaClient: MagicMock, + MockAlpacaClient: MagicMock, # noqa: N803 ) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.clear_positions.side_effect = HTTPException( diff --git a/infrastructure/buckets.py b/infrastructure/buckets.py index e48be9cb0..379e55991 100644 --- a/infrastructure/buckets.py +++ b/infrastructure/buckets.py @@ -1,7 +1,6 @@ +import project from pulumi import Config from pulumi_gcp import storage -import project - config = Config() diff --git a/infrastructure/images.py b/infrastructure/images.py index 0acf5cd96..32e469ea2 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -1,11 +1,12 @@ import os -from pathlib import Path from datetime import datetime, timezone from glob import glob +from pathlib import Path + import pulumi import pulumi_docker_build as docker_build -from pulumi import Config from loguru import logger +from pulumi import Config config = Config() dockerhub_username = config.require_secret("dockerhub_username") diff --git a/infrastructure/monitoring.py b/infrastructure/monitoring.py index 5ff0cc8b8..034e1aac6 100644 --- a/infrastructure/monitoring.py +++ b/infrastructure/monitoring.py @@ -1,6 +1,6 @@ -from pulumi_gcp import cloudrun, secretmanager -from pulumi import FileAsset import project +from pulumi import FileAsset +from pulumi_gcp import cloudrun, secretmanager prometheus_config_secret = secretmanager.Secret("prometheus-config") prometheus_config_version = secretmanager.SecretVersion( diff --git a/infrastructure/project.py b/infrastructure/project.py index 04ad148c5..0f733c333 100644 --- a/infrastructure/project.py +++ b/infrastructure/project.py @@ -1,5 +1,5 @@ import pulumi -from pulumi_gcp.projects import Service, IAMMember +from pulumi_gcp.projects import IAMMember, Service from pulumi_gcp.serviceaccount import Account PROJECT = pulumi.Config("gcp").require("project") diff --git a/pyproject.toml b/pyproject.toml index 87b2821f4..6f2e0a711 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,7 @@ select = [ "B", # bugbear "COM", # commas "C4", # comprehensions + "C90", # complexity "BLE", # no blind exceptions "DTZ", # datetimes "EM", # error messages @@ -95,16 +96,21 @@ select = [ "FA", # future annotations "FAST", # fastapi "FIX", # no fixme/todo comments + "FLY", # f strings "FBT", # boolean traps "G", # logging format "ICN", # import conventions "ISC", # implicit string concatenation + "I", # isort "LOG", # logging - "Q", # quotes + "N", # naming + "NPY", # numpy + "PD", # pandas "PIE", # misc lints "PT", # pytest style "PTH", # use pathlib "PYI", # type hints + "Q", # quotes "RSE", # raises "RET", # returns "S", # bandit (security) From 1a5a91e2e42e7186a2b8c3073cb272c4b737df04 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 23:44:29 -0400 Subject: [PATCH 18/41] leftover linting --- .../datamanager/src/datamanager/main.py | 19 +++++++++++++------ .../datamanager/src/datamanager/models.py | 4 ++-- .../src/positionmanager/clients.py | 14 +++++--------- .../src/positionmanager/main.py | 18 +++++++++--------- .../src/positionmanager/models.py | 8 ++++---- .../src/positionmanager/portfolio.py | 6 ++---- .../tests/test_positionmanager_main.py | 18 +++++++++--------- infrastructure/images.py | 6 ++---- pyproject.toml | 10 +++++++++- workflows/backfill_datamanager.py | 5 ++--- 10 files changed, 57 insertions(+), 51 deletions(-) diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index 37a3924db..9de041380 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -1,8 +1,8 @@ import os import traceback +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from datetime import date -from typing import AsyncGenerator import duckdb import httpx @@ -35,11 +35,14 @@ def bars_query(*, bucket: str, start_date: date, end_date: date) -> str: WHERE (year > {start_date.year} OR (year = {start_date.year} AND month > {start_date.month}) OR - (year = {start_date.year} AND month = {start_date.month} AND day >= {start_date.day})) + (year = {start_date.year} AND month = {start_date.month} + AND day >= {start_date.day})) AND (year < {end_date.year} OR (year = {end_date.year} AND month < {end_date.month}) OR - (year = {end_date.year} AND month = {end_date.month} AND day <= {end_date.day})) + (year = {end_date.year} + AND month = {end_date.month} + AND day <= {end_date.day})) """ # noqa: S608 @@ -103,11 +106,14 @@ async def get_equity_bars( with pa.ipc.RecordBatchStreamWriter(sink, data.schema) as writer: writer.write_table(data) + filename = f"equity_bars_{start_date}_{end_date}.arrow" + content_disposition = f"attachment; {filename=}" + return Response( content=sink.getvalue().to_pybytes(), media_type="application/vnd.apache.arrow.file", headers={ - "Content-Disposition": f"attachment; filename=equity_bars_{start_date}_{end_date}.arrow", + "Content-Disposition": content_disposition, "X-Row-Count": str(data.num_rows), "X-Start-Date": str(start_date), "X-End-Date": str(end_date), @@ -131,7 +137,8 @@ async def fetch_equity_bars(request: Request, summary_date: SummaryDate) -> Bars polygon = request.app.state.settings.polygon bucket = request.app.state.settings.gcp.bucket - url = f"{polygon.base_url}{polygon.daily_bars}{summary_date.date.strftime('%Y-%m-%d')}" + summary_date: str = summary_date.date.strftime("%Y-%m-%d") + url = f"{polygon.base_url}{polygon.daily_bars}{summary_date}" logger.info(f"polygon_api_endpoint={url}") params = {"adjusted": "true", "apiKey": polygon.api_key} @@ -171,7 +178,7 @@ async def fetch_equity_bars(request: Request, summary_date: SummaryDate) -> Bars status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to write data", ) from e - return BarsSummary(date=summary_date.date.strftime("%Y-%m-%d"), count=count) + return BarsSummary(date=summary_date, count=count) @application.delete("/equity-bars") diff --git a/application/datamanager/src/datamanager/models.py b/application/datamanager/src/datamanager/models.py index c2f4ed8a2..232eb936a 100644 --- a/application/datamanager/src/datamanager/models.py +++ b/application/datamanager/src/datamanager/models.py @@ -6,7 +6,7 @@ class SummaryDate(BaseModel): date: datetime.date = Field( - default_factory=lambda: datetime.datetime.now(tz=datetime.timezone.utc).date(), + default_factory=lambda: datetime.datetime.now(tz=datetime.UTC).date(), ) @field_validator("date", mode="before") @@ -17,7 +17,7 @@ def parse_date(cls, value: datetime.date | str) -> datetime.date: # noqa: N805 try: return ( datetime.datetime.strptime(value, fmt) - .replace(tzinfo=datetime.timezone.utc) + .replace(tzinfo=datetime.UTC) .date() ) except ValueError: diff --git a/application/positionmanager/src/positionmanager/clients.py b/application/positionmanager/src/positionmanager/clients.py index fe2364eb9..700e76d96 100644 --- a/application/positionmanager/src/positionmanager/clients.py +++ b/application/positionmanager/src/positionmanager/clients.py @@ -1,4 +1,4 @@ -from typing import Any, Dict +from typing import Any import polars as pl import requests @@ -35,7 +35,7 @@ def place_notional_order( self, ticker: str, notional_amount: Money, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: market_order_request = MarketOrderRequest( symbol=ticker, notional=float(notional_amount), @@ -47,10 +47,10 @@ def place_notional_order( return { "status": "success", - "message": f"Order placed for {ticker} with notional amount {notional_amount}", + "message": f"Order placed [{ticker=}, {notional_amount}]", } - def clear_positions(self) -> Dict[str, Any]: + def clear_positions(self) -> dict[str, Any]: self.trading_client.close_all_positions(cancel_orders=True) return { @@ -79,11 +79,7 @@ def get_data( msg = f"Data manager service call error: {err}" raise RuntimeError(msg) from err - if response.status_code != 200: - msg = ( - f"Data service error: {response.text}, status code: {response.status_code}", - ) - raise Exception(msg) + response.raise_for_status() response_data = response.json() diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index 63268c7a2..c13459bdb 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -1,6 +1,6 @@ import os -from datetime import datetime, timedelta, timezone -from typing import Any, Dict +from datetime import UTC, datetime, timedelta +from typing import Any import polars as pl from typing import Dict, Any @@ -30,7 +30,7 @@ def get_health() -> dict[str, str]: @application.post("/positions") -def create_position(payload: PredictionPayload) -> Dict[str, Any]: +def create_position(payload: PredictionPayload) -> dict[str, Any]: alpaca_client = AlpacaClient( api_key=os.getenv("ALPACA_API_KEY", ""), api_secret=os.getenv("ALPACA_API_SECRET", ""), @@ -50,12 +50,12 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: except (requests.RequestException, APIError, ValidationError) as e: raise HTTPException( status_code=500, - detail=f"Error getting cash balance: {str(e)}", + detail=f"Error getting cash balance: {e!r}", ) from e date_range = DateRange( - start=datetime.now(tz=timezone.utc) - timedelta(days=trading_days_per_year), - end=datetime.now(tz=timezone.utc), + start=datetime.now(tz=UTC) - timedelta(days=trading_days_per_year), + end=datetime.now(tz=UTC), ) try: @@ -64,7 +64,7 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: except (requests.RequestException, APIError, ValidationError) as e: raise HTTPException( status_code=500, - detail=f"Error getting historical data: {str(e)}", + detail=f"Error getting historical data: {e!r}", ) from e try: @@ -77,7 +77,7 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: except (requests.RequestException, APIError, ValidationError) as e: raise HTTPException( status_code=500, - detail=f"Error optimizing portfolio: {str(e)}", + detail=f"Error optimizing portfolio: {e!r}", ) from e executed_trades = [] @@ -139,7 +139,7 @@ def create_position(payload: PredictionPayload) -> Dict[str, Any]: @application.delete("/positions") -def delete_positions() -> Dict[str, Any]: +def delete_positions() -> dict[str, Any]: alpaca_client = AlpacaClient( api_key=os.getenv("ALPACA_API_KEY", ""), api_secret=os.getenv("ALPACA_API_SECRET", ""), diff --git a/application/positionmanager/src/positionmanager/models.py b/application/positionmanager/src/positionmanager/models.py index 1797fe36e..47c85c125 100644 --- a/application/positionmanager/src/positionmanager/models.py +++ b/application/positionmanager/src/positionmanager/models.py @@ -1,6 +1,6 @@ from datetime import datetime from decimal import ROUND_HALF_UP, Decimal -from typing import Any, Dict +from typing import Any from pydantic import BaseModel, Field, field_validator from pydantic_core import core_schema @@ -32,7 +32,7 @@ def __repr__(self) -> str: def from_float(cls, value: float) -> "Money": return cls(amount=Decimal(str(value))) - def to_dict(self) -> Dict[str, float]: + def to_dict(self) -> dict[str, float]: return {"amount": float(self.amount)} @@ -54,7 +54,7 @@ def check_end_after_start( return end_value - def to_payload(self) -> Dict[str, str]: + def to_payload(self) -> dict[str, str]: return { "start_date": self.start.isoformat(), "end_date": self.end.isoformat(), @@ -62,4 +62,4 @@ def to_payload(self) -> Dict[str, str]: class PredictionPayload(BaseModel): - predictions: Dict[str, Any] + predictions: dict[str, Any] diff --git a/application/positionmanager/src/positionmanager/portfolio.py b/application/positionmanager/src/positionmanager/portfolio.py index 72f7b8cca..0e9a10a27 100644 --- a/application/positionmanager/src/positionmanager/portfolio.py +++ b/application/positionmanager/src/positionmanager/portfolio.py @@ -1,5 +1,3 @@ -from typing import Dict - import pandas as pd import polars as pl from pypfopt import EfficientFrontier, expected_returns, risk_models @@ -21,9 +19,9 @@ def get_optimized_portfolio( self, data: pl.DataFrame, portfolio_value: Money, - predictions: Dict[str, float], + predictions: dict[str, float], prediction_weight: float = 0.3, - ) -> Dict[str, int]: + ) -> dict[str, int]: converted_data = data.to_pandas() if "date" in converted_data.columns: diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index 037623c0e..a5ce64c9b 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -3,7 +3,7 @@ from unittest.mock import MagicMock, patch import polars as pl -from fastapi import HTTPException +from fastapi import HTTPException, status from fastapi.testclient import TestClient from application.positionmanager.src.positionmanager.clients import ( @@ -19,7 +19,7 @@ def test_health_check() -> None: response = client.get("/health") - assert response.status_code == 200 + assert response.status_code == status.HTTP_200_OK assert response.json() == {"status": "healthy"} @@ -59,7 +59,7 @@ def test_create_position_success( payload = {"predictions": {"AAPL": 0.8, "MSFT": 0.7}} response = client.post("/positions", json=payload) - assert response.status_code == 200 + assert response.status_code == status.HTTP_200_OK assert response.json()["status"] == "success" assert "initial_cash_balance" in response.json() assert "final_cash_balance" in response.json() @@ -79,13 +79,13 @@ def test_create_position_success( assert "portfolio_value" in optimizer_args assert optimizer_args["portfolio_value"] == mock_cash_balance - assert mock_alpaca_instance.place_notional_order.call_count == 2 + assert mock_alpaca_instance.place_notional_order.call_count == 2 # noqa: PLR2004 @patch("application.positionmanager.src.positionmanager.main.AlpacaClient") def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None: # noqa: N803 mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.get_cash_balance.side_effect = HTTPException( - status_code=500, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance @@ -93,7 +93,7 @@ def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None payload = {"predictions": {"AAPL": 0.8}} response = client.post("/positions", json=payload) - assert response.status_code == 500 + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR assert "Error getting cash balance" in response.json()["detail"] MockAlpacaClient.assert_called_once() @@ -112,7 +112,7 @@ def test_delete_positions_success(self, MockAlpacaClient: MagicMock) -> None: # response = client.delete("/positions") - assert response.status_code == 200 + assert response.status_code == status.HTTP_200_OK assert response.json()["status"] == "success" assert "cash_balance" in response.json() assert response.json()["cash_balance"] == float(mock_cash_balance) @@ -128,14 +128,14 @@ def test_delete_positions_error( ) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.clear_positions.side_effect = HTTPException( - status_code=500, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance response = client.delete("/positions") - assert response.status_code == 500 + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR assert "Error" in response.json()["detail"] MockAlpacaClient.assert_called_once() diff --git a/infrastructure/images.py b/infrastructure/images.py index 32e469ea2..bb2408e9e 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -1,6 +1,4 @@ -import os -from datetime import datetime, timezone -from glob import glob +from datetime import UTC, datetime from pathlib import Path import pulumi @@ -19,7 +17,7 @@ tags = [ "latest", - datetime.now(tz=timezone.utc).strftime("%Y%m%d"), + datetime.now(tz=UTC).strftime("%Y%m%d"), ] images = {} diff --git a/pyproject.toml b/pyproject.toml index 6f2e0a711..24461cdc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,37 +90,45 @@ select = [ "C90", # complexity "BLE", # no blind exceptions "DTZ", # datetimes + "E", # whitespace "EM", # error messages "ERA", # dead code "EXE", # executables + "F", # pyflakes "FA", # future annotations "FAST", # fastapi "FIX", # no fixme/todo comments "FLY", # f strings "FBT", # boolean traps + "FURB", # refurb "G", # logging format "ICN", # import conventions "ISC", # implicit string concatenation - "I", # isort + "I", # isort "LOG", # logging "N", # naming "NPY", # numpy "PD", # pandas + "PERF", # performance "PIE", # misc lints + "PL", # pylint "PT", # pytest style "PTH", # use pathlib "PYI", # type hints "Q", # quotes "RSE", # raises "RET", # returns + "RUF", # ruff "S", # bandit (security) "SIM", # simplicity "SLF", # self "SLOT", # slots "TC", # type checking "TID", # tidy imports + "TRY", # trys "T10", # debugger "T20", # printing + "UP", # pyupgrade "YTT" # flake8 ] ignore = [ diff --git a/workflows/backfill_datamanager.py b/workflows/backfill_datamanager.py index c06cf692b..d1bcd1724 100644 --- a/workflows/backfill_datamanager.py +++ b/workflows/backfill_datamanager.py @@ -1,5 +1,4 @@ from datetime import date, timedelta -from typing import List import httpx from flytekit import task, workflow @@ -13,8 +12,8 @@ def backfill_single_date(base_url: str, day: date) -> int: @workflow -def backfill_equity_bars(base_url: str, start_date: date, end_date: date) -> List[int]: - results: List[int] = [] +def backfill_equity_bars(base_url: str, start_date: date, end_date: date) -> list[int]: + results: list[int] = [] current = start_date while current <= end_date: results.append(backfill_single_date(base_url=base_url, day=current)) From 1d47854ad60c2af3afaec808307b076119c1780b Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Mon, 2 Jun 2025 13:09:50 -0400 Subject: [PATCH 19/41] update infra infra updates --- infrastructure/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index ddfd0ddc8..1aef1c99d 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -13,7 +13,7 @@ ) from services import create_service import topics # noqa: F401 -import buckets # noqa: F401 # registers Pulumi `production_data_bucket` resource +import buckets # noqa: F401 datamanager_service = create_service( From b4d34b735d1ab8ca15ac4ded20069736f82eb4fd Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 4 Jun 2025 16:42:17 -0400 Subject: [PATCH 20/41] fix linting --- .flox/env/manifest.lock | 123 ------------------ .../datamanager/src/datamanager/config.py | 3 +- .../src/positionmanager/clients.py | 8 +- .../src/positionmanager/main.py | 7 +- .../src/predictionengine/dataset.py | 38 +++--- .../gated_residual_network.py | 16 +-- .../long_short_term_memory.py | 19 +-- .../src/predictionengine/loss_function.py | 17 +-- .../src/predictionengine/main.py | 45 ++++--- .../miniature_temporal_fusion_transformer.py | 76 ++++++----- .../src/predictionengine/models.py | 3 +- .../multi_head_self_attention.py | 42 +++--- .../src/predictionengine/post_processor.py | 36 ++--- .../predictionengine/tests/test_dataset.py | 41 ++++-- .../tests/test_gated_residual_network.py | 14 +- .../tests/test_long_short_term_memory.py | 35 +++-- .../tests/test_loss_function.py | 16 ++- .../tests/test_multi_head_self_attention.py | 32 +++-- .../tests/test_post_processor.py | 13 +- .../tests/test_ticker_embedding.py | 1 + infrastructure/__main__.py | 14 +- infrastructure/services.py | 14 +- workflows/prediction_model.py | 11 +- 23 files changed, 296 insertions(+), 328 deletions(-) diff --git a/.flox/env/manifest.lock b/.flox/env/manifest.lock index 964e64525..ff15369b9 100644 --- a/.flox/env/manifest.lock +++ b/.flox/env/manifest.lock @@ -12,9 +12,6 @@ "pulumi-python": { "pkg-path": "pulumiPackages.pulumi-python" }, - "pulumictl": { - "pkg-path": "pulumictl" - }, "ruff": { "pkg-path": "ruff" }, @@ -398,126 +395,6 @@ "group": "toplevel", "priority": 5 }, - { - "attr_path": "pulumictl", - "broken": false, - "derivation": "/nix/store/kx43jzcfslw28byvs6h5ngsgl432pvvv-pulumictl-0.0.49.drv", - "description": "Swiss Army Knife for Pulumi Development", - "install_id": "pulumictl", - "license": "Apache-2.0", - "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e", - "name": "pulumictl-0.0.49", - "pname": "pulumictl", - "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e", - "rev_count": 793735, - "rev_date": "2025-05-04T03:14:55Z", - "scrape_date": "2025-05-05T04:19:37.687142Z", - "stabilities": [ - "staging", - "unstable" - ], - "unfree": false, - "version": "0.0.49", - "outputs_to_install": [ - "out" - ], - "outputs": { - "out": "/nix/store/ny69c9bfkf4w179240ch45injfb2ajqr-pulumictl-0.0.49" - }, - "system": "aarch64-darwin", - "group": "toplevel", - "priority": 5 - }, - { - "attr_path": "pulumictl", - "broken": false, - "derivation": "/nix/store/www9nfncvv7l339n8dks22x5vs5lz1mk-pulumictl-0.0.49.drv", - "description": "Swiss Army Knife for Pulumi Development", - "install_id": "pulumictl", - "license": "Apache-2.0", - "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e", - "name": "pulumictl-0.0.49", - "pname": "pulumictl", - "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e", - "rev_count": 793735, - "rev_date": "2025-05-04T03:14:55Z", - "scrape_date": "2025-05-05T04:37:42.118866Z", - "stabilities": [ - "staging", - "unstable" - ], - "unfree": false, - "version": "0.0.49", - "outputs_to_install": [ - "out" - ], - "outputs": { - "out": "/nix/store/xpdh5dijdki4cngh7k7n4rg84i6c28zs-pulumictl-0.0.49" - }, - "system": "aarch64-linux", - "group": "toplevel", - "priority": 5 - }, - { - "attr_path": "pulumictl", - "broken": false, - "derivation": "/nix/store/17wf5x1kk3v5ch5npwhamnix629y07wg-pulumictl-0.0.49.drv", - "description": "Swiss Army Knife for Pulumi Development", - "install_id": "pulumictl", - "license": "Apache-2.0", - "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e", - "name": "pulumictl-0.0.49", - "pname": "pulumictl", - "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e", - "rev_count": 793735, - "rev_date": "2025-05-04T03:14:55Z", - "scrape_date": "2025-05-05T04:54:38.447587Z", - "stabilities": [ - "staging", - "unstable" - ], - "unfree": false, - "version": "0.0.49", - "outputs_to_install": [ - "out" - ], - "outputs": { - "out": "/nix/store/6wmig1w7f3vmfrlyg2qzv21bvacj3as8-pulumictl-0.0.49" - }, - "system": "x86_64-darwin", - "group": "toplevel", - "priority": 5 - }, - { - "attr_path": "pulumictl", - "broken": false, - "derivation": "/nix/store/ib7hqxg7xdf5kyh78jqggzdcs97q1224-pulumictl-0.0.49.drv", - "description": "Swiss Army Knife for Pulumi Development", - "install_id": "pulumictl", - "license": "Apache-2.0", - "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e", - "name": "pulumictl-0.0.49", - "pname": "pulumictl", - "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e", - "rev_count": 793735, - "rev_date": "2025-05-04T03:14:55Z", - "scrape_date": "2025-05-05T05:16:19.858098Z", - "stabilities": [ - "staging", - "unstable" - ], - "unfree": false, - "version": "0.0.49", - "outputs_to_install": [ - "out" - ], - "outputs": { - "out": "/nix/store/rmh9mjkxijxcc7cvjhsqc9657fbw0yyg-pulumictl-0.0.49" - }, - "system": "x86_64-linux", - "group": "toplevel", - "priority": 5 - }, { "attr_path": "ruff", "broken": false, diff --git a/application/datamanager/src/datamanager/config.py b/application/datamanager/src/datamanager/config.py index fa234e05a..089c640bc 100644 --- a/application/datamanager/src/datamanager/config.py +++ b/application/datamanager/src/datamanager/config.py @@ -19,7 +19,8 @@ class Bucket(BaseModel): @computed_field def daily_bars_path(self) -> str: if self.name is None: - raise ValueError("DATA_BUCKET environment variable is required") + msg = "DATA_BUCKET environment variable is required" + raise ValueError(msg) return f"gs://{self.name}/equity/bars/" diff --git a/application/positionmanager/src/positionmanager/clients.py b/application/positionmanager/src/positionmanager/clients.py index 700e76d96..8850d8390 100644 --- a/application/positionmanager/src/positionmanager/clients.py +++ b/application/positionmanager/src/positionmanager/clients.py @@ -12,6 +12,7 @@ class AlpacaClient: def __init__( self, + *, api_key: str | None = None, api_secret: str | None = None, paper: bool = True, @@ -20,14 +21,17 @@ def __init__( msg = "Alpaca API key and secret are required" raise ValueError(msg) - self.trading_client = TradingClient(api_key, api_secret, paper=paper) + self.trading_client: TradingClient = TradingClient( + api_key, api_secret, paper=paper + ) def get_cash_balance(self) -> Money: account = self.trading_client.get_account() cash_balance = getattr(account, "cash", None) if cash_balance is None: - raise ValueError("Cash balance is not available") + msg = "Cash balance is not available" + raise ValueError(msg) return Money.from_float(float(cash_balance)) diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index c13459bdb..1a7670c08 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -3,12 +3,7 @@ from typing import Any import polars as pl -from typing import Dict, Any -from .models import Money, DateRange, PredictionPayload -from .clients import AlpacaClient, DataClient -from .portfolio import PortfolioOptimizer -from prometheus_fastapi_instrumentator import Instrumentator - +import requests from alpaca.common.rest import APIError from fastapi import FastAPI, HTTPException from prometheus_fastapi_instrumentator import Instrumentator diff --git a/application/predictionengine/src/predictionengine/dataset.py b/application/predictionengine/src/predictionengine/dataset.py index a77761e09..a87fcf4aa 100644 --- a/application/predictionengine/src/predictionengine/dataset.py +++ b/application/predictionengine/src/predictionengine/dataset.py @@ -1,8 +1,9 @@ -from typing import Dict, List, Any, Tuple, Generator -from tinygrad.tensor import Tensor +from collections.abc import Generator +from typing import Any + import polars as pl from category_encoders import OrdinalEncoder - +from tinygrad.tensor import Tensor continuous_variable_columns = [ "open_price", @@ -20,13 +21,13 @@ def __init__( batch_size: int, sequence_length: int, sample_count: int, - scalers: Dict[str, Dict[str, Tensor]] = {}, + scalers: dict[str, dict[str, Tensor]] | None = None, ) -> None: - self.batch_size = batch_size - self.sequence_length = sequence_length - self.sample_count = sample_count - self.scalers = scalers if scalers is not None else {} - self.preprocessors: Dict[str, Any] = {} + self.batch_size: int = batch_size + self.sequence_length: int = sequence_length + self.sample_count: int = sample_count + self.scalers: dict[str, dict[str, Tensor]] = scalers or {} + self.preprocessors: dict[str, Any] = {} def __len__(self) -> int: return (self.sample_count + self.batch_size - 1) // self.batch_size @@ -106,7 +107,7 @@ def _encode_tickers(self, data: pl.DataFrame) -> pl.DataFrame: def _compute_scalers(self, data: pl.DataFrame) -> None: if len(self.scalers) == 0: - self.scalers: Dict[str, Dict[str, Tensor]] = {} + self.scalers: dict[str, dict[str, Tensor]] = {} for ticker_key, group in data.group_by("ticker"): ticker = ticker_key[0] means = group[continuous_variable_columns].mean() @@ -118,7 +119,7 @@ def _compute_scalers(self, data: pl.DataFrame) -> None: } def _scale_data(self, data: pl.DataFrame) -> Tensor: - groups: List[Tensor] = [] + groups: list[Tensor] = [] for ticker_key, group in data.group_by("ticker"): ticker = ticker_key[0] means = self.scalers[str(ticker)]["means"] @@ -133,7 +134,8 @@ def _scale_data(self, data: pl.DataFrame) -> Tensor: groups.append(combined_group) if not groups: - raise ValueError("No data available after preprocessing") + msg = "No data available after preprocessing" + raise ValueError(msg) output_data = Tensor.empty(groups[0].shape) return output_data.cat(*groups, dim=0) @@ -150,9 +152,10 @@ def load_data(self, data: pl.DataFrame) -> None: self._compute_scalers(data) self.data = self._scale_data(data) - def get_preprocessors(self) -> Dict[str, Any]: + def get_preprocessors(self) -> dict[str, Any]: if not self.preprocessors: - raise ValueError("Preprocessors have not been initialized.") + msg = "Preprocessors have not been initialized." + raise ValueError(msg) means_by_ticker = { ticker: values["means"] for ticker, values in self.scalers.items() @@ -169,7 +172,7 @@ def get_preprocessors(self) -> Dict[str, Any]: "indices": self.preprocessors["indices"], } - def batches(self) -> Generator[Tuple[Tensor, Tensor, Tensor], None, None]: + def batches(self) -> Generator[tuple[Tensor, Tensor, Tensor], None, None]: close_price_idx = self.preprocessors["indices"]["close_price"] for i in range(0, self.sample_count, self.batch_size): @@ -193,9 +196,8 @@ def batches(self) -> Generator[Tuple[Tensor, Tensor, Tensor], None, None]: ] if not batch_tensors: - raise ValueError( - "Cannot stack empty batch tensors (batch_size must be ≥ 1)" - ) + msg = "Cannot stack empty batch tensors (batch_size must be ≥ 1)" + raise ValueError(msg) if len(batch_tensors) == 1: historical_features = batch_tensors[0].unsqueeze(0) else: diff --git a/application/predictionengine/src/predictionengine/gated_residual_network.py b/application/predictionengine/src/predictionengine/gated_residual_network.py index 13ea9e2cb..617399692 100644 --- a/application/predictionengine/src/predictionengine/gated_residual_network.py +++ b/application/predictionengine/src/predictionengine/gated_residual_network.py @@ -1,7 +1,7 @@ from typing import cast + +from tinygrad.nn import LayerNorm, Linear from tinygrad.tensor import Tensor -from tinygrad.nn import Linear, LayerNorm -from typing import Optional class GatedResidualNetwork: @@ -9,7 +9,7 @@ def __init__( self, input_size: int, hidden_size: int, - output_size: Optional[int] = None, + output_size: int | None = None, ) -> None: output_size = output_size if output_size is not None else input_size @@ -30,18 +30,18 @@ def __init__( def forward( self, - input: Tensor, + input_: Tensor, ) -> Tensor: - hidden_state = self.dense_input(input).relu() + hidden_state = self.dense_input(input_).relu() output_state = self.dense_output(hidden_state) gate_state = self.gate(hidden_state).sigmoid() if self.residual_projection is not None: - residual = self.residual_projection(input) + residual = self.residual_projection(input_) else: - residual = input + residual = input_ - gated_output = cast(Tensor, gate_state * output_state + residual) + gated_output = cast("Tensor", gate_state * output_state + residual) return self.layer_normalizer(gated_output) diff --git a/application/predictionengine/src/predictionengine/long_short_term_memory.py b/application/predictionengine/src/predictionengine/long_short_term_memory.py index f3e4420cd..671d78e7f 100644 --- a/application/predictionengine/src/predictionengine/long_short_term_memory.py +++ b/application/predictionengine/src/predictionengine/long_short_term_memory.py @@ -1,6 +1,5 @@ -from typing import List, Tuple -from tinygrad.tensor import Tensor from tinygrad.nn import LSTMCell +from tinygrad.tensor import Tensor class LongShortTermMemory: @@ -15,16 +14,16 @@ def __init__( self.layer_count = layer_count self.dropout_rate = dropout_rate - self.layers: List[LSTMCell] = [] + self.layers: list[LSTMCell] = [] for index in range(layer_count): input_size = input_size if index == 0 else self.hidden_size self.layers.append(LSTMCell(input_size, self.hidden_size)) def forward( self, - input: Tensor, - ) -> Tuple[Tensor, Tuple[Tensor, Tensor]]: - batch_size, sequence_length, _ = input.shape + input_: Tensor, + ) -> tuple[Tensor, tuple[Tensor, Tensor]]: + batch_size, sequence_length, _ = input_.shape hidden_state = Tensor.zeros( self.layer_count, batch_size, self.hidden_size @@ -36,7 +35,7 @@ def forward( outputs = [] for t in range(int(sequence_length)): - layer_input = input[:, t] + layer_input = input_[:, t] for index, layer in enumerate(self.layers): layer_hidden_state, layer_cell_state = layer( @@ -59,8 +58,10 @@ def forward( outputs.append(hidden_state[-1]) if not outputs: - raise ValueError("Cannot stack empty outputs list") - elif len(outputs) == 1: + msg = "Cannot stack empty outputs list" + raise ValueError(msg) + + if len(outputs) == 1: output_tensor = outputs[0].unsqueeze(1) else: output_tensor = Tensor.stack(outputs[0], *outputs[1:], dim=1) diff --git a/application/predictionengine/src/predictionengine/loss_function.py b/application/predictionengine/src/predictionengine/loss_function.py index 66479e77a..0d3603305 100644 --- a/application/predictionengine/src/predictionengine/loss_function.py +++ b/application/predictionengine/src/predictionengine/loss_function.py @@ -1,6 +1,7 @@ -from tinygrad.tensor import Tensor from typing import cast +from tinygrad.tensor import Tensor + Quantiles = tuple[float, float, float] | tuple[float, float, float, float, float] @@ -11,18 +12,18 @@ def quantile_loss( quantiles = (0.25, 0.5, 0.75) if y_pred.shape != y_true.shape: - raise ValueError( - f"Shape mismatch: y_pred {y_pred.shape} vs y_true {y_true.shape}" - ) + msg = f"Shape mismatch: y_pred {y_pred.shape} vs y_true {y_true.shape}" + raise ValueError(msg) if not all(0 <= q <= 1 for q in quantiles): - raise ValueError("All quantiles must be between 0 and 1") + msg = "All quantiles must be between 0 and 1" + raise ValueError(msg) loss: Tensor = Tensor.zeros(1) - error = cast(Tensor, y_true - y_pred) + error = cast("Tensor", y_true - y_pred) for quantile in quantiles: - quantile_error = cast(Tensor, quantile * error) - quantile_minus_one_error = cast(Tensor, (quantile - 1) * error) + quantile_error = cast("Tensor", quantile * error) + quantile_minus_one_error = cast("Tensor", (quantile - 1) * error) loss += Tensor.maximum(quantile_error, quantile_minus_one_error).mean() return loss diff --git a/application/predictionengine/src/predictionengine/main.py b/application/predictionengine/src/predictionengine/main.py index 3c1c8d4b8..06c5a7cd3 100644 --- a/application/predictionengine/src/predictionengine/main.py +++ b/application/predictionengine/src/predictionengine/main.py @@ -1,17 +1,26 @@ import os import traceback -from typing import AsyncGenerator -from datetime import date, datetime, timedelta +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager -import requests +from datetime import UTC, date, datetime, timedelta +from pathlib import Path + import polars as pl -from fastapi import FastAPI, Request, Response, status, HTTPException -from prometheus_fastapi_instrumentator import Instrumentator +import requests +from fastapi import FastAPI, HTTPException, Request, Response, status from loguru import logger -from .miniature_temporal_fusion_transformer import MiniatureTemporalFusionTransformer +from prometheus_fastapi_instrumentator import Instrumentator + from .dataset import DataSet +from .miniature_temporal_fusion_transformer import MiniatureTemporalFusionTransformer from .models import PredictionResponse +LOOKBACK_DAYS = 30 + + +class LoadError(Exception): + """Raised when loading a file fails due to format or content issues.""" + @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: @@ -55,7 +64,7 @@ def fetch_historical_data( def load_or_initialize_model(data: pl.DataFrame) -> MiniatureTemporalFusionTransformer: dataset = DataSet( batch_size=32, - sequence_length=30, + sequence_length=LOOKBACK_DAYS, sample_count=len(data), ) dataset.load_data(data) @@ -76,22 +85,22 @@ def load_or_initialize_model(data: pl.DataFrame) -> MiniatureTemporalFusionTrans ) model_path = "miniature_temporal_fusion_transformer.safetensor" - if os.path.exists(model_path): + if Path(model_path).exists(): try: model.load(model_path) logger.info("Loaded existing model weights") - except Exception as e: - logger.warning(f"Failed to load model weights: {e}") + except LoadError as e: + logger.error(f"Failed to load model weights: {e}") return model -@application.post("/create-predictions", response_model=PredictionResponse) +@application.post("/create-predictions") async def create_predictions( request: Request, ) -> PredictionResponse: try: - end_date = datetime.now().date() + end_date = datetime.now(tz=UTC).date() start_date = end_date - timedelta(days=30) logger.info(f"Fetching data from {start_date} to {end_date}") @@ -100,7 +109,7 @@ async def create_predictions( ) if data.is_empty(): - raise HTTPException( + raise HTTPException( # noqa: TRY301 status_code=404, detail="No data available for prediction" ) @@ -115,15 +124,15 @@ async def create_predictions( for ticker in unique_tickers: ticker_data = data.filter(pl.col("ticker") == ticker) - if len(ticker_data) < 30: + if len(ticker_data) < LOOKBACK_DAYS: logger.warning(f"Insufficient data for ticker {ticker}") continue - recent_data = ticker_data.tail(30) + recent_data = ticker_data.tail(LOOKBACK_DAYS) dataset = DataSet( batch_size=1, - sequence_length=30, + sequence_length=LOOKBACK_DAYS, sample_count=1, ) dataset.load_data(recent_data) @@ -145,7 +154,7 @@ async def create_predictions( } if not predictions: - raise HTTPException( + raise HTTPException( # noqa: TRY301 status_code=404, detail="No predictions could be generated" ) @@ -157,5 +166,5 @@ async def create_predictions( logger.error(f"Error creating predictions: {e}") logger.error(traceback.format_exc()) raise HTTPException( - status_code=500, detail=f"Internal server error: {str(e)}" + status_code=500, detail=f"Internal server error: {e!s}" ) from e diff --git a/application/predictionengine/src/predictionengine/miniature_temporal_fusion_transformer.py b/application/predictionengine/src/predictionengine/miniature_temporal_fusion_transformer.py index 97f0a5037..36a3dc58d 100644 --- a/application/predictionengine/src/predictionengine/miniature_temporal_fusion_transformer.py +++ b/application/predictionengine/src/predictionengine/miniature_temporal_fusion_transformer.py @@ -1,27 +1,27 @@ -from typing import Dict +import numpy as np +import numpy.typing as npt from category_encoders import OrdinalEncoder -from .ticker_embedding import TickerEmbedding -from .long_short_term_memory import LongShortTermMemory -from .gated_residual_network import GatedResidualNetwork -from .multi_head_self_attention import MultiHeadSelfAttention -from .post_processor import PostProcessor -from tinygrad.tensor import Tensor from tinygrad.nn.optim import Adam from tinygrad.nn.state import ( get_parameters, get_state_dict, - safe_save, - safe_load, load_state_dict, + safe_load, + safe_save, ) -from typing import Tuple, List -import numpy as np +from tinygrad.tensor import Tensor + from .dataset import DataSet +from .gated_residual_network import GatedResidualNetwork +from .long_short_term_memory import LongShortTermMemory from .loss_function import quantile_loss +from .multi_head_self_attention import MultiHeadSelfAttention +from .post_processor import PostProcessor +from .ticker_embedding import TickerEmbedding class MiniatureTemporalFusionTransformer: - def __init__( + def __init__( # noqa: PLR0913 self, input_size: int, hidden_size: int, @@ -30,41 +30,41 @@ def __init__( ticker_count: int, embedding_size: int, attention_head_count: int, - means_by_ticker: Dict[str, Tensor], - standard_deviations_by_ticker: Dict[str, Tensor], + means_by_ticker: dict[str, Tensor], + standard_deviations_by_ticker: dict[str, Tensor], ticker_encoder: OrdinalEncoder, dropout_rate: float, # non-zero indicates training ) -> None: - self.ticker_embedding = TickerEmbedding( + self.ticker_embedding: TickerEmbedding = TickerEmbedding( ticker_count=ticker_count, embedding_size=embedding_size, ) - self.lstm_encoder = LongShortTermMemory( + self.lstm_encoder: LongShortTermMemory = LongShortTermMemory( input_size=input_size + embedding_size, hidden_size=hidden_size, layer_count=layer_count, dropout_rate=dropout_rate, ) - self.feature_processor = GatedResidualNetwork( + self.feature_processor: GatedResidualNetwork = GatedResidualNetwork( input_size=hidden_size, hidden_size=hidden_size, output_size=hidden_size, ) - self.self_attention = MultiHeadSelfAttention( + self.self_attention: MultiHeadSelfAttention = MultiHeadSelfAttention( heads_count=attention_head_count, embedding_size=hidden_size, ) - self.output_layer = GatedResidualNetwork( + self.output_layer: GatedResidualNetwork = GatedResidualNetwork( input_size=hidden_size, hidden_size=hidden_size, output_size=output_size, ) - self.post_processor = PostProcessor( + self.post_processor: PostProcessor = PostProcessor( means_by_ticker=means_by_ticker, standard_deviations_by_ticker=standard_deviations_by_ticker, ticker_encoder=ticker_encoder, @@ -76,7 +76,13 @@ def forward( self, tickers: Tensor, features: Tensor, - ) -> Tuple[Tensor, Tensor, Tuple[np.ndarray, np.ndarray, np.ndarray]]: + ) -> tuple[ + Tensor, + Tensor, + tuple[ + npt.NDArray[np.float64], npt.NDArray[np.float64], npt.NDArray[np.float64] + ], + ]: ticker_embeddings = self.ticker_embedding.forward( tickers ) # (batch_size, embedding_dim) @@ -109,14 +115,14 @@ def train( dataset: DataSet, epoch_count: int, learning_rate: float = 1e-3, - ) -> List[float]: + ) -> list[float]: optimizer = Adam(params=self.parameters, lr=learning_rate) - quantiles = (0.25, 0.50, 0.75) - losses: List[float] = [] + quantiles: tuple[float, float, float] = (0.25, 0.50, 0.75) + losses: list[float] = [] for _ in range(epoch_count): - epoch_loss = 0.0 + epoch_loss: float = 0.0 for tickers, historical_features, targets in dataset.batches(): predictions, _, _ = self.forward( @@ -124,15 +130,15 @@ def train( historical_features, ) - loss = quantile_loss(predictions, targets, quantiles) + loss: Tensor = quantile_loss(predictions, targets, quantiles) optimizer.zero_grad() - loss.backward() + _ = loss.backward() optimizer.step() epoch_loss += loss.numpy().item() - avgerage_epoch_loss = epoch_loss / len(dataset) + avgerage_epoch_loss: float = epoch_loss / len(dataset) losses.append(avgerage_epoch_loss) return losses @@ -150,9 +156,7 @@ def validate( total_loss += loss.item() batch_count += 1 - average_loss = total_loss / batch_count - - return average_loss + return total_loss / batch_count def save( self, @@ -166,14 +170,16 @@ def load( path_and_file: str = "miniature_temporal_fusion_transformer.safetensor", ) -> None: states = safe_load(path_and_file) - load_state_dict(self, states) + _ = load_state_dict(self, states) def predict( self, tickers: Tensor, - input: Tensor, - ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - predictions, _, _ = self.forward(tickers, input) + input_: Tensor, + ) -> tuple[ + npt.NDArray[np.float64], npt.NDArray[np.float64], npt.NDArray[np.float64] + ]: + predictions, _, _ = self.forward(tickers, input_) percentile_25, percentile_50, percentile_75 = ( self.post_processor.post_process_predictions( diff --git a/application/predictionengine/src/predictionengine/models.py b/application/predictionengine/src/predictionengine/models.py index 37ba3e0d9..123d30a33 100644 --- a/application/predictionengine/src/predictionengine/models.py +++ b/application/predictionengine/src/predictionengine/models.py @@ -1,6 +1,5 @@ from pydantic import BaseModel -from typing import Dict class PredictionResponse(BaseModel): - predictions: Dict[str, Dict[str, float]] + predictions: dict[str, dict[str, float]] diff --git a/application/predictionengine/src/predictionengine/multi_head_self_attention.py b/application/predictionengine/src/predictionengine/multi_head_self_attention.py index 5ba0f83b9..c947def60 100644 --- a/application/predictionengine/src/predictionengine/multi_head_self_attention.py +++ b/application/predictionengine/src/predictionengine/multi_head_self_attention.py @@ -1,7 +1,8 @@ -from typing import Tuple, cast -from tinygrad.tensor import Tensor -from tinygrad.nn import Linear +from typing import cast + from tinygrad.dtype import dtypes +from tinygrad.nn import Linear +from tinygrad.tensor import Tensor class MultiHeadSelfAttention: @@ -11,29 +12,32 @@ def __init__( embedding_size: int, ) -> None: if embedding_size % heads_count != 0: - raise ValueError("Embedding dimension must be divisible by heads count") + msg = "Embedding dimension must be divisible by heads count" + raise ValueError(msg) - self.heads_count = heads_count - self.embedding_size = embedding_size - self.heads_dimension = embedding_size // heads_count + self.heads_count: int = heads_count + self.embedding_size: int = embedding_size + self.heads_dimension: int = embedding_size // heads_count - self.query_weight = Linear(self.embedding_size, self.embedding_size) - self.key_weight = Linear(self.embedding_size, self.embedding_size) - self.value_weight = Linear(self.embedding_size, self.embedding_size) + self.query_weight: Linear = Linear(self.embedding_size, self.embedding_size) + self.key_weight: Linear = Linear(self.embedding_size, self.embedding_size) + self.value_weight: Linear = Linear(self.embedding_size, self.embedding_size) - self.fully_connected_out = Linear(self.embedding_size, self.embedding_size) + self.fully_connected_out: Linear = Linear( + self.embedding_size, self.embedding_size + ) - self.scale = Tensor(self.heads_dimension**0.5, dtype=dtypes.float32) + self.scale: Tensor = Tensor(self.heads_dimension**0.5, dtype=dtypes.float32) def forward( self, - input: Tensor, - ) -> Tuple[Tensor, Tensor]: - batch_size, sequence_length, _ = input.shape + input_: Tensor, + ) -> tuple[Tensor, Tensor]: + batch_size, sequence_length, _ = input_.shape - query_weights = self.query_weight(input) - key_weights = self.key_weight(input) - value_weights = self.value_weight(input) + query_weights = self.query_weight(input_) + key_weights = self.key_weight(input_) + value_weights = self.value_weight(input_) query_weights = query_weights.view( (batch_size, sequence_length, self.heads_count, self.heads_dimension), @@ -49,7 +53,7 @@ def forward( query_weights.matmul(key_weights.transpose(-2, -1)) / self.scale ) - attention_weights: Tensor = cast(Tensor, attention_scores).softmax(axis=-1) + attention_weights: Tensor = cast("Tensor", attention_scores).softmax(axis=-1) attention_output = attention_weights.matmul(value_weights) diff --git a/application/predictionengine/src/predictionengine/post_processor.py b/application/predictionengine/src/predictionengine/post_processor.py index d0b615010..7f07c2409 100644 --- a/application/predictionengine/src/predictionengine/post_processor.py +++ b/application/predictionengine/src/predictionengine/post_processor.py @@ -1,29 +1,33 @@ -from typing import Dict, Tuple, Any -from tinygrad.tensor import Tensor -from category_encoders import OrdinalEncoder import numpy as np +import numpy.typing as npt import polars as pl +from category_encoders import OrdinalEncoder +from tinygrad.tensor import Tensor + +TensorMapping = dict[str, Tensor] class PostProcessor: def __init__( self, - means_by_ticker: Dict[str, Tensor], - standard_deviations_by_ticker: Dict[str, Tensor], + means_by_ticker: TensorMapping, + standard_deviations_by_ticker: TensorMapping, ticker_encoder: OrdinalEncoder, ) -> None: - self.means_by_ticker = means_by_ticker - self.standard_deviations_by_ticker = standard_deviations_by_ticker - self.ticker_encoder = ticker_encoder + self.means_by_ticker: TensorMapping = means_by_ticker + self.standard_deviations_by_ticker: TensorMapping = ( + standard_deviations_by_ticker + ) + self.ticker_encoder: OrdinalEncoder = ticker_encoder def post_process_predictions( self, - encoded_tickers: np.ndarray, - predictions: np.ndarray, - ) -> Tuple[ - np.ndarray[Any, np.dtype[np.float64]], - np.ndarray[Any, np.dtype[np.float64]], - np.ndarray[Any, np.dtype[np.float64]], + encoded_tickers: npt.NDArray[np.float64], + predictions: npt.NDArray[np.float64], + ) -> tuple[ + npt.NDArray[np.float64], + npt.NDArray[np.float64], + npt.NDArray[np.float64], ]: decoded_tickers = self.ticker_encoder.inverse_transform( pl.DataFrame( @@ -40,7 +44,9 @@ def post_process_predictions( ticker not in self.means_by_ticker or ticker not in self.standard_deviations_by_ticker ): - raise ValueError(f"Statistics not found for ticker: {ticker}") + msg = f"Statistics not found for ticker: {ticker}" + raise ValueError(msg) + mean = self.means_by_ticker[ticker].numpy() standard_deviation = self.standard_deviations_by_ticker[ticker].numpy() rescaled_predictions[i, :] = predictions[i, :] * standard_deviation + mean diff --git a/application/predictionengine/tests/test_dataset.py b/application/predictionengine/tests/test_dataset.py index c43cb19a6..9d3b804c0 100644 --- a/application/predictionengine/tests/test_dataset.py +++ b/application/predictionengine/tests/test_dataset.py @@ -1,5 +1,8 @@ +from typing import NamedTuple + import polars as pl import pytest + from application.predictionengine.src.predictionengine.dataset import DataSet @@ -10,10 +13,18 @@ def test_dataset_initialization() -> None: sample_count=3, ) - assert dataset.batch_size == 2 - assert dataset.sequence_length == 3 - assert dataset.sample_count == 3 - assert len(dataset) == 2 + class Expected(NamedTuple): + batch_size: int = 2 + sequence_length: int = 3 + sample_count: int = 3 + observations: int = 2 + + expected = Expected() + + assert dataset.batch_size == expected.batch_size + assert dataset.sequence_length == expected.sequence_length + assert dataset.sample_count == expected.sample_count + assert len(dataset) == expected.observations def test_dataset_load_data() -> None: @@ -102,12 +113,26 @@ def test_dataset_batches() -> None: dataset.load_data(data) + class Expected(NamedTuple): + batch_size: int = 1 + sequence_length: int = 2 + sample_count: int = 3 + observations: int = 2 + features: int = 6 + target: int = 1 + + expected = Expected() + batch_count = 0 for tickers, features, targets in dataset.batches(): batch_count += 1 - assert tickers.shape[0] == 1 # batch_size - assert features.shape == (1, 2, 6) # batch_size, sequence_length, features - assert targets.shape == (1, 1) # batch_size, 1 + assert tickers.shape[0] == expected.batch_size + assert features.shape == ( + expected.batch_size, + expected.sequence_length, + expected.features, + ) + assert targets.shape == (expected.batch_size, expected.target) assert batch_count > 0 @@ -120,4 +145,4 @@ def test_dataset_preprocessors_validation() -> None: ) with pytest.raises(ValueError, match="Preprocessors have not been initialized"): - dataset.get_preprocessors() + _ = dataset.get_preprocessors() diff --git a/application/predictionengine/tests/test_gated_residual_network.py b/application/predictionengine/tests/test_gated_residual_network.py index 617676302..dd6a6d432 100644 --- a/application/predictionengine/tests/test_gated_residual_network.py +++ b/application/predictionengine/tests/test_gated_residual_network.py @@ -1,9 +1,13 @@ -from tinygrad.tensor import Tensor import numpy as np +from numpy.random import PCG64, Generator +from tinygrad.tensor import Tensor + from application.predictionengine.src.predictionengine.gated_residual_network import ( GatedResidualNetwork, ) +rng = Generator(PCG64()) + def test_gated_residual_network_initialization() -> None: input_size = 64 @@ -26,7 +30,7 @@ def test_gated_residual_network_initialization() -> None: def test_gated_residual_network_forward() -> None: grn = GatedResidualNetwork(input_size=32, hidden_size=64, output_size=32) - input_tensor = Tensor(np.random.randn(8, 32)) + input_tensor = Tensor(rng.standard_normal((8, 32))) output = grn.forward(input_tensor) assert output.shape == (8, 32) @@ -35,7 +39,7 @@ def test_gated_residual_network_forward() -> None: def test_gated_residual_network_different_sizes() -> None: grn = GatedResidualNetwork(input_size=16, hidden_size=32, output_size=8) - input_tensor = Tensor(np.random.randn(4, 16)) + input_tensor = Tensor(rng.standard_normal((4, 16))) output = grn.forward(input_tensor) assert output.shape == (4, 8) @@ -44,7 +48,7 @@ def test_gated_residual_network_different_sizes() -> None: def test_gated_residual_network_single_sample() -> None: grn = GatedResidualNetwork(input_size=10, hidden_size=20, output_size=10) - input_tensor = Tensor(np.random.randn(1, 10)) + input_tensor = Tensor(rng.standard_normal((1, 10))) output = grn.forward(input_tensor) assert output.shape == (1, 10) @@ -53,7 +57,7 @@ def test_gated_residual_network_single_sample() -> None: def test_gated_residual_network_consistency() -> None: grn = GatedResidualNetwork(input_size=16, hidden_size=32, output_size=16) - input_tensor = Tensor(np.random.randn(2, 16)) + input_tensor = Tensor(rng.standard_normal((2, 16))) output1 = grn.forward(input_tensor) output2 = grn.forward(input_tensor) diff --git a/application/predictionengine/tests/test_long_short_term_memory.py b/application/predictionengine/tests/test_long_short_term_memory.py index dd631c1bf..2e08fbba9 100644 --- a/application/predictionengine/tests/test_long_short_term_memory.py +++ b/application/predictionengine/tests/test_long_short_term_memory.py @@ -1,18 +1,31 @@ -from tinygrad.tensor import Tensor +from typing import NamedTuple + import numpy as np +from numpy.random import PCG64, Generator +from tinygrad.tensor import Tensor + from application.predictionengine.src.predictionengine.long_short_term_memory import ( LongShortTermMemory, ) +rng = Generator(PCG64()) + def test_lstm_initialization() -> None: lstm = LongShortTermMemory( input_size=32, hidden_size=64, layer_count=2, dropout_rate=0.1 ) - assert lstm.hidden_size == 64 - assert lstm.layer_count == 2 - assert lstm.dropout_rate == 0.1 + class Expected(NamedTuple): + hidden_state: int = 64 + layer_count: int = 2 + dropout_rate: float = 0.1 + + expected = Expected(hidden_state=64, layer_count=2, dropout_rate=0.1) + + assert lstm.hidden_size == expected.hidden_state + assert lstm.layer_count == expected.layer_count + assert lstm.dropout_rate == expected.dropout_rate def test_lstm_forward() -> None: @@ -20,12 +33,14 @@ def test_lstm_forward() -> None: input_size=16, hidden_size=32, layer_count=1, dropout_rate=0.0 ) - input_tensor = Tensor(np.random.randn(4, 10, 16)) + input_tensor = Tensor(rng.standard_normal((4, 10, 16))) output, hidden_state = lstm.forward(input_tensor) + expected_hidden_state = 2 + assert output.shape == (4, 10, 32) assert isinstance(hidden_state, tuple) - assert len(hidden_state) == 2 + assert len(hidden_state) == expected_hidden_state def test_lstm_different_sequence_lengths() -> None: @@ -34,7 +49,7 @@ def test_lstm_different_sequence_lengths() -> None: ) for seq_len in [5, 10, 20]: - input_tensor = Tensor(np.random.randn(2, seq_len, 8)) + input_tensor = Tensor(rng.standard_normal((2, seq_len, 8))) output, hidden_state = lstm.forward(input_tensor) assert output.shape == (2, seq_len, 16) @@ -45,7 +60,7 @@ def test_lstm_multiple_layers() -> None: input_size=10, hidden_size=20, layer_count=3, dropout_rate=0.0 ) - input_tensor = Tensor(np.random.randn(2, 5, 10)) + input_tensor = Tensor(rng.standard_normal((2, 5, 10))) output, hidden_state = lstm.forward(input_tensor) assert output.shape == (2, 5, 20) @@ -57,7 +72,7 @@ def test_lstm_single_timestep() -> None: input_size=12, hidden_size=24, layer_count=1, dropout_rate=0.0 ) - input_tensor = Tensor(np.random.randn(3, 1, 12)) + input_tensor = Tensor(rng.standard_normal((3, 1, 12))) output, _ = lstm.forward(input_tensor) assert output.shape == (3, 1, 24) @@ -68,7 +83,7 @@ def test_lstm_consistency() -> None: input_size=6, hidden_size=12, layer_count=1, dropout_rate=0.0 ) - input_tensor = Tensor(np.random.randn(1, 3, 6)) + input_tensor = Tensor(rng.standard_normal((1, 3, 6))) first_output, _ = lstm.forward(input_tensor) second_output, _ = lstm.forward(input_tensor) diff --git a/application/predictionengine/tests/test_loss_function.py b/application/predictionengine/tests/test_loss_function.py index 6bebf839f..d9e5bd43f 100644 --- a/application/predictionengine/tests/test_loss_function.py +++ b/application/predictionengine/tests/test_loss_function.py @@ -1,10 +1,14 @@ -from tinygrad.tensor import Tensor import numpy as np import pytest +from numpy.random import PCG64, Generator +from tinygrad.tensor import Tensor + from application.predictionengine.src.predictionengine.loss_function import ( quantile_loss, ) +rng = Generator(PCG64()) + def test_quantile_loss_basic() -> None: predictions = Tensor([[1.0], [2.0], [3.0]]) @@ -14,7 +18,7 @@ def test_quantile_loss_basic() -> None: loss = quantile_loss(predictions, targets, quantiles) assert isinstance(loss, Tensor) - assert loss.shape == () or loss.shape == (1,) + assert len(loss.shape) == 0 or loss.shape in [(), (0,), (1,)] def test_quantile_loss_multiple_samples() -> None: @@ -22,10 +26,10 @@ def test_quantile_loss_multiple_samples() -> None: targets = Tensor([[2.5], [5.5]]) quantiles = (0.25, 0.5, 0.75) - loss = quantile_loss(predictions, targets, quantiles) + loss: Tensor = quantile_loss(predictions, targets, quantiles) assert isinstance(loss, Tensor) - assert loss.shape == () or loss.shape == (1,) + assert len(loss.shape) == 0 or loss.shape in [(), (0,), (1,)] def test_quantile_loss_perfect_prediction() -> None: @@ -51,8 +55,8 @@ def test_quantile_loss_different_quantiles() -> None: def test_quantile_loss_shapes() -> None: for batch_size in [1, 2, 4, 8]: - predictions = Tensor(np.random.randn(batch_size, 1).astype(np.float32)) - targets = Tensor(np.random.randn(batch_size, 1).astype(np.float32)) + predictions = Tensor(rng.standard_normal((batch_size, 1)).astype(np.float32)) + targets = Tensor(rng.standard_normal((batch_size, 1)).astype(np.float32)) quantiles = (0.25, 0.5, 0.75) loss = quantile_loss(predictions, targets, quantiles) diff --git a/application/predictionengine/tests/test_multi_head_self_attention.py b/application/predictionengine/tests/test_multi_head_self_attention.py index d818eea00..692f99b2c 100644 --- a/application/predictionengine/tests/test_multi_head_self_attention.py +++ b/application/predictionengine/tests/test_multi_head_self_attention.py @@ -1,26 +1,34 @@ +from numpy.random import PCG64, Generator from tinygrad.tensor import Tensor -import numpy as np -from application.predictionengine.src.predictionengine.multi_head_self_attention import ( + +from application.predictionengine.src.predictionengine.multi_head_self_attention import ( # noqa: E501 MultiHeadSelfAttention, ) +rng = Generator(PCG64()) + def test_multi_head_attention_initialization() -> None: + heads_count = 8 + embedding_size = 64 attention = MultiHeadSelfAttention(heads_count=8, embedding_size=64) - assert attention.heads_count == 8 - assert attention.embedding_size == 64 + assert attention.heads_count == heads_count + assert attention.embedding_size == embedding_size def test_multi_head_attention_forward() -> None: attention = MultiHeadSelfAttention(heads_count=4, embedding_size=32) - input_tensor = Tensor(np.random.randn(2, 10, 32)) + input_tensor = Tensor(rng.standard_normal((2, 10, 32))) output, attention_weights = attention.forward(input_tensor) - assert output.shape == (2, 10, 32) - assert attention_weights.shape[0] == 2 # batch size - assert attention_weights.shape[1] == 4 # heads count + batch_size = 2 + heads_count = 4 + + assert output.shape == (batch_size, 10, 32) + assert attention_weights.shape[0] == batch_size + assert attention_weights.shape[1] == heads_count def test_multi_head_attention_different_heads() -> None: @@ -30,7 +38,7 @@ def test_multi_head_attention_different_heads() -> None: heads_count=heads_count, embedding_size=embedding_size ) - input_tensor = Tensor(np.random.randn(1, 5, embedding_size)) + input_tensor = Tensor(rng.standard_normal((1, 5, embedding_size))) output, attention_weights = attention.forward(input_tensor) assert output.shape == (1, 5, embedding_size) @@ -40,7 +48,7 @@ def test_multi_head_attention_different_heads() -> None: def test_multi_head_attention_single_sequence() -> None: attention = MultiHeadSelfAttention(heads_count=2, embedding_size=16) - input_tensor = Tensor(np.random.randn(1, 1, 16)) + input_tensor = Tensor(rng.standard_normal((1, 1, 16))) output, _ = attention.forward(input_tensor) assert output.shape == (1, 1, 16) @@ -50,7 +58,7 @@ def test_multi_head_attention_longer_sequences() -> None: attention = MultiHeadSelfAttention(heads_count=4, embedding_size=64) for seq_len in [10, 20, 50]: - input_tensor = Tensor(np.random.randn(1, seq_len, 64)) + input_tensor = Tensor(rng.standard_normal((1, seq_len, 64))) output, _ = attention.forward(input_tensor) assert output.shape == (1, seq_len, 64) @@ -60,7 +68,7 @@ def test_multi_head_attention_batch_processing() -> None: attention = MultiHeadSelfAttention(heads_count=2, embedding_size=32) for batch_size in [1, 2, 4, 8]: - input_tensor = Tensor(np.random.randn(batch_size, 5, 32)) + input_tensor = Tensor(rng.standard_normal((batch_size, 5, 32))) output, attention_weights = attention.forward(input_tensor) assert output.shape == (batch_size, 5, 32) diff --git a/application/predictionengine/tests/test_post_processor.py b/application/predictionengine/tests/test_post_processor.py index 0caf8c0cb..dd08caaa4 100644 --- a/application/predictionengine/tests/test_post_processor.py +++ b/application/predictionengine/tests/test_post_processor.py @@ -1,7 +1,8 @@ -from category_encoders import OrdinalEncoder +import numpy as np import polars as pl +from category_encoders import OrdinalEncoder from tinygrad.tensor import Tensor -import numpy as np + from application.predictionengine.src.predictionengine.post_processor import ( PostProcessor, ) @@ -66,9 +67,11 @@ def test_post_processor_predictions() -> None: assert isinstance(percentile_25, np.ndarray) assert isinstance(percentile_50, np.ndarray) assert isinstance(percentile_75, np.ndarray) - assert len(percentile_25) == 2 - assert len(percentile_50) == 2 - assert len(percentile_75) == 2 + + percentile_size = 2 + assert len(percentile_25) == percentile_size + assert len(percentile_50) == percentile_size + assert len(percentile_75) == percentile_size assert np.all(percentile_25 <= percentile_50) assert np.all(percentile_50 <= percentile_75) diff --git a/application/predictionengine/tests/test_ticker_embedding.py b/application/predictionengine/tests/test_ticker_embedding.py index abb70d349..5928832d6 100644 --- a/application/predictionengine/tests/test_ticker_embedding.py +++ b/application/predictionengine/tests/test_ticker_embedding.py @@ -1,4 +1,5 @@ from tinygrad.tensor import Tensor + from application.predictionengine.src.predictionengine.ticker_embedding import ( TickerEmbedding, ) diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index 1aef1c99d..a309cac1b 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -1,20 +1,20 @@ import base64 -from pulumi_gcp import pubsub, cloudscheduler -from project import platform_service_account + +import buckets # noqa: F401 +import topics from environment_variables import ( - create_environment_variable, ALPACA_API_KEY_ID, ALPACA_API_SECRET_KEY, - GCP_PROJECT, DATA_BUCKET, DUCKDB_ACCESS_KEY, DUCKDB_SECRET, + GCP_PROJECT, POLYGON_API_KEY, + create_environment_variable, ) +from project import platform_service_account +from pulumi_gcp import cloudscheduler, pubsub from services import create_service -import topics # noqa: F401 -import buckets # noqa: F401 - datamanager_service = create_service( name="datamanager", diff --git a/infrastructure/services.py b/infrastructure/services.py index 0f7ca6772..5ad1c8e29 100644 --- a/infrastructure/services.py +++ b/infrastructure/services.py @@ -1,9 +1,10 @@ -from pathlib import Path import tomllib +from pathlib import Path import project import pulumi_docker_build as docker_build from environment_variables import ENVIRONMENT_VARIABLE +from pulumi import ResourceOptions from pulumi.config import Config from pulumi_gcp.cloudrun import ( Service, @@ -27,13 +28,15 @@ def create_service( with Path("pyproject.toml").open("rb") as f: project_data = tomllib.load(f) version = project_data.get("project", {}).get("version") - if not version: - raise ValueError("Version not found in pyproject.toml") + except (FileNotFoundError, tomllib.TOMLDecodeError, ValueError) as e: - raise RuntimeError(f"Failed to read version from pyproject.toml: {e}") from e + msg = f"Failed to read version from pyproject.toml: {e}" + raise RuntimeError(msg) from e + service_dir = Path("../application") / name if not service_dir.exists(): - raise FileNotFoundError(f"Service directory not found: {service_dir}") + msg = f"Service directory not found: {service_dir}" + raise FileNotFoundError(msg) image = docker_build.Image( f"{name}-image", @@ -55,6 +58,7 @@ def create_service( return Service( name, + opts=ResourceOptions(depends_on=[image]), location=project.REGION, template=ServiceTemplateArgs( spec=ServiceTemplateSpecArgs( diff --git a/workflows/prediction_model.py b/workflows/prediction_model.py index 986821143..c381e0c05 100644 --- a/workflows/prediction_model.py +++ b/workflows/prediction_model.py @@ -4,14 +4,14 @@ import uuid from datetime import datetime from pathlib import Path -from typing import Any, Dict, List +from typing import Any import requests from flytekit import task, workflow @task -def fetch_data(start_date: datetime, end_date: datetime) -> List[Dict[str, Any]]: +def fetch_data(start_date: datetime, end_date: datetime) -> list[dict[str, Any]]: base_url = os.getenv("DATAMANAGER_BASE_URL", "http://localhost:8080") response = requests.get( f"{base_url}/equity-bars", @@ -23,7 +23,7 @@ def fetch_data(start_date: datetime, end_date: datetime) -> List[Dict[str, Any]] @task -def train_dummy_model(data: List[Dict[str, Any]]) -> bytes: +def train_dummy_model(data: list[dict[str, Any]]) -> bytes: """Train a trivial model that stores the average close price.""" close_prices = [row.get("close_price", 0.0) for row in data] mean_close = statistics.mean(close_prices) if close_prices else 0.0 @@ -34,7 +34,7 @@ def train_dummy_model(data: List[Dict[str, Any]]) -> bytes: @task def store_model(model_bytes: bytes) -> str: """Store the serialized model in blob storage.""" - bucket_path = os.getenv("MODEL_BUCKET", "/tmp") + bucket_path = os.getenv("MODEL_BUCKET") filename = f"model-{uuid.uuid4().hex}.pkl" path = Path(bucket_path) / filename path.write_bytes(model_bytes) @@ -45,5 +45,4 @@ def store_model(model_bytes: bytes) -> str: def training_workflow(start_date: datetime, end_date: datetime) -> None: data = fetch_data(start_date=start_date, end_date=end_date) model_bytes = train_dummy_model(data=data) - artifact_path = store_model(model_bytes=model_bytes) - return + store_model(model_bytes=model_bytes) From 03e64e3a69f3dce5760d44416ed9e9d6367af242 Mon Sep 17 00:00:00 2001 From: Chris Addy Date: Wed, 4 Jun 2025 20:15:27 -0400 Subject: [PATCH 21/41] Update pyproject.toml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 24461cdc0..707b2d2d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,8 @@ requires-python = ">=3.12" dependencies = [ "flytekit>=1.15.4", "httpx>=0.28.1", - "pulumi-docker-build>=0.0.12", - "pulumi-gcp>=8.32.0", + "pulumi-docker-build>=0.0.6", + "pulumi-gcp>=8.16.0", "requests>=2.32.3", ] From c6f435679edd29fe6417ac921d3f65d9be2f61af Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 15:29:47 -0400 Subject: [PATCH 22/41] add type annotations where missing --- application/datamanager/features/environment.py | 2 +- .../datamanager/features/steps/equity_bars_steps.py | 4 ++-- .../datamanager/features/steps/health_steps.py | 3 +++ pyproject.toml | 12 ++++++++++-- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/application/datamanager/features/environment.py b/application/datamanager/features/environment.py index 796fe865c..5c659cbd1 100644 --- a/application/datamanager/features/environment.py +++ b/application/datamanager/features/environment.py @@ -1,7 +1,7 @@ import os - from behave.runner import Context +from behave.runner import Context def before_all(context: Context) -> None: context.base_url = os.environ.get("BASE_URL", "http://datamanager:8080") diff --git a/application/datamanager/features/steps/equity_bars_steps.py b/application/datamanager/features/steps/equity_bars_steps.py index 3e3b386f2..55b4f8acb 100644 --- a/application/datamanager/features/steps/equity_bars_steps.py +++ b/application/datamanager/features/steps/equity_bars_steps.py @@ -5,7 +5,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent)) import requests -from behave import given, then, when +from behave import given, when, then from behave.runner import Context @@ -55,7 +55,7 @@ def step_impl(context: Context, endpoint: str, date_str: str) -> None: @then('the equity bars data for "{date_str}" should be deleted') -def step_impl_equity_bars(context: Context, date_str: str) -> None: # noqa: ARG001 +def step_impl_equity_bars(context: Context, date_str: str) -> None: if os.environ.get("GCP_GCS_BUCKET"): assert True, "GCS bucket deletion check would go here" else: diff --git a/application/datamanager/features/steps/health_steps.py b/application/datamanager/features/steps/health_steps.py index b6be9a63c..d1017ed96 100644 --- a/application/datamanager/features/steps/health_steps.py +++ b/application/datamanager/features/steps/health_steps.py @@ -1,3 +1,6 @@ +from behave import when +from behave.runner import Context + import requests from behave import when from behave.runner import Context diff --git a/pyproject.toml b/pyproject.toml index 707b2d2d9..fbab8fecb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,8 @@ requires-python = ">=3.12" dependencies = [ "flytekit>=1.15.4", "httpx>=0.28.1", - "pulumi-docker-build>=0.0.6", - "pulumi-gcp>=8.16.0", + "pulumi-docker-build>=0.0.12", + "pulumi-gcp>=8.32.0", "requests>=2.32.3", ] @@ -139,8 +139,16 @@ ignore = [ "**/tests/**/*.py" = ["S101"] "**/features/steps/**/*.py" = ["S101"] +[tool.ruff] +select = [ + "ANN" +] + [tool.ty.rules] unresolved-import = "ignore" invalid-return-type = "error" invalid-argument-type = "error" unresolved-reference = "error" + +[tool.pyright] +reportMissingImports = "none" From 27670146925304a979277cd40768e95e0a82028d Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 15:45:57 -0400 Subject: [PATCH 23/41] ruff fixes From f02beec0ac4904ea3a089711cb924035d04c4727 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 15:33:27 -0400 Subject: [PATCH 24/41] add fastapi linting fix bandit issues fixing ruff issues --- pyproject.toml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fbab8fecb..3bde44403 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,10 +139,18 @@ ignore = [ "**/tests/**/*.py" = ["S101"] "**/features/steps/**/*.py" = ["S101"] -[tool.ruff] +[tool.ruff.lint] select = [ - "ANN" + "ANN", # type annotations + "ASYNC", + "ERA", # dead code + "FAST", # fastapi + "S", # bandit (security) + "YTT" # flake8 ] +[tool.ruff.lint.per-file-ignores] +"**/tests/**/*.py" = ["S101"] +"**/features/steps/**/*.py" = ["S101"] [tool.ty.rules] unresolved-import = "ignore" From bfcc52f893f2a74b3bd5e9cf99118e5bf3a69430 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 16:41:39 -0400 Subject: [PATCH 25/41] no blind exceptions update to fix blind exceptions --- application/datamanager/src/datamanager/main.py | 4 ++-- .../positionmanager/src/positionmanager/main.py | 2 ++ .../tests/test_positionmanager_main.py | 11 ++++++----- pyproject.toml | 1 + 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index 9de041380..14bb0dab7 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -121,7 +121,7 @@ async def get_equity_bars( ) except ( - requests.RequestException, + requests.RequestsException, ComputeError, IOException, GoogleAPIError, @@ -166,7 +166,7 @@ async def fetch_equity_bars(request: Request, summary_date: SummaryDate) -> Bars partition_by=["year", "month", "day"], ) except ( - requests.RequestException, + requests.RequestsException, ComputeError, IOException, GoogleAPIError, diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index 1a7670c08..69c8e48e7 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -1,3 +1,5 @@ +from fastapi import FastAPI, HTTPException +import requests import os from datetime import UTC, datetime, timedelta from typing import Any diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index a5ce64c9b..2d54241e0 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -1,3 +1,6 @@ +from fastapi.testclient import TestClient +from fastapi import HTTPException + import unittest from decimal import Decimal from unittest.mock import MagicMock, patch @@ -85,8 +88,7 @@ def test_create_position_success( def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None: # noqa: N803 mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.get_cash_balance.side_effect = HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Error getting cash balance", + status_code=500, detail="Error getting cash balance" ) MockAlpacaClient.return_value = mock_alpaca_instance @@ -128,14 +130,13 @@ def test_delete_positions_error( ) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.clear_positions.side_effect = HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Error getting cash balance", + status_code=500, detail="Error getting cash balance" ) MockAlpacaClient.return_value = mock_alpaca_instance response = client.delete("/positions") - assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + assert response.status_code == 500 assert "Error" in response.json()["detail"] MockAlpacaClient.assert_called_once() diff --git a/pyproject.toml b/pyproject.toml index 3bde44403..ea9971265 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -143,6 +143,7 @@ ignore = [ select = [ "ANN", # type annotations "ASYNC", + "BLE", # no blind exceptions "ERA", # dead code "FAST", # fastapi "S", # bandit (security) From 4bd5c41b9474f2e60fc4c27eb2d6e732aad51598 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 17:11:44 -0400 Subject: [PATCH 26/41] fix boolean traps --- application/datamanager/src/datamanager/main.py | 4 ++-- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index 14bb0dab7..9de041380 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -121,7 +121,7 @@ async def get_equity_bars( ) except ( - requests.RequestsException, + requests.RequestException, ComputeError, IOException, GoogleAPIError, @@ -166,7 +166,7 @@ async def fetch_equity_bars(request: Request, summary_date: SummaryDate) -> Bars partition_by=["year", "month", "day"], ) except ( - requests.RequestsException, + requests.RequestException, ComputeError, IOException, GoogleAPIError, diff --git a/pyproject.toml b/pyproject.toml index ea9971265..60f160c7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -146,6 +146,7 @@ select = [ "BLE", # no blind exceptions "ERA", # dead code "FAST", # fastapi + "FBT", # boolean traps "S", # bandit (security) "YTT" # flake8 ] From 767c4c1b0b4d5e38f938cdb9c42b3a40822931e7 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 17:12:24 -0400 Subject: [PATCH 27/41] fix boolean traps add bugbear add comma linting fix bugbear and commas add timezones --- .../positionmanager/tests/test_positionmanager_main.py | 6 ++++-- infrastructure/images.py | 2 +- pyproject.toml | 9 +++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index 2d54241e0..db3748b9b 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -88,7 +88,8 @@ def test_create_position_success( def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None: # noqa: N803 mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.get_cash_balance.side_effect = HTTPException( - status_code=500, detail="Error getting cash balance" + status_code=500, + detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance @@ -130,7 +131,8 @@ def test_delete_positions_error( ) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.clear_positions.side_effect = HTTPException( - status_code=500, detail="Error getting cash balance" + status_code=500, + detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance diff --git a/infrastructure/images.py b/infrastructure/images.py index bb2408e9e..b7cf1d224 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -17,7 +17,7 @@ tags = [ "latest", - datetime.now(tz=UTC).strftime("%Y%m%d"), + datetime.now(tz=timezone.utc).strftime("%Y%m%d"), ] images = {} diff --git a/pyproject.toml b/pyproject.toml index 60f160c7f..3e013db35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,15 +141,24 @@ ignore = [ [tool.ruff.lint] select = [ + "A", # flake8 builtins "ANN", # type annotations "ASYNC", + "B", # bugbear + "COM", # commas + "C4", # comprehensions "BLE", # no blind exceptions + "DTZ", # datetimes "ERA", # dead code "FAST", # fastapi "FBT", # boolean traps "S", # bandit (security) "YTT" # flake8 ] +ignore = [ + "COM812", +] + [tool.ruff.lint.per-file-ignores] "**/tests/**/*.py" = ["S101"] "**/features/steps/**/*.py" = ["S101"] From 7c1320c88efe898f0af6a7bc3c2d3ddb23b865c0 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 23:19:57 -0400 Subject: [PATCH 28/41] lint error messages no fixme/todos implicit string concatenation import conventions logging pathlib --- .../features/steps/equity_bars_steps.py | 2 +- .../datamanager/src/datamanager/main.py | 1 + infrastructure/images.py | 1 + pyproject.toml | 47 ++++++++++++++----- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/application/datamanager/features/steps/equity_bars_steps.py b/application/datamanager/features/steps/equity_bars_steps.py index 55b4f8acb..5700e7433 100644 --- a/application/datamanager/features/steps/equity_bars_steps.py +++ b/application/datamanager/features/steps/equity_bars_steps.py @@ -55,7 +55,7 @@ def step_impl(context: Context, endpoint: str, date_str: str) -> None: @then('the equity bars data for "{date_str}" should be deleted') -def step_impl_equity_bars(context: Context, date_str: str) -> None: +def step_impl_equity_bars(context: Context, date_str: str) -> None: # noqa: ARG001 if os.environ.get("GCP_GCS_BUCKET"): assert True, "GCS bucket deletion check would go here" else: diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index 9de041380..23aefa2d1 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -9,6 +9,7 @@ import polars as pl import pyarrow as pa import pyarrow.lib +from duckdb import IOException import requests from duckdb import IOException from fastapi import FastAPI, HTTPException, Request, Response, status diff --git a/infrastructure/images.py b/infrastructure/images.py index b7cf1d224..b305f4d6a 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -5,6 +5,7 @@ import pulumi_docker_build as docker_build from loguru import logger from pulumi import Config +from loguru import logger config = Config() dockerhub_username = config.require_secret("dockerhub_username") diff --git a/pyproject.toml b/pyproject.toml index 3e013db35..5207fcc01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,19 +141,42 @@ ignore = [ [tool.ruff.lint] select = [ - "A", # flake8 builtins - "ANN", # type annotations + "A", # flake8 builtins + "ANN", # type annotations + "ARG", # unused args "ASYNC", - "B", # bugbear - "COM", # commas - "C4", # comprehensions - "BLE", # no blind exceptions - "DTZ", # datetimes - "ERA", # dead code - "FAST", # fastapi - "FBT", # boolean traps - "S", # bandit (security) - "YTT" # flake8 + "B", # bugbear + "COM", # commas + "C4", # comprehensions + "BLE", # no blind exceptions + "DTZ", # datetimes + "EM", # error messages + "ERA", # dead code + "EXE", # executables + "FA", # future annotations + "FAST", # fastapi + "FIX", # no fixme/todo comments + "FBT", # boolean traps + "G", # logging format + "ICN", # import conventions + "ISC", # implicit string concatenation + "LOG", # logging + "Q", # quotes + "PIE", # misc lints + "PT", # pytest style + "PTH", # use pathlib + "PYI", # type hints + "RSE", # raises + "RET", # returns + "S", # bandit (security) + "SIM", # simplicity + "SLF", # self + "SLOT", # slots + "TC", # type checking + "TID", # tidy imports + "T10", # debugger + "T20", # printing + "YTT" # flake8 ] ignore = [ "COM812", From 1ec02e3e95b0dfb36d7ade5ed5c03f21257db640 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 23:20:26 -0400 Subject: [PATCH 29/41] a bunch of linting sort imports fix naming conventions --- application/datamanager/features/environment.py | 1 + .../datamanager/features/steps/equity_bars_steps.py | 2 +- application/datamanager/features/steps/health_steps.py | 1 + application/datamanager/src/datamanager/main.py | 1 - application/positionmanager/src/positionmanager/main.py | 6 ++---- .../positionmanager/tests/test_positionmanager_main.py | 3 --- infrastructure/images.py | 1 + pyproject.toml | 8 +++++++- 8 files changed, 13 insertions(+), 10 deletions(-) diff --git a/application/datamanager/features/environment.py b/application/datamanager/features/environment.py index 5c659cbd1..dd9dc7ae0 100644 --- a/application/datamanager/features/environment.py +++ b/application/datamanager/features/environment.py @@ -1,4 +1,5 @@ import os + from behave.runner import Context from behave.runner import Context diff --git a/application/datamanager/features/steps/equity_bars_steps.py b/application/datamanager/features/steps/equity_bars_steps.py index 5700e7433..3e3b386f2 100644 --- a/application/datamanager/features/steps/equity_bars_steps.py +++ b/application/datamanager/features/steps/equity_bars_steps.py @@ -5,7 +5,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent)) import requests -from behave import given, when, then +from behave import given, then, when from behave.runner import Context diff --git a/application/datamanager/features/steps/health_steps.py b/application/datamanager/features/steps/health_steps.py index d1017ed96..c9129abf3 100644 --- a/application/datamanager/features/steps/health_steps.py +++ b/application/datamanager/features/steps/health_steps.py @@ -1,3 +1,4 @@ +import requests from behave import when from behave.runner import Context diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index 23aefa2d1..9de041380 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -9,7 +9,6 @@ import polars as pl import pyarrow as pa import pyarrow.lib -from duckdb import IOException import requests from duckdb import IOException from fastapi import FastAPI, HTTPException, Request, Response, status diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index 69c8e48e7..2c8e57a2c 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -1,8 +1,6 @@ -from fastapi import FastAPI, HTTPException -import requests import os -from datetime import UTC, datetime, timedelta -from typing import Any +from datetime import datetime, timedelta, timezone +from typing import Any, Dict import polars as pl import requests diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index db3748b9b..b7634bc9e 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -1,6 +1,3 @@ -from fastapi.testclient import TestClient -from fastapi import HTTPException - import unittest from decimal import Decimal from unittest.mock import MagicMock, patch diff --git a/infrastructure/images.py b/infrastructure/images.py index b305f4d6a..9a3e8b0e2 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -6,6 +6,7 @@ from loguru import logger from pulumi import Config from loguru import logger +from pulumi import Config config = Config() dockerhub_username = config.require_secret("dockerhub_username") diff --git a/pyproject.toml b/pyproject.toml index 5207fcc01..e1c451bcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -148,6 +148,7 @@ select = [ "B", # bugbear "COM", # commas "C4", # comprehensions + "C90", # complexity "BLE", # no blind exceptions "DTZ", # datetimes "EM", # error messages @@ -156,16 +157,21 @@ select = [ "FA", # future annotations "FAST", # fastapi "FIX", # no fixme/todo comments + "FLY", # f strings "FBT", # boolean traps "G", # logging format "ICN", # import conventions "ISC", # implicit string concatenation + "I", # isort "LOG", # logging - "Q", # quotes + "N", # naming + "NPY", # numpy + "PD", # pandas "PIE", # misc lints "PT", # pytest style "PTH", # use pathlib "PYI", # type hints + "Q", # quotes "RSE", # raises "RET", # returns "S", # bandit (security) From 6395b008ea011b690c7377ad45b25e79fc33de0f Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 23:44:29 -0400 Subject: [PATCH 30/41] leftover linting --- .../positionmanager/src/positionmanager/main.py | 4 ++-- .../positionmanager/tests/test_positionmanager_main.py | 6 +++--- infrastructure/images.py | 2 +- pyproject.toml | 10 +++++++++- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index 2c8e57a2c..1a7670c08 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -1,6 +1,6 @@ import os -from datetime import datetime, timedelta, timezone -from typing import Any, Dict +from datetime import UTC, datetime, timedelta +from typing import Any import polars as pl import requests diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index b7634bc9e..a5ce64c9b 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -85,7 +85,7 @@ def test_create_position_success( def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None: # noqa: N803 mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.get_cash_balance.side_effect = HTTPException( - status_code=500, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance @@ -128,14 +128,14 @@ def test_delete_positions_error( ) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.clear_positions.side_effect = HTTPException( - status_code=500, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance response = client.delete("/positions") - assert response.status_code == 500 + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR assert "Error" in response.json()["detail"] MockAlpacaClient.assert_called_once() diff --git a/infrastructure/images.py b/infrastructure/images.py index 9a3e8b0e2..203265c9a 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -19,7 +19,7 @@ tags = [ "latest", - datetime.now(tz=timezone.utc).strftime("%Y%m%d"), + datetime.now(tz=UTC).strftime("%Y%m%d"), ] images = {} diff --git a/pyproject.toml b/pyproject.toml index e1c451bcc..f59048381 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,37 +151,45 @@ select = [ "C90", # complexity "BLE", # no blind exceptions "DTZ", # datetimes + "E", # whitespace "EM", # error messages "ERA", # dead code "EXE", # executables + "F", # pyflakes "FA", # future annotations "FAST", # fastapi "FIX", # no fixme/todo comments "FLY", # f strings "FBT", # boolean traps + "FURB", # refurb "G", # logging format "ICN", # import conventions "ISC", # implicit string concatenation - "I", # isort + "I", # isort "LOG", # logging "N", # naming "NPY", # numpy "PD", # pandas + "PERF", # performance "PIE", # misc lints + "PL", # pylint "PT", # pytest style "PTH", # use pathlib "PYI", # type hints "Q", # quotes "RSE", # raises "RET", # returns + "RUF", # ruff "S", # bandit (security) "SIM", # simplicity "SLF", # self "SLOT", # slots "TC", # type checking "TID", # tidy imports + "TRY", # trys "T10", # debugger "T20", # printing + "UP", # pyupgrade "YTT" # flake8 ] ignore = [ From dcb94c44fb8a978881fa044273de6aa1e33ab638 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 15:29:47 -0400 Subject: [PATCH 31/41] add type annotations where missing --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f59048381..0ed25c9e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,6 +200,11 @@ ignore = [ "**/tests/**/*.py" = ["S101"] "**/features/steps/**/*.py" = ["S101"] +[tool.ruff] +select = [ + "ANN" +] + [tool.ty.rules] unresolved-import = "ignore" invalid-return-type = "error" From 18a1900f67930ecb3b062d56589bcce119face06 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 15:45:57 -0400 Subject: [PATCH 32/41] ruff fixes From 74507d3aa63a2f2e99a07c34a4f1ef5b2137ed9a Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 15:33:27 -0400 Subject: [PATCH 33/41] add fastapi linting fix bandit issues fixing ruff issues --- pyproject.toml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0ed25c9e9..1d505dbf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,10 +200,18 @@ ignore = [ "**/tests/**/*.py" = ["S101"] "**/features/steps/**/*.py" = ["S101"] -[tool.ruff] +[tool.ruff.lint] select = [ - "ANN" + "ANN", # type annotations + "ASYNC", + "ERA", # dead code + "FAST", # fastapi + "S", # bandit (security) + "YTT" # flake8 ] +[tool.ruff.lint.per-file-ignores] +"**/tests/**/*.py" = ["S101"] +"**/features/steps/**/*.py" = ["S101"] [tool.ty.rules] unresolved-import = "ignore" From 19bac73be147dd231665230fd8c5945929000922 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 16:41:39 -0400 Subject: [PATCH 34/41] no blind exceptions update to fix blind exceptions --- application/datamanager/src/datamanager/main.py | 4 ++-- .../positionmanager/src/positionmanager/main.py | 9 ++------- .../tests/test_positionmanager_main.py | 11 ++++++----- pyproject.toml | 1 + 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index 9de041380..14bb0dab7 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -121,7 +121,7 @@ async def get_equity_bars( ) except ( - requests.RequestException, + requests.RequestsException, ComputeError, IOException, GoogleAPIError, @@ -166,7 +166,7 @@ async def fetch_equity_bars(request: Request, summary_date: SummaryDate) -> Bars partition_by=["year", "month", "day"], ) except ( - requests.RequestException, + requests.RequestsException, ComputeError, IOException, GoogleAPIError, diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index 1a7670c08..f9dc7e44e 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -1,17 +1,12 @@ +from fastapi import FastAPI, HTTPException +import requests import os from datetime import UTC, datetime, timedelta from typing import Any -import polars as pl -import requests from alpaca.common.rest import APIError -from fastapi import FastAPI, HTTPException -from prometheus_fastapi_instrumentator import Instrumentator from pydantic import ValidationError -from .clients import AlpacaClient, DataClient -from .models import DateRange, Money, PredictionPayload -from .portfolio import PortfolioOptimizer trading_days_per_year = 252 diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index a5ce64c9b..2d54241e0 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -1,3 +1,6 @@ +from fastapi.testclient import TestClient +from fastapi import HTTPException + import unittest from decimal import Decimal from unittest.mock import MagicMock, patch @@ -85,8 +88,7 @@ def test_create_position_success( def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None: # noqa: N803 mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.get_cash_balance.side_effect = HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Error getting cash balance", + status_code=500, detail="Error getting cash balance" ) MockAlpacaClient.return_value = mock_alpaca_instance @@ -128,14 +130,13 @@ def test_delete_positions_error( ) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.clear_positions.side_effect = HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Error getting cash balance", + status_code=500, detail="Error getting cash balance" ) MockAlpacaClient.return_value = mock_alpaca_instance response = client.delete("/positions") - assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + assert response.status_code == 500 assert "Error" in response.json()["detail"] MockAlpacaClient.assert_called_once() diff --git a/pyproject.toml b/pyproject.toml index 1d505dbf8..d4d08f794 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -204,6 +204,7 @@ ignore = [ select = [ "ANN", # type annotations "ASYNC", + "BLE", # no blind exceptions "ERA", # dead code "FAST", # fastapi "S", # bandit (security) From 9b4f7621798773521ab6fe9ae51e13f035b66b88 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 17:11:44 -0400 Subject: [PATCH 35/41] fix boolean traps --- application/datamanager/src/datamanager/main.py | 4 ++-- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index 14bb0dab7..9de041380 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -121,7 +121,7 @@ async def get_equity_bars( ) except ( - requests.RequestsException, + requests.RequestException, ComputeError, IOException, GoogleAPIError, @@ -166,7 +166,7 @@ async def fetch_equity_bars(request: Request, summary_date: SummaryDate) -> Bars partition_by=["year", "month", "day"], ) except ( - requests.RequestsException, + requests.RequestException, ComputeError, IOException, GoogleAPIError, diff --git a/pyproject.toml b/pyproject.toml index d4d08f794..7ffa87a33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -207,6 +207,7 @@ select = [ "BLE", # no blind exceptions "ERA", # dead code "FAST", # fastapi + "FBT", # boolean traps "S", # bandit (security) "YTT" # flake8 ] From bda76418ef9b858cb10f68a915733c8fee3b0bc8 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 17:12:24 -0400 Subject: [PATCH 36/41] fix boolean traps add bugbear add comma linting fix bugbear and commas add timezones --- application/datamanager/src/datamanager/models.py | 2 +- application/positionmanager/src/positionmanager/main.py | 4 ++-- .../positionmanager/tests/test_positionmanager_main.py | 6 ++++-- infrastructure/images.py | 2 +- pyproject.toml | 9 +++++++++ 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/application/datamanager/src/datamanager/models.py b/application/datamanager/src/datamanager/models.py index 232eb936a..675cf36cf 100644 --- a/application/datamanager/src/datamanager/models.py +++ b/application/datamanager/src/datamanager/models.py @@ -17,7 +17,7 @@ def parse_date(cls, value: datetime.date | str) -> datetime.date: # noqa: N805 try: return ( datetime.datetime.strptime(value, fmt) - .replace(tzinfo=datetime.UTC) + .replace(tzinfo=datetime.timezone.utc) .date() ) except ValueError: diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index f9dc7e44e..5b149e5e4 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -44,8 +44,8 @@ def create_position(payload: PredictionPayload) -> dict[str, Any]: ) from e date_range = DateRange( - start=datetime.now(tz=UTC) - timedelta(days=trading_days_per_year), - end=datetime.now(tz=UTC), + start=datetime.now(tz=timezone.utc) - timedelta(days=trading_days_per_year), + end=datetime.now(tz=timezone.utc), ) try: diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index 2d54241e0..db3748b9b 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -88,7 +88,8 @@ def test_create_position_success( def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None: # noqa: N803 mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.get_cash_balance.side_effect = HTTPException( - status_code=500, detail="Error getting cash balance" + status_code=500, + detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance @@ -130,7 +131,8 @@ def test_delete_positions_error( ) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.clear_positions.side_effect = HTTPException( - status_code=500, detail="Error getting cash balance" + status_code=500, + detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance diff --git a/infrastructure/images.py b/infrastructure/images.py index 203265c9a..9a3e8b0e2 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -19,7 +19,7 @@ tags = [ "latest", - datetime.now(tz=UTC).strftime("%Y%m%d"), + datetime.now(tz=timezone.utc).strftime("%Y%m%d"), ] images = {} diff --git a/pyproject.toml b/pyproject.toml index 7ffa87a33..649f00b75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -202,15 +202,24 @@ ignore = [ [tool.ruff.lint] select = [ + "A", # flake8 builtins "ANN", # type annotations "ASYNC", + "B", # bugbear + "COM", # commas + "C4", # comprehensions "BLE", # no blind exceptions + "DTZ", # datetimes "ERA", # dead code "FAST", # fastapi "FBT", # boolean traps "S", # bandit (security) "YTT" # flake8 ] +ignore = [ + "COM812", +] + [tool.ruff.lint.per-file-ignores] "**/tests/**/*.py" = ["S101"] "**/features/steps/**/*.py" = ["S101"] From 31e33f9843d318061ad9afccd67574f18f81b9b2 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 23:19:57 -0400 Subject: [PATCH 37/41] lint error messages no fixme/todos implicit string concatenation import conventions logging pathlib --- .../datamanager/src/datamanager/config.py | 2 + .../datamanager/src/datamanager/main.py | 1 + .../src/positionmanager/clients.py | 6 ++- infrastructure/images.py | 5 +- pyproject.toml | 47 ++++++++++++++----- 5 files changed, 46 insertions(+), 15 deletions(-) diff --git a/application/datamanager/src/datamanager/config.py b/application/datamanager/src/datamanager/config.py index 089c640bc..552e6d295 100644 --- a/application/datamanager/src/datamanager/config.py +++ b/application/datamanager/src/datamanager/config.py @@ -1,3 +1,5 @@ +import os +from pathlib import Path import json import os from functools import cached_property diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index 9de041380..23aefa2d1 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -9,6 +9,7 @@ import polars as pl import pyarrow as pa import pyarrow.lib +from duckdb import IOException import requests from duckdb import IOException from fastapi import FastAPI, HTTPException, Request, Response, status diff --git a/application/positionmanager/src/positionmanager/clients.py b/application/positionmanager/src/positionmanager/clients.py index 8850d8390..2921e8a87 100644 --- a/application/positionmanager/src/positionmanager/clients.py +++ b/application/positionmanager/src/positionmanager/clients.py @@ -83,7 +83,11 @@ def get_data( msg = f"Data manager service call error: {err}" raise RuntimeError(msg) from err - response.raise_for_status() + if response.status_code != 200: + msg = ( + f"Data service error: {response.text}, status code: {response.status_code}", + ) + raise Exception(msg) response_data = response.json() diff --git a/infrastructure/images.py b/infrastructure/images.py index 9a3e8b0e2..193ecf1a3 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -1,6 +1,7 @@ -from datetime import UTC, datetime +import os from pathlib import Path - +from datetime import datetime, timezone +from glob import glob import pulumi import pulumi_docker_build as docker_build from loguru import logger diff --git a/pyproject.toml b/pyproject.toml index 649f00b75..d2c78b7c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -202,19 +202,42 @@ ignore = [ [tool.ruff.lint] select = [ - "A", # flake8 builtins - "ANN", # type annotations + "A", # flake8 builtins + "ANN", # type annotations + "ARG", # unused args "ASYNC", - "B", # bugbear - "COM", # commas - "C4", # comprehensions - "BLE", # no blind exceptions - "DTZ", # datetimes - "ERA", # dead code - "FAST", # fastapi - "FBT", # boolean traps - "S", # bandit (security) - "YTT" # flake8 + "B", # bugbear + "COM", # commas + "C4", # comprehensions + "BLE", # no blind exceptions + "DTZ", # datetimes + "EM", # error messages + "ERA", # dead code + "EXE", # executables + "FA", # future annotations + "FAST", # fastapi + "FIX", # no fixme/todo comments + "FBT", # boolean traps + "G", # logging format + "ICN", # import conventions + "ISC", # implicit string concatenation + "LOG", # logging + "Q", # quotes + "PIE", # misc lints + "PT", # pytest style + "PTH", # use pathlib + "PYI", # type hints + "RSE", # raises + "RET", # returns + "S", # bandit (security) + "SIM", # simplicity + "SLF", # self + "SLOT", # slots + "TC", # type checking + "TID", # tidy imports + "T10", # debugger + "T20", # printing + "YTT" # flake8 ] ignore = [ "COM812", From a48edb93a1b7944253c4b4f5933c28104b99b1f2 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 23:20:26 -0400 Subject: [PATCH 38/41] a bunch of linting sort imports fix naming conventions --- .../datamanager/src/datamanager/config.py | 2 -- .../datamanager/src/datamanager/main.py | 1 - .../src/positionmanager/main.py | 18 ++++++++++++++---- .../tests/test_positionmanager_main.py | 3 --- infrastructure/images.py | 3 ++- pyproject.toml | 8 +++++++- 6 files changed, 23 insertions(+), 12 deletions(-) diff --git a/application/datamanager/src/datamanager/config.py b/application/datamanager/src/datamanager/config.py index 552e6d295..089c640bc 100644 --- a/application/datamanager/src/datamanager/config.py +++ b/application/datamanager/src/datamanager/config.py @@ -1,5 +1,3 @@ -import os -from pathlib import Path import json import os from functools import cached_property diff --git a/application/datamanager/src/datamanager/main.py b/application/datamanager/src/datamanager/main.py index 23aefa2d1..9de041380 100644 --- a/application/datamanager/src/datamanager/main.py +++ b/application/datamanager/src/datamanager/main.py @@ -9,7 +9,6 @@ import polars as pl import pyarrow as pa import pyarrow.lib -from duckdb import IOException import requests from duckdb import IOException from fastapi import FastAPI, HTTPException, Request, Response, status diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index 5b149e5e4..ae54f246d 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -1,12 +1,22 @@ -from fastapi import FastAPI, HTTPException -import requests import os -from datetime import UTC, datetime, timedelta -from typing import Any +from datetime import datetime, timedelta, timezone +from typing import Any, Dict + +import polars as pl +from typing import Dict, Any +from .models import Money, DateRange, PredictionPayload +from .clients import AlpacaClient, DataClient +from .portfolio import PortfolioOptimizer +from prometheus_fastapi_instrumentator import Instrumentator from alpaca.common.rest import APIError +from fastapi import FastAPI, HTTPException +from prometheus_fastapi_instrumentator import Instrumentator from pydantic import ValidationError +from .clients import AlpacaClient, DataClient +from .models import DateRange, Money, PredictionPayload +from .portfolio import PortfolioOptimizer trading_days_per_year = 252 diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index db3748b9b..b7634bc9e 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -1,6 +1,3 @@ -from fastapi.testclient import TestClient -from fastapi import HTTPException - import unittest from decimal import Decimal from unittest.mock import MagicMock, patch diff --git a/infrastructure/images.py b/infrastructure/images.py index 193ecf1a3..22cfebb41 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -1,7 +1,8 @@ import os -from pathlib import Path from datetime import datetime, timezone from glob import glob +from pathlib import Path + import pulumi import pulumi_docker_build as docker_build from loguru import logger diff --git a/pyproject.toml b/pyproject.toml index d2c78b7c7..d3ca56dd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -209,6 +209,7 @@ select = [ "B", # bugbear "COM", # commas "C4", # comprehensions + "C90", # complexity "BLE", # no blind exceptions "DTZ", # datetimes "EM", # error messages @@ -217,16 +218,21 @@ select = [ "FA", # future annotations "FAST", # fastapi "FIX", # no fixme/todo comments + "FLY", # f strings "FBT", # boolean traps "G", # logging format "ICN", # import conventions "ISC", # implicit string concatenation + "I", # isort "LOG", # logging - "Q", # quotes + "N", # naming + "NPY", # numpy + "PD", # pandas "PIE", # misc lints "PT", # pytest style "PTH", # use pathlib "PYI", # type hints + "Q", # quotes "RSE", # raises "RET", # returns "S", # bandit (security) From c5b1be271a9961f049f20015500b9cc457b1ee57 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 28 May 2025 23:44:29 -0400 Subject: [PATCH 39/41] leftover linting --- application/datamanager/src/datamanager/models.py | 2 +- .../positionmanager/src/positionmanager/clients.py | 6 +----- .../positionmanager/src/positionmanager/main.py | 8 ++++---- .../positionmanager/tests/test_positionmanager_main.py | 6 +++--- infrastructure/images.py | 6 ++---- pyproject.toml | 10 +++++++++- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/application/datamanager/src/datamanager/models.py b/application/datamanager/src/datamanager/models.py index 675cf36cf..232eb936a 100644 --- a/application/datamanager/src/datamanager/models.py +++ b/application/datamanager/src/datamanager/models.py @@ -17,7 +17,7 @@ def parse_date(cls, value: datetime.date | str) -> datetime.date: # noqa: N805 try: return ( datetime.datetime.strptime(value, fmt) - .replace(tzinfo=datetime.timezone.utc) + .replace(tzinfo=datetime.UTC) .date() ) except ValueError: diff --git a/application/positionmanager/src/positionmanager/clients.py b/application/positionmanager/src/positionmanager/clients.py index 2921e8a87..8850d8390 100644 --- a/application/positionmanager/src/positionmanager/clients.py +++ b/application/positionmanager/src/positionmanager/clients.py @@ -83,11 +83,7 @@ def get_data( msg = f"Data manager service call error: {err}" raise RuntimeError(msg) from err - if response.status_code != 200: - msg = ( - f"Data service error: {response.text}, status code: {response.status_code}", - ) - raise Exception(msg) + response.raise_for_status() response_data = response.json() diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index ae54f246d..c13459bdb 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -1,6 +1,6 @@ import os -from datetime import datetime, timedelta, timezone -from typing import Any, Dict +from datetime import UTC, datetime, timedelta +from typing import Any import polars as pl from typing import Dict, Any @@ -54,8 +54,8 @@ def create_position(payload: PredictionPayload) -> dict[str, Any]: ) from e date_range = DateRange( - start=datetime.now(tz=timezone.utc) - timedelta(days=trading_days_per_year), - end=datetime.now(tz=timezone.utc), + start=datetime.now(tz=UTC) - timedelta(days=trading_days_per_year), + end=datetime.now(tz=UTC), ) try: diff --git a/application/positionmanager/tests/test_positionmanager_main.py b/application/positionmanager/tests/test_positionmanager_main.py index b7634bc9e..a5ce64c9b 100644 --- a/application/positionmanager/tests/test_positionmanager_main.py +++ b/application/positionmanager/tests/test_positionmanager_main.py @@ -85,7 +85,7 @@ def test_create_position_success( def test_create_position_alpaca_error(self, MockAlpacaClient: MagicMock) -> None: # noqa: N803 mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.get_cash_balance.side_effect = HTTPException( - status_code=500, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance @@ -128,14 +128,14 @@ def test_delete_positions_error( ) -> None: mock_alpaca_instance = MagicMock(spec=AlpacaClient) mock_alpaca_instance.clear_positions.side_effect = HTTPException( - status_code=500, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error getting cash balance", ) MockAlpacaClient.return_value = mock_alpaca_instance response = client.delete("/positions") - assert response.status_code == 500 + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR assert "Error" in response.json()["detail"] MockAlpacaClient.assert_called_once() diff --git a/infrastructure/images.py b/infrastructure/images.py index 22cfebb41..203265c9a 100644 --- a/infrastructure/images.py +++ b/infrastructure/images.py @@ -1,6 +1,4 @@ -import os -from datetime import datetime, timezone -from glob import glob +from datetime import UTC, datetime from pathlib import Path import pulumi @@ -21,7 +19,7 @@ tags = [ "latest", - datetime.now(tz=timezone.utc).strftime("%Y%m%d"), + datetime.now(tz=UTC).strftime("%Y%m%d"), ] images = {} diff --git a/pyproject.toml b/pyproject.toml index d3ca56dd4..390f15999 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -212,37 +212,45 @@ select = [ "C90", # complexity "BLE", # no blind exceptions "DTZ", # datetimes + "E", # whitespace "EM", # error messages "ERA", # dead code "EXE", # executables + "F", # pyflakes "FA", # future annotations "FAST", # fastapi "FIX", # no fixme/todo comments "FLY", # f strings "FBT", # boolean traps + "FURB", # refurb "G", # logging format "ICN", # import conventions "ISC", # implicit string concatenation - "I", # isort + "I", # isort "LOG", # logging "N", # naming "NPY", # numpy "PD", # pandas + "PERF", # performance "PIE", # misc lints + "PL", # pylint "PT", # pytest style "PTH", # use pathlib "PYI", # type hints "Q", # quotes "RSE", # raises "RET", # returns + "RUF", # ruff "S", # bandit (security) "SIM", # simplicity "SLF", # self "SLOT", # slots "TC", # type checking "TID", # tidy imports + "TRY", # trys "T10", # debugger "T20", # printing + "UP", # pyupgrade "YTT" # flake8 ] ignore = [ From 117ebf14c86b759f69e3aadd7a68bbe779f081ad Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 4 Jun 2025 16:42:17 -0400 Subject: [PATCH 40/41] fix linting --- application/positionmanager/src/positionmanager/main.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index c13459bdb..1a7670c08 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -3,12 +3,7 @@ from typing import Any import polars as pl -from typing import Dict, Any -from .models import Money, DateRange, PredictionPayload -from .clients import AlpacaClient, DataClient -from .portfolio import PortfolioOptimizer -from prometheus_fastapi_instrumentator import Instrumentator - +import requests from alpaca.common.rest import APIError from fastapi import FastAPI, HTTPException from prometheus_fastapi_instrumentator import Instrumentator From 3007ebc38dc3e549a782b555f97c1fb3f190c9f9 Mon Sep 17 00:00:00 2001 From: chrisaddy Date: Wed, 4 Jun 2025 20:38:02 -0400 Subject: [PATCH 41/41] rebasing --- .../datamanager/features/environment.py | 1 - .../features/steps/health_steps.py | 4 - .../src/positionmanager/main.py | 9 +- infrastructure/images.py | 53 ----- pyproject.toml | 183 ------------------ 5 files changed, 2 insertions(+), 248 deletions(-) delete mode 100644 infrastructure/images.py diff --git a/application/datamanager/features/environment.py b/application/datamanager/features/environment.py index dd9dc7ae0..796fe865c 100644 --- a/application/datamanager/features/environment.py +++ b/application/datamanager/features/environment.py @@ -2,7 +2,6 @@ from behave.runner import Context -from behave.runner import Context def before_all(context: Context) -> None: context.base_url = os.environ.get("BASE_URL", "http://datamanager:8080") diff --git a/application/datamanager/features/steps/health_steps.py b/application/datamanager/features/steps/health_steps.py index c9129abf3..b6be9a63c 100644 --- a/application/datamanager/features/steps/health_steps.py +++ b/application/datamanager/features/steps/health_steps.py @@ -2,10 +2,6 @@ from behave import when from behave.runner import Context -import requests -from behave import when -from behave.runner import Context - @when('I send a GET request to "{endpoint}"') def step_impl(context: Context, endpoint: str) -> None: diff --git a/application/positionmanager/src/positionmanager/main.py b/application/positionmanager/src/positionmanager/main.py index a64764b4b..1a7670c08 100644 --- a/application/positionmanager/src/positionmanager/main.py +++ b/application/positionmanager/src/positionmanager/main.py @@ -1,11 +1,6 @@ import os -from datetime import datetime, timedelta, timezone -import polars as pl -from typing import Dict, Any -from .models import Money, DateRange, PredictionPayload -from .clients import AlpacaClient, DataClient -from .portfolio import PortfolioOptimizer -from prometheus_fastapi_instrumentator import Instrumentator +from datetime import UTC, datetime, timedelta +from typing import Any import polars as pl import requests diff --git a/infrastructure/images.py b/infrastructure/images.py deleted file mode 100644 index 203265c9a..000000000 --- a/infrastructure/images.py +++ /dev/null @@ -1,53 +0,0 @@ -from datetime import UTC, datetime -from pathlib import Path - -import pulumi -import pulumi_docker_build as docker_build -from loguru import logger -from pulumi import Config -from loguru import logger -from pulumi import Config - -config = Config() -dockerhub_username = config.require_secret("dockerhub_username") -dockerhub_password = config.require_secret("dockerhub_password") - -application_path = Path("../application/").resolve() -dockerfile_paths = [ - app.relative_to(application_path) for app in application_path.glob("*/Dockerfile") -] - -tags = [ - "latest", - datetime.now(tz=UTC).strftime("%Y%m%d"), -] - -images = {} -for dockerfile in dockerfile_paths: - service_dir = dockerfile.parent - service_name = dockerfile.name - logger.info(f"Creating image for service: {service_name}") - - images[service_name] = docker_build.Image( - f"{service_name}-image", - tags=[f"pocketsizefund/{service_name}:{tag}" for tag in tags], - context=docker_build.BuildContextArgs( - location=service_dir, - ), - platforms=[ - docker_build.Platform.LINUX_AMD64, - docker_build.Platform.LINUX_ARM64, - ], - push=True, - registries=[ - docker_build.RegistryArgs( - address="docker.io", - username=dockerhub_username, - password=dockerhub_password, - ), - ], - ) - - pulumi.export(f"{service_name}-ref", images[service_name].ref) - -logger.info(f"Available image services: {list(images.keys())}") diff --git a/pyproject.toml b/pyproject.toml index 746d9faa4..0ee81df90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,189 +133,6 @@ ignore = [ "**/tests/**/*.py" = ["S101"] "**/features/steps/**/*.py" = ["S101"] -[tool.ruff.lint] -select = [ - "A", # flake8 builtins - "ANN", # type annotations - "ARG", # unused args - "ASYNC", - "B", # bugbear - "COM", # commas - "C4", # comprehensions - "C90", # complexity - "BLE", # no blind exceptions - "DTZ", # datetimes - "E", # whitespace - "EM", # error messages - "ERA", # dead code - "EXE", # executables - "F", # pyflakes - "FA", # future annotations - "FAST", # fastapi - "FIX", # no fixme/todo comments - "FLY", # f strings - "FBT", # boolean traps - "FURB", # refurb - "G", # logging format - "ICN", # import conventions - "ISC", # implicit string concatenation - "I", # isort - "LOG", # logging - "N", # naming - "NPY", # numpy - "PD", # pandas - "PERF", # performance - "PIE", # misc lints - "PL", # pylint - "PT", # pytest style - "PTH", # use pathlib - "PYI", # type hints - "Q", # quotes - "RSE", # raises - "RET", # returns - "RUF", # ruff - "S", # bandit (security) - "SIM", # simplicity - "SLF", # self - "SLOT", # slots - "TC", # type checking - "TID", # tidy imports - "TRY", # trys - "T10", # debugger - "T20", # printing - "UP", # pyupgrade - "YTT" # flake8 -] -ignore = [ - "COM812", -] - -[tool.ruff.lint.per-file-ignores] -"**/tests/**/*.py" = ["S101"] -"**/features/steps/**/*.py" = ["S101"] - -[tool.ruff.lint] -select = [ - "A", # flake8 builtins - "ANN", # type annotations - "ARG", # unused args - "ASYNC", - "B", # bugbear - "COM", # commas - "C4", # comprehensions - "C90", # complexity - "BLE", # no blind exceptions - "DTZ", # datetimes - "E", # whitespace - "EM", # error messages - "ERA", # dead code - "EXE", # executables - "F", # pyflakes - "FA", # future annotations - "FAST", # fastapi - "FIX", # no fixme/todo comments - "FLY", # f strings - "FBT", # boolean traps - "FURB", # refurb - "G", # logging format - "ICN", # import conventions - "ISC", # implicit string concatenation - "I", # isort - "LOG", # logging - "N", # naming - "NPY", # numpy - "PD", # pandas - "PERF", # performance - "PIE", # misc lints - "PL", # pylint - "PT", # pytest style - "PTH", # use pathlib - "PYI", # type hints - "Q", # quotes - "RSE", # raises - "RET", # returns - "RUF", # ruff - "S", # bandit (security) - "SIM", # simplicity - "SLF", # self - "SLOT", # slots - "TC", # type checking - "TID", # tidy imports - "TRY", # trys - "T10", # debugger - "T20", # printing - "UP", # pyupgrade - "YTT" # flake8 -] -ignore = [ - "COM812", -] - -[tool.ruff.lint.per-file-ignores] -"**/tests/**/*.py" = ["S101"] -"**/features/steps/**/*.py" = ["S101"] - -[tool.ruff.lint] -select = [ - "A", # flake8 builtins - "ANN", # type annotations - "ARG", # unused args - "ASYNC", - "B", # bugbear - "COM", # commas - "C4", # comprehensions - "C90", # complexity - "BLE", # no blind exceptions - "DTZ", # datetimes - "E", # whitespace - "EM", # error messages - "ERA", # dead code - "EXE", # executables - "F", # pyflakes - "FA", # future annotations - "FAST", # fastapi - "FIX", # no fixme/todo comments - "FLY", # f strings - "FBT", # boolean traps - "FURB", # refurb - "G", # logging format - "ICN", # import conventions - "ISC", # implicit string concatenation - "I", # isort - "LOG", # logging - "N", # naming - "NPY", # numpy - "PD", # pandas - "PERF", # performance - "PIE", # misc lints - "PL", # pylint - "PT", # pytest style - "PTH", # use pathlib - "PYI", # type hints - "Q", # quotes - "RSE", # raises - "RET", # returns - "RUF", # ruff - "S", # bandit (security) - "SIM", # simplicity - "SLF", # self - "SLOT", # slots - "TC", # type checking - "TID", # tidy imports - "TRY", # trys - "T10", # debugger - "T20", # printing - "UP", # pyupgrade - "YTT" # flake8 -] -ignore = [ - "COM812", -] - -[tool.ruff.lint.per-file-ignores] -"**/tests/**/*.py" = ["S101"] -"**/features/steps/**/*.py" = ["S101"] - [tool.ty.rules] unresolved-import = "ignore" invalid-return-type = "error"