Data fetcher

.flox/env/manifest.lock

@@ -12,6 +12,9 @@
       "pulumi-python": {
         "pkg-path": "pulumiPackages.pulumi-python"
       },
+      "pulumictl": {
+        "pkg-path": "pulumictl"
+      },
       "ruff": {
         "pkg-path": "ruff"
       },
@@ -395,6 +398,126 @@
       "group": "toplevel",
       "priority": 5
     },
+    {
+      "attr_path": "pulumictl",
+      "broken": false,
+      "derivation": "/nix/store/kx43jzcfslw28byvs6h5ngsgl432pvvv-pulumictl-0.0.49.drv",
+      "description": "Swiss Army Knife for Pulumi Development",
+      "install_id": "pulumictl",
+      "license": "Apache-2.0",
+      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "name": "pulumictl-0.0.49",
+      "pname": "pulumictl",
+      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "rev_count": 793735,
+      "rev_date": "2025-05-04T03:14:55Z",
+      "scrape_date": "2025-05-05T04:19:37.687142Z",
+      "stabilities": [
+        "staging",
+        "unstable"
+      ],
+      "unfree": false,
+      "version": "0.0.49",
+      "outputs_to_install": [
+        "out"
+      ],
+      "outputs": {
+        "out": "/nix/store/ny69c9bfkf4w179240ch45injfb2ajqr-pulumictl-0.0.49"
+      },
+      "system": "aarch64-darwin",
+      "group": "toplevel",
+      "priority": 5
+    },
+    {
+      "attr_path": "pulumictl",
+      "broken": false,
+      "derivation": "/nix/store/www9nfncvv7l339n8dks22x5vs5lz1mk-pulumictl-0.0.49.drv",
+      "description": "Swiss Army Knife for Pulumi Development",
+      "install_id": "pulumictl",
+      "license": "Apache-2.0",
+      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "name": "pulumictl-0.0.49",
+      "pname": "pulumictl",
+      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "rev_count": 793735,
+      "rev_date": "2025-05-04T03:14:55Z",
+      "scrape_date": "2025-05-05T04:37:42.118866Z",
+      "stabilities": [
+        "staging",
+        "unstable"
+      ],
+      "unfree": false,
+      "version": "0.0.49",
+      "outputs_to_install": [
+        "out"
+      ],
+      "outputs": {
+        "out": "/nix/store/xpdh5dijdki4cngh7k7n4rg84i6c28zs-pulumictl-0.0.49"
+      },
+      "system": "aarch64-linux",
+      "group": "toplevel",
+      "priority": 5
+    },
+    {
+      "attr_path": "pulumictl",
+      "broken": false,
+      "derivation": "/nix/store/17wf5x1kk3v5ch5npwhamnix629y07wg-pulumictl-0.0.49.drv",
+      "description": "Swiss Army Knife for Pulumi Development",
+      "install_id": "pulumictl",
+      "license": "Apache-2.0",
+      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "name": "pulumictl-0.0.49",
+      "pname": "pulumictl",
+      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "rev_count": 793735,
+      "rev_date": "2025-05-04T03:14:55Z",
+      "scrape_date": "2025-05-05T04:54:38.447587Z",
+      "stabilities": [
+        "staging",
+        "unstable"
+      ],
+      "unfree": false,
+      "version": "0.0.49",
+      "outputs_to_install": [
+        "out"
+      ],
+      "outputs": {
+        "out": "/nix/store/6wmig1w7f3vmfrlyg2qzv21bvacj3as8-pulumictl-0.0.49"
+      },
+      "system": "x86_64-darwin",
+      "group": "toplevel",
+      "priority": 5
+    },
+    {
+      "attr_path": "pulumictl",
+      "broken": false,
+      "derivation": "/nix/store/ib7hqxg7xdf5kyh78jqggzdcs97q1224-pulumictl-0.0.49.drv",
+      "description": "Swiss Army Knife for Pulumi Development",
+      "install_id": "pulumictl",
+      "license": "Apache-2.0",
+      "locked_url": "https://github.com/flox/nixpkgs?rev=979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "name": "pulumictl-0.0.49",
+      "pname": "pulumictl",
+      "rev": "979daf34c8cacebcd917d540070b52a3c2b9b16e",
+      "rev_count": 793735,
+      "rev_date": "2025-05-04T03:14:55Z",
+      "scrape_date": "2025-05-05T05:16:19.858098Z",
+      "stabilities": [
+        "staging",
+        "unstable"
+      ],
+      "unfree": false,
+      "version": "0.0.49",
+      "outputs_to_install": [
+        "out"
+      ],
+      "outputs": {
+        "out": "/nix/store/rmh9mjkxijxcc7cvjhsqc9657fbw0yyg-pulumictl-0.0.49"
+      },
+      "system": "x86_64-linux",
+      "group": "toplevel",
+      "priority": 5
+    },
     {
       "attr_path": "ruff",
       "broken": false,

.gitignore

@@ -9,4 +9,4 @@ infrastructure/Pulumi.infrastructure.yaml
 .envrc
 .coverage*
 .coverage/
-coverage.xml
+coverage.xml

.mise.toml

@@ -41,7 +41,7 @@ run = """
 docker run \
 --env-file .env \
 --publish 8080:8080 \
-pocketsizefund/{{arg(name="service_name")}}:latest \
+pocketsizefund/{{arg(name="application_name")}}:latest \
 """
 
 [tasks."application:service:run:development"]

application/datamanager/features/environment.py

@@ -2,4 +2,5 @@
 
 
 def before_all(context):
+    """Set up test environment."""
     context.base_url = os.environ.get("BASE_URL", "http://datamanager:8080")

application/datamanager/mise.toml

@@ -0,0 +1,11 @@
+[tasks."test:docker:behave"]
+description = "Run behave tests with Docker Compose"
+run = """
+docker-compose up --build --abort-on-container-exit
+"""
+
+[tasks."test:docker:behave:cleanup"]
+description = "Clean up after Docker Compose tests"
+run = """
+docker-compose down -v
+"""

application/datamanager/pyproject.toml

@@ -9,6 +9,7 @@ dependencies = [
     "duckdb>=1.2.2",
     "polars>=1.29.0",
     "pyarrow>=20.0.0",
+    "loguru>=0.7.3",
     "google-cloud-storage>=2.16.0",
     "httpx>=0.28.1",
     "datamanager"

application/datamanager/src/datamanager/main.py

@@ -167,5 +167,4 @@ async def delete_equity_bars(request: Request, summary_date: SummaryDate):
 
     logger.info(f"deleting {prefix=}")
     bucket.delete_blobs(blobs)
-
     return Response(status_code=status.HTTP_204_NO_CONTENT)

infrastructure/cloud_run.py

@@ -46,6 +46,17 @@
                             name="DUCKDB_SECRET",
                             value=duckdb_secret,
                         ),
+                        cloudrun.ServiceTemplateSpecContainerEnvArgs(
+                            name="DATA_BUCKET",
+                            value=buckets.production_data_bucket.name,
+                        ),
+                        cloudrun.ServiceTemplateSpecContainerEnvArgs(
+                            name="DUCKDB_ACCESS_KEY", value=duckdb_access_key
+                        ),
+                        cloudrun.ServiceTemplateSpecContainerEnvArgs(
+                            name="DUCKDB_SECRET",
+                            value=duckdb_secret,
+                            ),
                     ],
                 )
             ],