Skip to content

Create initial datamanager helper classes #618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
forstmeier merged 8 commits into from
Aug 23, 2025
Merged

Create initial datamanager helper classes #618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
fc5e48f
Create initial datamanager helper classes
forstmeier Aug 21, 2025
4b9f516
docker swarm
chrisaddy Aug 14, 2025
b376faa
Merge pull request #613 from pocketsizefund/08-14-docker_swarm
chrisaddy Aug 23, 2025
56df7be
Create initial datamanager helper classes
forstmeier Aug 21, 2025
5f6c071
Merge branch '08-20-create_initial_datamanager_helper_classes' of git…
forstmeier Aug 23, 2025
d1e45c9
Add lint ignore checks to infrastructure resources
forstmeier Aug 23, 2025
059dc83
Add lint ignore checks to infrastructure resources
forstmeier Aug 23, 2025
089ab53
Merge branch '08-20-create_initial_datamanager_helper_classes' of git…
forstmeier Aug 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .claude/settings.local.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@
"Bash(cat:*)",
"Bash(mise run:*)",
"Bash(gh issue close:*)",
"Bash(ln:*)"
"Bash(ln:*)",
"WebSearch",
"Bash(docker build:*)",
"Bash(uv lock:*)"
Comment thread
forstmeier marked this conversation as resolved.
],
"deny": [],
"defaultMode": "acceptEdits"
Expand Down
245 changes: 242 additions & 3 deletions .flox/env/manifest.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@
"manifest": {
"version": 1,
"install": {
"awscli": {
"pkg-path": "awscli"
"awscli2": {
"pkg-path": "awscli2"
},
"cargo": {
"pkg-path": "cargo"
},
"fselect": {
"pkg-path": "fselect"
Expand Down Expand Up @@ -51,7 +54,243 @@
},
"packages": [
{
"attr_path": "awscli",
"attr_path": "awscli2",
"broken": false,
"derivation": "/nix/store/v5gxglicwhyknfwswhngrbawzdwnapp5-awscli2-2.27.2.drv",
"description": "Unified tool to manage your AWS services",
"install_id": "awscli2",
"license": "Apache-2.0",
"locked_url": "https://github.com/flox/nixpkgs?rev=96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"name": "awscli2-2.27.2",
"pname": "awscli2",
"rev": "96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"rev_count": 807377,
"rev_date": "2025-05-28T19:24:49Z",
"scrape_date": "2025-05-31T04:22:53.969182Z",
"stabilities": [
"unstable"
],
"unfree": false,
"version": "2.27.2",
"outputs_to_install": [
"out"
],
"outputs": {
"dist": "/nix/store/sc270lyggbhz78lcxh9khqmggayqpp6i-awscli2-2.27.2-dist",
"out": "/nix/store/b5fa0fplfpiyx8zwc9y7d08a57lsk229-awscli2-2.27.2"
},
"system": "aarch64-darwin",
"group": "toplevel",
"priority": 5
},
{
"attr_path": "awscli2",
"broken": false,
"derivation": "/nix/store/7ykn42yxjf7ncyl2z00700hq214xs4jn-awscli2-2.27.2.drv",
"description": "Unified tool to manage your AWS services",
"install_id": "awscli2",
"license": "Apache-2.0",
"locked_url": "https://github.com/flox/nixpkgs?rev=96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"name": "awscli2-2.27.2",
"pname": "awscli2",
"rev": "96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"rev_count": 807377,
"rev_date": "2025-05-28T19:24:49Z",
"scrape_date": "2025-05-31T04:41:01.084855Z",
"stabilities": [
"unstable"
],
"unfree": false,
"version": "2.27.2",
"outputs_to_install": [
"out"
],
"outputs": {
"dist": "/nix/store/ss8xqzwn1l8qdbwlml439bx1ik13vxlk-awscli2-2.27.2-dist",
"out": "/nix/store/161s22pbddllc4mabgr5rvrpmhiffsp4-awscli2-2.27.2"
},
"system": "aarch64-linux",
"group": "toplevel",
"priority": 5
},
{
"attr_path": "awscli2",
"broken": false,
"derivation": "/nix/store/id7b0p6cwlkcsnfb767s820ig9xj0qnp-awscli2-2.27.2.drv",
"description": "Unified tool to manage your AWS services",
"install_id": "awscli2",
"license": "Apache-2.0",
"locked_url": "https://github.com/flox/nixpkgs?rev=96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"name": "awscli2-2.27.2",
"pname": "awscli2",
"rev": "96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"rev_count": 807377,
"rev_date": "2025-05-28T19:24:49Z",
"scrape_date": "2025-05-31T04:56:58.790359Z",
"stabilities": [
"unstable"
],
"unfree": false,
"version": "2.27.2",
"outputs_to_install": [
"out"
],
"outputs": {
"dist": "/nix/store/q9rzj2ikpl9vn4sns3ydsy7i8sachnha-awscli2-2.27.2-dist",
"out": "/nix/store/l18j8wgidf145m0zw6ydg8blim44hrkq-awscli2-2.27.2"
},
"system": "x86_64-darwin",
"group": "toplevel",
"priority": 5
},
{
"attr_path": "awscli2",
"broken": false,
"derivation": "/nix/store/j2ajkr3y4c0fg9lxl22z30hd4spb74d9-awscli2-2.27.2.drv",
"description": "Unified tool to manage your AWS services",
"install_id": "awscli2",
"license": "Apache-2.0",
"locked_url": "https://github.com/flox/nixpkgs?rev=96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"name": "awscli2-2.27.2",
"pname": "awscli2",
"rev": "96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"rev_count": 807377,
"rev_date": "2025-05-28T19:24:49Z",
"scrape_date": "2025-05-31T05:15:55.508841Z",
"stabilities": [
"unstable"
],
"unfree": false,
"version": "2.27.2",
"outputs_to_install": [
"out"
],
"outputs": {
"dist": "/nix/store/r5cw3141myaf5lnzra9c0nc8prb2zv6y-awscli2-2.27.2-dist",
"out": "/nix/store/gx3hx0b3lbm11sishxl4k7a7ha4w405q-awscli2-2.27.2"
},
"system": "x86_64-linux",
"group": "toplevel",
"priority": 5
},
{
"attr_path": "cargo",
"broken": false,
"derivation": "/nix/store/6665a8zg1cpw2fjamds6261fmg1pa458-cargo-1.86.0.drv",
"description": "Downloads your Rust project's dependencies and builds your project",
"install_id": "cargo",
"license": "[ MIT, Apache-2.0 ]",
"locked_url": "https://github.com/flox/nixpkgs?rev=96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"name": "cargo-1.86.0",
"pname": "cargo",
"rev": "96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"rev_count": 807377,
"rev_date": "2025-05-28T19:24:49Z",
"scrape_date": "2025-05-31T04:22:54.464772Z",
"stabilities": [
"unstable"
],
"unfree": false,
"version": "1.86.0",
"outputs_to_install": [
"out"
],
"outputs": {
"out": "/nix/store/v9kavzgzbym8adb6p7nxgid13jz76411-cargo-1.86.0"
},
"system": "aarch64-darwin",
"group": "toplevel",
"priority": 5
},
{
"attr_path": "cargo",
"broken": false,
"derivation": "/nix/store/jq0483mr9g99x5swkgvdf8w0hhj7yq73-cargo-1.86.0.drv",
"description": "Downloads your Rust project's dependencies and builds your project",
"install_id": "cargo",
"license": "[ MIT, Apache-2.0 ]",
"locked_url": "https://github.com/flox/nixpkgs?rev=96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"name": "cargo-1.86.0",
"pname": "cargo",
"rev": "96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"rev_count": 807377,
"rev_date": "2025-05-28T19:24:49Z",
"scrape_date": "2025-05-31T04:41:01.216082Z",
"stabilities": [
"unstable"
],
"unfree": false,
"version": "1.86.0",
"outputs_to_install": [
"out"
],
"outputs": {
"out": "/nix/store/4y830m5qm2xfvdacbsy3q1afxqnawihc-cargo-1.86.0"
},
"system": "aarch64-linux",
"group": "toplevel",
"priority": 5
},
{
"attr_path": "cargo",
"broken": false,
"derivation": "/nix/store/za3xhiav2xk7z8g0synrxz3az6ai0i2f-cargo-1.86.0.drv",
"description": "Downloads your Rust project's dependencies and builds your project",
"install_id": "cargo",
"license": "[ MIT, Apache-2.0 ]",
"locked_url": "https://github.com/flox/nixpkgs?rev=96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"name": "cargo-1.86.0",
"pname": "cargo",
"rev": "96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"rev_count": 807377,
"rev_date": "2025-05-28T19:24:49Z",
"scrape_date": "2025-05-31T04:56:58.878560Z",
"stabilities": [
"unstable"
],
"unfree": false,
"version": "1.86.0",
"outputs_to_install": [
"out"
],
"outputs": {
"out": "/nix/store/mdqqq9g5zrisd3ng2j56a9gijr5fawp5-cargo-1.86.0"
},
"system": "x86_64-darwin",
"group": "toplevel",
"priority": 5
},
{
"attr_path": "cargo",
"broken": false,
"derivation": "/nix/store/3f2cdkg4a3gjsw9mnb604ihqk1kq2dp2-cargo-1.86.0.drv",
"description": "Downloads your Rust project's dependencies and builds your project",
"install_id": "cargo",
"license": "[ MIT, Apache-2.0 ]",
"locked_url": "https://github.com/flox/nixpkgs?rev=96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"name": "cargo-1.86.0",
"pname": "cargo",
"rev": "96ec055edbe5ee227f28cdbc3f1ddf1df5965102",
"rev_count": 807377,
"rev_date": "2025-05-28T19:24:49Z",
"scrape_date": "2025-05-31T05:15:55.650404Z",
"stabilities": [
"unstable"
],
"unfree": false,
"version": "1.86.0",
"outputs_to_install": [
"out"
],
"outputs": {
"out": "/nix/store/npqlgsia03kfhv8m9mav6hfnbawpg0yg-cargo-1.86.0"
},
"system": "x86_64-linux",
"group": "toplevel",
"priority": 5
},
{
"attr_path": "fselect",
"broken": false,
"derivation": "/nix/store/admmm4b6lm869qnc1d487slsr6a9hl29-awscli-1.37.21.drv",
"description": "Unified tool to manage your AWS services",
Expand Down
4 changes: 3 additions & 1 deletion .flox/env/manifest.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ yamllint.pkg-path = "yamllint"
nushell.pkg-path = "nushell"
fselect.pkg-path = "fselect"
google-cloud-sdk.pkg-path = "google-cloud-sdk"
awscli.pkg-path = "awscli"
awscli2.pkg-path = "awscli2"
cargo.pkg-path = "cargo"

Comment thread
forstmeier marked this conversation as resolved.

[hook]
on-activate = '''
Expand Down
5 changes: 1 addition & 4 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,4 @@ infrastructure/Pulumi.production.yaml
.coverage*
.coverage/
coverage.xml
infrastructure/kubeconfig.json
*.egg-info
wandb/
*.csv
infrastructure/swarm.pem
10 changes: 3 additions & 7 deletions .mise.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,15 @@ depends = ["python:lint"]
description = "Run code quality checks"
run = """
nu linter.nu
yamllint -d "{extends: relaxed, rules: {line-length: {max: 110}}}" .
yamllint -d "{extends: relaxed, rules: {line-length: {max: 110}}}" .
"""

[tasks."infrastructure:up"]
description = "Launch cloud infrastructure"
# temporarily comment out ping test due to non-exposed endpoints
run = """
set -e
cd infrastructure
uv run pulumi up --yes --stack pocketsizefund/pocketsizefund/production
pulumi stack output KUBECONFIG > kubeconfig.json
export KUBECONFIG=<(pulumi stack output KUBECONFIG)
# nu ping.nu
source main.nu
infrastructure up
Comment thread
forstmeier marked this conversation as resolved.
"""

[tasks."infrastructure:down"]
Expand Down
8 changes: 4 additions & 4 deletions Dockerfile.tests
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM python:3.13
FROM python:3.12.10

COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv

Expand All @@ -9,8 +9,6 @@ RUN apt-get update && apt-get install -y \
linux-libc-dev \
&& rm -rf /var/lib/apt/lists/*

ENV PYTEST_ADDOPTS="--rootdir=/tests"

ENV PYTHON=1

WORKDIR /tests
Expand All @@ -27,10 +25,12 @@ COPY infrastructure/pyproject.toml ./infrastructure/

COPY libraries/python/pyproject.toml ./libraries/python/

RUN uv sync --all-packages --dev
RUN uv sync --all-packages --all-groups

COPY applications/ ./applications/

COPY libraries/ ./libraries/

RUN uv sync --all-packages --all-groups
Comment thread
forstmeier marked this conversation as resolved.

RUN mkdir -p /tests/coverage_output
12 changes: 11 additions & 1 deletion applications/datamanager/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,17 @@ name = "datamanager"
version = "0.1.0"
description = "Data collection and management service"
requires-python = "==3.12.10"
dependencies = ["internal"]
dependencies = [
"internal",
"loguru>=0.7.3",
"alpaca-py>=0.42.0",
"boto3>=1.38.23",
"duckdb>=1.2.2",
]

[tool.uv]
package = true
src = ["src"]

[tool.uv.sources]
internal = { workspace = true }
Loading