diff --git a/.flox/env/manifest.lock b/.flox/env/manifest.lock index 622e1ecf6..6237063cc 100644 --- a/.flox/env/manifest.lock +++ b/.flox/env/manifest.lock @@ -13,12 +13,13 @@ "pkg-path": "cargo", "pkg-group": "rust-toolchain" }, + "cargo-llvm-cov": { + "pkg-path": "cargo-llvm-cov", + "pkg-group": "rust-toolchain" + }, "cargo-nextest": { "pkg-path": "cargo-nextest" }, - "cargo-tarpaulin": { - "pkg-path": "cargo-tarpaulin" - }, "cargo-watch": { "pkg-path": "cargo-watch" }, @@ -65,6 +66,10 @@ "x86_64-darwin" ] }, + "llvm": { + "pkg-path": "llvm", + "pkg-group": "rust-toolchain" + }, "markdownlint-cli": { "pkg-path": "markdownlint-cli" }, @@ -80,6 +85,9 @@ "pkgconf": { "pkg-path": "pkgconf" }, + "pre-commit": { + "pkg-path": "pre-commit" + }, "pulumi": { "pkg-path": "pulumi" }, @@ -110,7 +118,7 @@ "pkg-path": "uv" }, "vulture": { - "pkg-path": "python313Packages.vulture" + "pkg-path": "python312Packages.vulture" }, "yamllint": { "pkg-path": "yamllint" @@ -248,27 +256,28 @@ { "attr_path": "cargo", "broken": false, - "derivation": "/nix/store/5r0l78j53v9vqvc8yb28kw4ml9wabivm-cargo-1.89.0.drv", + "derivation": "/nix/store/smqakq8ly0wv3q2jvhvcihx2qd4bmi9i-cargo-1.92.0.drv", "description": "Downloads your Rust project's dependencies and builds your project", "install_id": "cargo", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "cargo-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "cargo-1.92.0", "pname": "cargo", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-04T00:35:18.202964Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T04:35:54.555765Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "out" ], "outputs": { - "out": "/nix/store/f61dvjzzqj7c9xv07naqx6ga84vj10ip-cargo-1.89.0" + "out": "/nix/store/v4bvnkm0p5x41fhybskr0cf2zvkgyrvv-cargo-1.92.0" }, "system": "aarch64-darwin", "group": "rust-toolchain", @@ -277,27 +286,28 @@ { "attr_path": "cargo", "broken": false, - "derivation": "/nix/store/9b48lwz5mva14p0p0m2m6xxcc4vq6yzg-cargo-1.89.0.drv", + "derivation": "/nix/store/y2jp95g3j7vprw96lllsxliklvxh4icv-cargo-1.92.0.drv", "description": "Downloads your Rust project's dependencies and builds your project", "install_id": "cargo", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "cargo-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "cargo-1.92.0", "pname": "cargo", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T16:28:57.461564Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:05:22.879191Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "out" ], "outputs": { - "out": "/nix/store/cp19l4m09lp45cm86ilvqpl1n1bwf5fz-cargo-1.89.0" + "out": "/nix/store/piiqs6x2m8gv0n3z3pys8scn0y673piy-cargo-1.92.0" }, "system": "aarch64-linux", "group": "rust-toolchain", @@ -306,27 +316,28 @@ { "attr_path": "cargo", "broken": false, - "derivation": "/nix/store/9c96ifqn99sc26fyvrwgrl7217bn9qil-cargo-1.89.0.drv", + "derivation": "/nix/store/60bc9hzhvc4bl212dnj4450y58hwmaaw-cargo-1.92.0.drv", "description": "Downloads your Rust project's dependencies and builds your project", "install_id": "cargo", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "cargo-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "cargo-1.92.0", "pname": "cargo", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T16:54:57.240090Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:33:32.762094Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "out" ], "outputs": { - "out": "/nix/store/mdrdfm4q2076c8pp3gk798nzd6g0b5jn-cargo-1.89.0" + "out": "/nix/store/wf6279pgydd1nny4s5nx1msian6dbf9p-cargo-1.92.0" }, "system": "x86_64-darwin", "group": "rust-toolchain", @@ -335,27 +346,148 @@ { "attr_path": "cargo", "broken": false, - "derivation": "/nix/store/cdxbvr8pkywshy5h5x1s8lcmxfmyks6b-cargo-1.89.0.drv", + "derivation": "/nix/store/3gdr50smf0vcnv3v8slv9v3dd4splibj-cargo-1.92.0.drv", "description": "Downloads your Rust project's dependencies and builds your project", "install_id": "cargo", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "cargo-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "cargo-1.92.0", "pname": "cargo", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T17:20:46.748193Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T06:05:45.729819Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "1.92.0", + "outputs_to_install": [ + "out" + ], + "outputs": { + "out": "/nix/store/yqcsaywfvcyy9wmbzb5fawp29icgi7cb-cargo-1.92.0" + }, + "system": "x86_64-linux", + "group": "rust-toolchain", + "priority": 5 + }, + { + "attr_path": "cargo-llvm-cov", + "broken": false, + "derivation": "/nix/store/c6cqp91i92qbj0w0mwrcam7wjnmri3az-cargo-llvm-cov-0.8.1.drv", + "description": "Cargo subcommand to easily use LLVM source-based code coverage", + "install_id": "cargo-llvm-cov", + "license": "[ Apache-2.0, MIT ]", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "cargo-llvm-cov-0.8.1", + "pname": "cargo-llvm-cov", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T04:35:54.565395Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "0.8.1", + "outputs_to_install": [ + "out" + ], + "outputs": { + "out": "/nix/store/7frfknj5qf10i9cpv0a9mds57hvfxkqi-cargo-llvm-cov-0.8.1" + }, + "system": "aarch64-darwin", + "group": "rust-toolchain", + "priority": 5 + }, + { + "attr_path": "cargo-llvm-cov", + "broken": false, + "derivation": "/nix/store/spga7zbajn56rizapaz94snkx6g5pzqm-cargo-llvm-cov-0.8.1.drv", + "description": "Cargo subcommand to easily use LLVM source-based code coverage", + "install_id": "cargo-llvm-cov", + "license": "[ Apache-2.0, MIT ]", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "cargo-llvm-cov-0.8.1", + "pname": "cargo-llvm-cov", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:05:22.889428Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "0.8.1", + "outputs_to_install": [ + "out" + ], + "outputs": { + "out": "/nix/store/f0nmca75idfm62cn0fvm9h15zkibxnzb-cargo-llvm-cov-0.8.1" + }, + "system": "aarch64-linux", + "group": "rust-toolchain", + "priority": 5 + }, + { + "attr_path": "cargo-llvm-cov", + "broken": false, + "derivation": "/nix/store/9bszrzikv706sxs21lkhfyl9q0nrvr6q-cargo-llvm-cov-0.8.1.drv", + "description": "Cargo subcommand to easily use LLVM source-based code coverage", + "install_id": "cargo-llvm-cov", + "license": "[ Apache-2.0, MIT ]", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "cargo-llvm-cov-0.8.1", + "pname": "cargo-llvm-cov", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:33:32.771124Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "0.8.1", + "outputs_to_install": [ + "out" + ], + "outputs": { + "out": "/nix/store/86g1j4w8icws77b2340csly671i9fxf8-cargo-llvm-cov-0.8.1" + }, + "system": "x86_64-darwin", + "group": "rust-toolchain", + "priority": 5 + }, + { + "attr_path": "cargo-llvm-cov", + "broken": false, + "derivation": "/nix/store/wm1hi0ai83c2r2bv5jf1gai532z4nmx2-cargo-llvm-cov-0.8.1.drv", + "description": "Cargo subcommand to easily use LLVM source-based code coverage", + "install_id": "cargo-llvm-cov", + "license": "[ Apache-2.0, MIT ]", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "cargo-llvm-cov-0.8.1", + "pname": "cargo-llvm-cov", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T06:05:45.741110Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "0.8.1", "outputs_to_install": [ "out" ], "outputs": { - "out": "/nix/store/ynhfvh8h1da3l2hnm15ab2fh3h8a6lf4-cargo-1.89.0" + "out": "/nix/store/33h1j6s89qvs16hsymd10xilqynpk9fm-cargo-llvm-cov-0.8.1" }, "system": "x86_64-linux", "group": "rust-toolchain", @@ -364,27 +496,28 @@ { "attr_path": "clippy", "broken": false, - "derivation": "/nix/store/p5hwl8zlmrp3219ixq5gjz22iv8mr592-clippy-1.89.0.drv", + "derivation": "/nix/store/75mwy56m7jc5vwbq5d2gjzgad0inn4sx-clippy-1.92.0.drv", "description": "Bunch of lints to catch common mistakes and improve your Rust code", "install_id": "clippy", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "clippy-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "clippy-1.92.0", "pname": "clippy", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-04T00:35:18.375302Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T04:35:54.732255Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "out" ], "outputs": { - "out": "/nix/store/i3390dnz9cddhzz3s9kh7vqmmbxq5fbk-clippy-1.89.0" + "out": "/nix/store/phhksxd0vv7ml9imsr0lwiqvvmiaz23p-clippy-1.92.0" }, "system": "aarch64-darwin", "group": "rust-toolchain", @@ -393,28 +526,29 @@ { "attr_path": "clippy", "broken": false, - "derivation": "/nix/store/b9rfh88naayckmnn6gc3zf81xs4xnzff-clippy-1.89.0.drv", + "derivation": "/nix/store/s5g78r9ab453qrfxhj27id4zibj9jcs3-clippy-1.92.0.drv", "description": "Bunch of lints to catch common mistakes and improve your Rust code", "install_id": "clippy", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "clippy-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "clippy-1.92.0", "pname": "clippy", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T16:28:57.680916Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:05:23.107330Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "out" ], "outputs": { - "debug": "/nix/store/b71z3inpp723452f3jyq7rhng3khadyn-clippy-1.89.0-debug", - "out": "/nix/store/rsyy2pz3hbwghyq1l09jqpg3lr31v365-clippy-1.89.0" + "debug": "/nix/store/anj7w4nnbw9cvsrsq85vnwqlw8miss5n-clippy-1.92.0-debug", + "out": "/nix/store/1iqky89dxzj12yy9dsbyrarknswx6iyj-clippy-1.92.0" }, "system": "aarch64-linux", "group": "rust-toolchain", @@ -423,27 +557,28 @@ { "attr_path": "clippy", "broken": false, - "derivation": "/nix/store/qp6mmi777aflzi2qmkqp90wsw9dwvc9a-clippy-1.89.0.drv", + "derivation": "/nix/store/0i67r0aa9kp6njf0ji01pwbp76y6xn96-clippy-1.92.0.drv", "description": "Bunch of lints to catch common mistakes and improve your Rust code", "install_id": "clippy", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "clippy-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "clippy-1.92.0", "pname": "clippy", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T16:54:57.417700Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:33:32.940903Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "out" ], "outputs": { - "out": "/nix/store/0cdfy1i477bfzhacv93cs8i13282c2jq-clippy-1.89.0" + "out": "/nix/store/34yqzpcjb1cms86lij7wkszcjxdivx6f-clippy-1.92.0" }, "system": "x86_64-darwin", "group": "rust-toolchain", @@ -452,28 +587,162 @@ { "attr_path": "clippy", "broken": false, - "derivation": "/nix/store/0z47sqchm2pczv3h4fp730aqqy76igfq-clippy-1.89.0.drv", + "derivation": "/nix/store/jpbxcqbqahvws2n40fx7j7iksn0nmb61-clippy-1.92.0.drv", "description": "Bunch of lints to catch common mistakes and improve your Rust code", "install_id": "clippy", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "clippy-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "clippy-1.92.0", "pname": "clippy", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T17:20:46.988856Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T06:05:45.974631Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "out" ], "outputs": { - "debug": "/nix/store/ypqv62aqhanfvkrnfg612pbby11wsbnf-clippy-1.89.0-debug", - "out": "/nix/store/0yy1f5k22s6x9n6hva5n7v2fpnv3r7zw-clippy-1.89.0" + "debug": "/nix/store/yvamzvy4r4ml91sl0fap3jvp12wgwflx-clippy-1.92.0-debug", + "out": "/nix/store/zmrbcbd77w6nylgwyagnxh87all5swjf-clippy-1.92.0" + }, + "system": "x86_64-linux", + "group": "rust-toolchain", + "priority": 5 + }, + { + "attr_path": "llvm", + "broken": false, + "derivation": "/nix/store/a0z2gcp85jm7568c9hrbl5zdbflgjnsl-llvm-21.1.8.drv", + "description": "Collection of modular and reusable compiler and toolchain technologies", + "install_id": "llvm", + "license": "[ NCSA, Apache-2.0, LLVM-exception ]", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "llvm-21.1.8", + "pname": "llvm", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T04:36:12.840423Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "21.1.8", + "outputs_to_install": [ + "out", + "out" + ], + "outputs": { + "dev": "/nix/store/y96jxh4yxjvq6dhri26x35370w4vzpz9-llvm-21.1.8-dev", + "lib": "/nix/store/ixyxn9ipysnrr3q2x5mlfhbkl4ada78j-llvm-21.1.8-lib", + "out": "/nix/store/yq033yr0zwx5zcqmfgpdzl751xs9qzzb-llvm-21.1.8", + "python": "/nix/store/454pgdykfy2nwhd0cz1lgdi0v7ngk7yl-llvm-21.1.8-python" + }, + "system": "aarch64-darwin", + "group": "rust-toolchain", + "priority": 5 + }, + { + "attr_path": "llvm", + "broken": false, + "derivation": "/nix/store/lfqpgrr022iyxrs952i7376i2nw4nnw1-llvm-21.1.8.drv", + "description": "Collection of modular and reusable compiler and toolchain technologies", + "install_id": "llvm", + "license": "[ NCSA, Apache-2.0, LLVM-exception ]", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "llvm-21.1.8", + "pname": "llvm", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:05:51.100366Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "21.1.8", + "outputs_to_install": [ + "out" + ], + "outputs": { + "dev": "/nix/store/4s30hy5240j3rvn10w4v55c9xvdn2slb-llvm-21.1.8-dev", + "lib": "/nix/store/fq36ddj5says0g96sv408zzrzvrdxb42-llvm-21.1.8-lib", + "out": "/nix/store/p15k615lkir3sxkj066yww4sfiz1b81n-llvm-21.1.8", + "python": "/nix/store/5mvv9pv1kadg1frjs0kmsk8x905x5kmd-llvm-21.1.8-python" + }, + "system": "aarch64-linux", + "group": "rust-toolchain", + "priority": 5 + }, + { + "attr_path": "llvm", + "broken": false, + "derivation": "/nix/store/va5vrxggkampw9jh2dhycwijyyqs7si8-llvm-21.1.8.drv", + "description": "Collection of modular and reusable compiler and toolchain technologies", + "install_id": "llvm", + "license": "[ NCSA, Apache-2.0, LLVM-exception ]", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "llvm-21.1.8", + "pname": "llvm", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:33:51.129745Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "21.1.8", + "outputs_to_install": [ + "out" + ], + "outputs": { + "dev": "/nix/store/z2l891zsd5rzxsjhyzi552mg1lnq1yc0-llvm-21.1.8-dev", + "lib": "/nix/store/39h8zyrkknws8966p5kxgbyqcbc28q52-llvm-21.1.8-lib", + "out": "/nix/store/8wqrv04hhq3smj5id34402gdnjr44j2v-llvm-21.1.8", + "python": "/nix/store/kwia8hcfl8xfmgcqrvw7ii85cbdmb03k-llvm-21.1.8-python" + }, + "system": "x86_64-darwin", + "group": "rust-toolchain", + "priority": 5 + }, + { + "attr_path": "llvm", + "broken": false, + "derivation": "/nix/store/ilgysj1s90haj2ks4fr3v1a1h5kqplci-llvm-21.1.8.drv", + "description": "Collection of modular and reusable compiler and toolchain technologies", + "install_id": "llvm", + "license": "[ NCSA, Apache-2.0, LLVM-exception ]", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "llvm-21.1.8", + "pname": "llvm", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T06:06:16.945436Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "21.1.8", + "outputs_to_install": [ + "out" + ], + "outputs": { + "dev": "/nix/store/f36xxmmzivp37h20k4id9a1vximc508n-llvm-21.1.8-dev", + "lib": "/nix/store/apf2d7f948m7qch6vnk6i3rhrky0pd9v-llvm-21.1.8-lib", + "out": "/nix/store/5chxlq8nygpr9y6y9y9aqc3fsa28ng8w-llvm-21.1.8", + "python": "/nix/store/hyvq2nabmfhan1751dwb0dadrqc6nnlw-llvm-21.1.8-python" }, "system": "x86_64-linux", "group": "rust-toolchain", @@ -482,16 +751,17 @@ { "attr_path": "rustPlatform.rustLibSrc", "broken": false, - "derivation": "/nix/store/jrkzzcy77ckj0fka38kg65yab9a2f0lx-rust-lib-src.drv", + "derivation": "/nix/store/3gvixi5dyjqywk1i737lp4ajvpcbgz16-rust-lib-src.drv", "install_id": "rust-lib-src", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", "name": "rust-lib-src", "pname": "rustLibSrc", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-04T00:37:54.881351Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T04:38:28.843465Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, @@ -500,7 +770,7 @@ "out" ], "outputs": { - "out": "/nix/store/fyggs0s1lab9ax9gicqc1mgs4n5xj2d5-rust-lib-src" + "out": "/nix/store/2750983ggk4djlbx3m52x0zjy4mnf58z-rust-lib-src" }, "system": "aarch64-darwin", "group": "rust-toolchain", @@ -509,16 +779,17 @@ { "attr_path": "rustPlatform.rustLibSrc", "broken": false, - "derivation": "/nix/store/aqnjcfnafka5mgh694xzlg0nakdh5plh-rust-lib-src.drv", + "derivation": "/nix/store/yrg1gqsh9m34r85nkx9l03rlwqnimwjx-rust-lib-src.drv", "install_id": "rust-lib-src", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", "name": "rust-lib-src", "pname": "rustLibSrc", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T16:32:27.224394Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:08:48.346586Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, @@ -527,7 +798,7 @@ "out" ], "outputs": { - "out": "/nix/store/fjv3wbnhg8j5jp923mdin804xxw6jfqi-rust-lib-src" + "out": "/nix/store/kq9vnb8219xqv7xa734px0a6klwdc783-rust-lib-src" }, "system": "aarch64-linux", "group": "rust-toolchain", @@ -536,16 +807,17 @@ { "attr_path": "rustPlatform.rustLibSrc", "broken": false, - "derivation": "/nix/store/i7sxmg5w1x62xq6nl2rwx0klyibcjmc1-rust-lib-src.drv", + "derivation": "/nix/store/idyn0k1jq4cwqaxk2h3rkmi8dk7qp3qg-rust-lib-src.drv", "install_id": "rust-lib-src", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", "name": "rust-lib-src", "pname": "rustLibSrc", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T16:57:26.442568Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:36:08.079151Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, @@ -554,7 +826,7 @@ "out" ], "outputs": { - "out": "/nix/store/9bjj0ilq2a4l4ybva4g44vadp725fmgc-rust-lib-src" + "out": "/nix/store/yqrhjivksan0hryskrvsqks89c155hrd-rust-lib-src" }, "system": "x86_64-darwin", "group": "rust-toolchain", @@ -563,16 +835,17 @@ { "attr_path": "rustPlatform.rustLibSrc", "broken": false, - "derivation": "/nix/store/ws9yx7lpqska14v8irsyc792aga0nvnw-rust-lib-src.drv", + "derivation": "/nix/store/2f741glprwf1hn7gs99cyxw1bq6zb7vj-rust-lib-src.drv", "install_id": "rust-lib-src", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", "name": "rust-lib-src", "pname": "rustLibSrc", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T17:24:38.625914Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T06:09:22.470277Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, @@ -581,7 +854,7 @@ "out" ], "outputs": { - "out": "/nix/store/77xvx5vjwapbik9pwy5clbiz0r72279i-rust-lib-src" + "out": "/nix/store/jslljr5js1wnn7hqlmlbs6cx43n4az8g-rust-lib-src" }, "system": "x86_64-linux", "group": "rust-toolchain", @@ -590,30 +863,31 @@ { "attr_path": "rustc", "broken": false, - "derivation": "/nix/store/9v4pdmxxrd8dv915h45pqd2sn6q0gf5h-rustc-wrapper-1.89.0.drv", + "derivation": "/nix/store/cn0lz3fcyjn5208k0l0wcsrks0dr1cbn-rustc-wrapper-1.92.0.drv", "description": "Safe, concurrent, practical language (wrapper script)", "install_id": "rustc", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "rustc-wrapper-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "rustc-wrapper-1.92.0", "pname": "rustc", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-04T00:36:47.325918Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T04:37:21.198398Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "man", "out" ], "outputs": { - "doc": "/nix/store/b38f38i4l6pvw5x06ng3y2g34xx97ain-rustc-wrapper-1.89.0-doc", - "man": "/nix/store/sp5l2qbp6drmkbk80z6ks5hdnbaivsq4-rustc-wrapper-1.89.0-man", - "out": "/nix/store/5j36vpc3dcbqpjy73nxbsp6bx0g1k3fv-rustc-wrapper-1.89.0" + "doc": "/nix/store/09yhz82jqxwbmn4dbjy7p9hrvbr4g0mn-rustc-wrapper-1.92.0-doc", + "man": "/nix/store/ch54xfkz0dlqvhbinzlbkva2898nvihl-rustc-wrapper-1.92.0-man", + "out": "/nix/store/ymskl36napcfgl6wjz1xdjn0jd25inrv-rustc-wrapper-1.92.0" }, "system": "aarch64-darwin", "group": "rust-toolchain", @@ -622,30 +896,31 @@ { "attr_path": "rustc", "broken": false, - "derivation": "/nix/store/r3wd9nbl7ydl5zv8v0nb1yl0ax2895c2-rustc-wrapper-1.89.0.drv", + "derivation": "/nix/store/m43c677crygc100zvns1jh3kn84nxid2-rustc-wrapper-1.92.0.drv", "description": "Safe, concurrent, practical language (wrapper script)", "install_id": "rustc", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "rustc-wrapper-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "rustc-wrapper-1.92.0", "pname": "rustc", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T16:31:02.763829Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:07:19.088812Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "man", "out" ], "outputs": { - "doc": "/nix/store/ky97k3ch6szgmwj4jvhhzwxs2dpj87ri-rustc-wrapper-1.89.0-doc", - "man": "/nix/store/9nxjc3iv8h9qd1fcgdk4nrd0gjpmr9z0-rustc-wrapper-1.89.0-man", - "out": "/nix/store/59s3wpvdgm48kphsfn7gpnd3793c1rji-rustc-wrapper-1.89.0" + "doc": "/nix/store/camfpqmi94ssc1p8vkygp2s621ykq80m-rustc-wrapper-1.92.0-doc", + "man": "/nix/store/5ggpm5m3wkxj05si2id4b6sq4alf90qg-rustc-wrapper-1.92.0-man", + "out": "/nix/store/qnvqgfiqh8s08cqp452665l2b60a811h-rustc-wrapper-1.92.0" }, "system": "aarch64-linux", "group": "rust-toolchain", @@ -654,30 +929,31 @@ { "attr_path": "rustc", "broken": false, - "derivation": "/nix/store/gw59ljiqcjivxzzdphv7rsvw4fch7ig3-rustc-wrapper-1.89.0.drv", + "derivation": "/nix/store/c1vlh5i24x1p7gnp1gbm0v78kdijj4jr-rustc-wrapper-1.92.0.drv", "description": "Safe, concurrent, practical language (wrapper script)", "install_id": "rustc", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "rustc-wrapper-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "rustc-wrapper-1.92.0", "pname": "rustc", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T16:56:23.595084Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:34:59.543236Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "man", "out" ], "outputs": { - "doc": "/nix/store/f5fnw9k37frcag45pqcd1z4q9f22q7lb-rustc-wrapper-1.89.0-doc", - "man": "/nix/store/cvjs8z1q0k86i6bwyhcdbrw90pbf6vl3-rustc-wrapper-1.89.0-man", - "out": "/nix/store/83wfwlnxqrb6nazzi3gb42baqbjn1n6f-rustc-wrapper-1.89.0" + "doc": "/nix/store/7271gqq93bbxrqb9rw3hf5b5x6wdm2cn-rustc-wrapper-1.92.0-doc", + "man": "/nix/store/2k3fdy9xjwkx06rlfwfn449w442vgk0i-rustc-wrapper-1.92.0-man", + "out": "/nix/store/zkcwgmgli06nsh0v8yv82gnh5whgcdyl-rustc-wrapper-1.92.0" }, "system": "x86_64-darwin", "group": "rust-toolchain", @@ -686,30 +962,31 @@ { "attr_path": "rustc", "broken": false, - "derivation": "/nix/store/scfhx31076gi40xisf1gaqw6f7830sv0-rustc-wrapper-1.89.0.drv", + "derivation": "/nix/store/xh6m92k61nzgxdssfhhigkbdwybdjdji-rustc-wrapper-1.92.0.drv", "description": "Safe, concurrent, practical language (wrapper script)", "install_id": "rustc", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "rustc-wrapper-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "rustc-wrapper-1.92.0", "pname": "rustc", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T17:23:04.117531Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T06:07:49.637637Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "man", "out" ], "outputs": { - "doc": "/nix/store/15vjayx5plj1h5bqvlif6wdg6m6p6kkh-rustc-wrapper-1.89.0-doc", - "man": "/nix/store/20pcgwwz1hpbzimv9j4vmpbn83yyjrq0-rustc-wrapper-1.89.0-man", - "out": "/nix/store/kgln61a78z8aj4amxgp0lyjj508xxwkr-rustc-wrapper-1.89.0" + "doc": "/nix/store/425gvb2xnhkwb1izjr97wpwdwndwi516-rustc-wrapper-1.92.0-doc", + "man": "/nix/store/n1d4093lcx7ljgks1j352fbrf5w551v9-rustc-wrapper-1.92.0-man", + "out": "/nix/store/qvpg842zrjkywv7sqgw2h05spdyzcj86-rustc-wrapper-1.92.0" }, "system": "x86_64-linux", "group": "rust-toolchain", @@ -718,27 +995,28 @@ { "attr_path": "rustfmt", "broken": false, - "derivation": "/nix/store/3kgv2ac1vcdz1q8yn2w3w4h5kd066ipy-rustfmt-1.89.0.drv", + "derivation": "/nix/store/0g8v0vbjgh1qmmspf721vrl5m59jx4xy-rustfmt-1.92.0.drv", "description": "Tool for formatting Rust code according to style guidelines", "install_id": "rustfmt", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "rustfmt-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "rustfmt-1.92.0", "pname": "rustfmt", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-04T00:36:47.346012Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T04:37:21.216938Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "out" ], "outputs": { - "out": "/nix/store/5f1n6pbc2hy76n2gzhwa78b3zix3ggg5-rustfmt-1.89.0" + "out": "/nix/store/max1hp93q51kfj074lw6lg8w9m5nmh10-rustfmt-1.92.0" }, "system": "aarch64-darwin", "group": "rust-toolchain", @@ -747,27 +1025,28 @@ { "attr_path": "rustfmt", "broken": false, - "derivation": "/nix/store/z1naxivxg7kx9vmksh51gcqzljr643xb-rustfmt-1.89.0.drv", + "derivation": "/nix/store/ynbxbsbi3byjzxd24g987vbz86bpf2ip-rustfmt-1.92.0.drv", "description": "Tool for formatting Rust code according to style guidelines", "install_id": "rustfmt", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "rustfmt-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "rustfmt-1.92.0", "pname": "rustfmt", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T16:31:02.795214Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:07:19.116220Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "out" ], "outputs": { - "out": "/nix/store/38j5bn7l5qk5panxsdr7fi92zg7nz9hh-rustfmt-1.89.0" + "out": "/nix/store/vq6cxhd7iv2g444imys9wkmwqz3ffqbc-rustfmt-1.92.0" }, "system": "aarch64-linux", "group": "rust-toolchain", @@ -776,27 +1055,28 @@ { "attr_path": "rustfmt", "broken": false, - "derivation": "/nix/store/3ismbnwhj4mhyiynnh2wn4y6fjbszz00-rustfmt-1.89.0.drv", + "derivation": "/nix/store/8zs4i0m5m2bp9yvpw0hihhri3g5zk45i-rustfmt-1.92.0.drv", "description": "Tool for formatting Rust code according to style guidelines", "install_id": "rustfmt", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "rustfmt-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "rustfmt-1.92.0", "pname": "rustfmt", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T16:56:23.613817Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T05:34:59.562152Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "out" ], "outputs": { - "out": "/nix/store/ihazzv14jps09j69ijla4npx1wbqydxp-rustfmt-1.89.0" + "out": "/nix/store/dnjwscz1r33dkmikx3ms58qmhr60db9i-rustfmt-1.92.0" }, "system": "x86_64-darwin", "group": "rust-toolchain", @@ -805,27 +1085,28 @@ { "attr_path": "rustfmt", "broken": false, - "derivation": "/nix/store/7wrnhk2ic58zdl14zrip24gn6l3czpag-rustfmt-1.89.0.drv", + "derivation": "/nix/store/y9j3y5l6303n7vw930ymwa86bq2na72x-rustfmt-1.92.0.drv", "description": "Tool for formatting Rust code according to style guidelines", "install_id": "rustfmt", "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "name": "rustfmt-1.89.0", + "locked_url": "https://github.com/flox/nixpkgs?rev=00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "name": "rustfmt-1.92.0", "pname": "rustfmt", - "rev": "b3d51a0365f6695e7dd5cdf3e180604530ed33b4", - "rev_count": 888552, - "rev_date": "2025-11-02T19:18:41Z", - "scrape_date": "2025-11-05T17:23:04.149887Z", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev_count": 940249, + "rev_date": "2026-02-04T09:32:58Z", + "scrape_date": "2026-02-06T06:07:49.665407Z", "stabilities": [ + "staging", "unstable" ], "unfree": false, - "version": "1.89.0", + "version": "1.92.0", "outputs_to_install": [ "out" ], "outputs": { - "out": "/nix/store/a6yc6xyjml0lggp1kr4mn8zccyq5rkiy-rustfmt-1.89.0" + "out": "/nix/store/gl4y2v4lwiyllh85z712w8ajydia053l-rustfmt-1.92.0" }, "system": "x86_64-linux", "group": "rust-toolchain", @@ -1185,122 +1466,6 @@ "group": "toplevel", "priority": 5 }, - { - "attr_path": "cargo-tarpaulin", - "broken": false, - "derivation": "/nix/store/247jy6mkjq0hk6a4csxa4kqmpqg6v3j7-cargo-tarpaulin-0.34.1.drv", - "description": "Code coverage tool for Rust projects", - "install_id": "cargo-tarpaulin", - "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", - "name": "cargo-tarpaulin-0.34.1", - "pname": "cargo-tarpaulin", - "rev": "f61125a668a320878494449750330ca58b78c557", - "rev_count": 907002, - "rev_date": "2025-12-05T15:54:32Z", - "scrape_date": "2025-12-07T02:54:46.855491Z", - "stabilities": [ - "unstable" - ], - "unfree": false, - "version": "0.34.1", - "outputs_to_install": [ - "out" - ], - "outputs": { - "out": "/nix/store/qgni7qpyjxw0bl6wv6rzj4k8ynjk7n0q-cargo-tarpaulin-0.34.1" - }, - "system": "aarch64-darwin", - "group": "toplevel", - "priority": 5 - }, - { - "attr_path": "cargo-tarpaulin", - "broken": false, - "derivation": "/nix/store/q4mzxb5d6338byzg1l6vchkrg4hr60wv-cargo-tarpaulin-0.34.1.drv", - "description": "Code coverage tool for Rust projects", - "install_id": "cargo-tarpaulin", - "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", - "name": "cargo-tarpaulin-0.34.1", - "pname": "cargo-tarpaulin", - "rev": "f61125a668a320878494449750330ca58b78c557", - "rev_count": 907002, - "rev_date": "2025-12-05T15:54:32Z", - "scrape_date": "2025-12-07T03:04:24.719115Z", - "stabilities": [ - "unstable" - ], - "unfree": false, - "version": "0.34.1", - "outputs_to_install": [ - "out" - ], - "outputs": { - "out": "/nix/store/la343c25ikgcckbflxbyb021bdkfb91w-cargo-tarpaulin-0.34.1" - }, - "system": "aarch64-linux", - "group": "toplevel", - "priority": 5 - }, - { - "attr_path": "cargo-tarpaulin", - "broken": false, - "derivation": "/nix/store/jszjdjknhqdhkkl8zx5i5b4pj3a44h4j-cargo-tarpaulin-0.34.1.drv", - "description": "Code coverage tool for Rust projects", - "install_id": "cargo-tarpaulin", - "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", - "name": "cargo-tarpaulin-0.34.1", - "pname": "cargo-tarpaulin", - "rev": "f61125a668a320878494449750330ca58b78c557", - "rev_count": 907002, - "rev_date": "2025-12-05T15:54:32Z", - "scrape_date": "2025-12-07T03:14:44.983255Z", - "stabilities": [ - "unstable" - ], - "unfree": false, - "version": "0.34.1", - "outputs_to_install": [ - "out" - ], - "outputs": { - "out": "/nix/store/vlj059zdwq36rgbbn3n4dsppbhr1acyj-cargo-tarpaulin-0.34.1" - }, - "system": "x86_64-darwin", - "group": "toplevel", - "priority": 5 - }, - { - "attr_path": "cargo-tarpaulin", - "broken": false, - "derivation": "/nix/store/1mr9y80xv7gikfainpxcj252brri6sg1-cargo-tarpaulin-0.34.1.drv", - "description": "Code coverage tool for Rust projects", - "install_id": "cargo-tarpaulin", - "license": "[ MIT, Apache-2.0 ]", - "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", - "name": "cargo-tarpaulin-0.34.1", - "pname": "cargo-tarpaulin", - "rev": "f61125a668a320878494449750330ca58b78c557", - "rev_count": 907002, - "rev_date": "2025-12-05T15:54:32Z", - "scrape_date": "2025-12-07T03:23:59.199467Z", - "stabilities": [ - "unstable" - ], - "unfree": false, - "version": "0.34.1", - "outputs_to_install": [ - "out" - ], - "outputs": { - "out": "/nix/store/265q582fxg6hlir9cybv03yvpdswmczk-cargo-tarpaulin-0.34.1" - }, - "system": "x86_64-linux", - "group": "toplevel", - "priority": 5 - }, { "attr_path": "cargo-watch", "broken": false, @@ -2302,6 +2467,8 @@ "out", "out", "out", + "out", + "out", "out" ], "outputs": { @@ -2711,6 +2878,7 @@ "unfree": false, "version": "3.6.0", "outputs_to_install": [ + "bin", "bin", "bin", "man" @@ -2825,6 +2993,8 @@ "bin", "bin", "bin", + "bin", + "bin", "man" ], "outputs": { @@ -2967,6 +3137,128 @@ "group": "toplevel", "priority": 5 }, + { + "attr_path": "pre-commit", + "broken": false, + "derivation": "/nix/store/v0pqg0x6mdg0msjsi39kqqixm5swcmxd-pre-commit-4.5.0.drv", + "description": "Framework for managing and maintaining multi-language pre-commit hooks", + "install_id": "pre-commit", + "license": "MIT", + "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", + "name": "pre-commit-4.5.0", + "pname": "pre-commit", + "rev": "f61125a668a320878494449750330ca58b78c557", + "rev_count": 907002, + "rev_date": "2025-12-05T15:54:32Z", + "scrape_date": "2025-12-07T02:55:24.843048Z", + "stabilities": [ + "unstable" + ], + "unfree": false, + "version": "4.5.0", + "outputs_to_install": [ + "out" + ], + "outputs": { + "dist": "/nix/store/1laiwl4i8f23d7faqvxbcq954sxy247j-pre-commit-4.5.0-dist", + "out": "/nix/store/kbsd895yi5zzlnnyzlb314kqzkjcxirm-pre-commit-4.5.0" + }, + "system": "aarch64-darwin", + "group": "toplevel", + "priority": 5 + }, + { + "attr_path": "pre-commit", + "broken": false, + "derivation": "/nix/store/2fd8jhih5p7932zzg7vyvhmk6m87ya3x-pre-commit-4.5.0.drv", + "description": "Framework for managing and maintaining multi-language pre-commit hooks", + "install_id": "pre-commit", + "license": "MIT", + "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", + "name": "pre-commit-4.5.0", + "pname": "pre-commit", + "rev": "f61125a668a320878494449750330ca58b78c557", + "rev_count": 907002, + "rev_date": "2025-12-05T15:54:32Z", + "scrape_date": "2025-12-07T03:05:22.398323Z", + "stabilities": [ + "unstable" + ], + "unfree": false, + "version": "4.5.0", + "outputs_to_install": [ + "out" + ], + "outputs": { + "dist": "/nix/store/7pwsavmn3fwpc06b88lxavlhjmmrp0jl-pre-commit-4.5.0-dist", + "out": "/nix/store/9wcnpjj79915r15cgahl7s9kz9s1d6cz-pre-commit-4.5.0" + }, + "system": "aarch64-linux", + "group": "toplevel", + "priority": 5 + }, + { + "attr_path": "pre-commit", + "broken": false, + "derivation": "/nix/store/5aw5dm74j9x1y2dw5s3sbgkhrn7q06fy-pre-commit-4.5.0.drv", + "description": "Framework for managing and maintaining multi-language pre-commit hooks", + "install_id": "pre-commit", + "license": "MIT", + "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", + "name": "pre-commit-4.5.0", + "pname": "pre-commit", + "rev": "f61125a668a320878494449750330ca58b78c557", + "rev_count": 907002, + "rev_date": "2025-12-05T15:54:32Z", + "scrape_date": "2025-12-07T03:15:22.176355Z", + "stabilities": [ + "unstable" + ], + "unfree": false, + "version": "4.5.0", + "outputs_to_install": [ + "out" + ], + "outputs": { + "dist": "/nix/store/pzkk3zs6wgq0d1dsm3ri24hxp7c81h1h-pre-commit-4.5.0-dist", + "out": "/nix/store/vkbhfi3vphnas6347gbxsdh8pwnj5cl6-pre-commit-4.5.0" + }, + "system": "x86_64-darwin", + "group": "toplevel", + "priority": 5 + }, + { + "attr_path": "pre-commit", + "broken": false, + "derivation": "/nix/store/3w9393hd0p57w0a40nh1c3s2nk11lcpc-pre-commit-4.5.0.drv", + "description": "Framework for managing and maintaining multi-language pre-commit hooks", + "install_id": "pre-commit", + "license": "MIT", + "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", + "name": "pre-commit-4.5.0", + "pname": "pre-commit", + "rev": "f61125a668a320878494449750330ca58b78c557", + "rev_count": 907002, + "rev_date": "2025-12-05T15:54:32Z", + "scrape_date": "2025-12-07T03:24:59.340953Z", + "stabilities": [ + "unstable" + ], + "unfree": false, + "version": "4.5.0", + "outputs_to_install": [ + "out", + "out", + "out" + ], + "outputs": { + "dist": "/nix/store/n6dxcjwniqcgp9vqnypi82057brqz24w-pre-commit-4.5.0-dist", + "out": "/nix/store/hhzpwcr2r29603ygc147wr58vlkf4qv5-pre-commit-4.5.0" + }, + "system": "x86_64-linux", + "group": "toplevel", + "priority": 5 + }, { "attr_path": "pulumi", "broken": false, @@ -3432,19 +3724,19 @@ "priority": 5 }, { - "attr_path": "python313Packages.vulture", + "attr_path": "python312Packages.vulture", "broken": false, - "derivation": "/nix/store/6rp823v7s40fqscx1j8dp5m0m7v9vi4r-python3.13-vulture-2.14.drv", + "derivation": "/nix/store/9114gkd7g53hxhz7m1bylz9yd7fc426h-python3.12-vulture-2.14.drv", "description": "Finds unused code in Python programs", "install_id": "vulture", "license": "MIT", "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", - "name": "python3.13-vulture-2.14", + "name": "python3.12-vulture-2.14", "pname": "vulture", "rev": "f61125a668a320878494449750330ca58b78c557", "rev_count": 907002, "rev_date": "2025-12-05T15:54:32Z", - "scrape_date": "2025-12-07T02:56:08.936314Z", + "scrape_date": "2025-12-07T02:55:44.875335Z", "stabilities": [ "unstable" ], @@ -3454,27 +3746,27 @@ "out" ], "outputs": { - "dist": "/nix/store/jr2vfc6h5d2haplqmg0pjxli287ija62-python3.13-vulture-2.14-dist", - "out": "/nix/store/58qgn7q850qrimkw4gmxxbcd0yxzsmdf-python3.13-vulture-2.14" + "dist": "/nix/store/bdshb85zpxppz46pydjxkgclasq0yyri-python3.12-vulture-2.14-dist", + "out": "/nix/store/zhp5z4rk8xvgf80xw6jmh8f312qd954q-python3.12-vulture-2.14" }, "system": "aarch64-darwin", "group": "toplevel", "priority": 5 }, { - "attr_path": "python313Packages.vulture", + "attr_path": "python312Packages.vulture", "broken": false, - "derivation": "/nix/store/fydfhidy6snq18gcrwkgmzs8byxyd8n5-python3.13-vulture-2.14.drv", + "derivation": "/nix/store/chp666kk074lyjdjhrszfmwrkksg685p-python3.12-vulture-2.14.drv", "description": "Finds unused code in Python programs", "install_id": "vulture", "license": "MIT", "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", - "name": "python3.13-vulture-2.14", + "name": "python3.12-vulture-2.14", "pname": "vulture", "rev": "f61125a668a320878494449750330ca58b78c557", "rev_count": 907002, "rev_date": "2025-12-05T15:54:32Z", - "scrape_date": "2025-12-07T03:06:15.917900Z", + "scrape_date": "2025-12-07T03:05:47.101114Z", "stabilities": [ "unstable" ], @@ -3484,27 +3776,27 @@ "out" ], "outputs": { - "dist": "/nix/store/r034gyxcdwwagh6jsfc72hybb3jxiai0-python3.13-vulture-2.14-dist", - "out": "/nix/store/7vr41jnwznkwlvskdi28c4k9brjcpwz4-python3.13-vulture-2.14" + "dist": "/nix/store/v2hl6w0zyi72lsz5gc89g0xxrpjmwmwv-python3.12-vulture-2.14-dist", + "out": "/nix/store/0l40s5a0n79m0dbwqv7am96hpswvs0d9-python3.12-vulture-2.14" }, "system": "aarch64-linux", "group": "toplevel", "priority": 5 }, { - "attr_path": "python313Packages.vulture", + "attr_path": "python312Packages.vulture", "broken": false, - "derivation": "/nix/store/0hnax5inwn7swy0jfbpnlab1jdm9pgs8-python3.13-vulture-2.14.drv", + "derivation": "/nix/store/rrk2s4n9fbpclxawacrhs1h82br0kgn3-python3.12-vulture-2.14.drv", "description": "Finds unused code in Python programs", "install_id": "vulture", "license": "MIT", "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", - "name": "python3.13-vulture-2.14", + "name": "python3.12-vulture-2.14", "pname": "vulture", "rev": "f61125a668a320878494449750330ca58b78c557", "rev_count": 907002, "rev_date": "2025-12-05T15:54:32Z", - "scrape_date": "2025-12-07T03:16:02.603893Z", + "scrape_date": "2025-12-07T03:15:40.617644Z", "stabilities": [ "unstable" ], @@ -3514,27 +3806,27 @@ "out" ], "outputs": { - "dist": "/nix/store/8hds3dm0ha3figidiadrrw1qgf8dp4vh-python3.13-vulture-2.14-dist", - "out": "/nix/store/3yas5bqdwk3z7f0nrx68cc1lr5p6gg18-python3.13-vulture-2.14" + "dist": "/nix/store/1hm0s2yrw9f3h40kp1ijsqxb9n5s8q7q-python3.12-vulture-2.14-dist", + "out": "/nix/store/va0zb5zab6xnkjhk15k0a69p62i20xq7-python3.12-vulture-2.14" }, "system": "x86_64-darwin", "group": "toplevel", "priority": 5 }, { - "attr_path": "python313Packages.vulture", + "attr_path": "python312Packages.vulture", "broken": false, - "derivation": "/nix/store/k29xbyn1in0wlypzj8976jpmp3fk8dk0-python3.13-vulture-2.14.drv", + "derivation": "/nix/store/2pa24ibjdcsxb5nh1zxbx1i80mvmhj6p-python3.12-vulture-2.14.drv", "description": "Finds unused code in Python programs", "install_id": "vulture", "license": "MIT", "locked_url": "https://github.com/flox/nixpkgs?rev=f61125a668a320878494449750330ca58b78c557", - "name": "python3.13-vulture-2.14", + "name": "python3.12-vulture-2.14", "pname": "vulture", "rev": "f61125a668a320878494449750330ca58b78c557", "rev_count": 907002, "rev_date": "2025-12-05T15:54:32Z", - "scrape_date": "2025-12-07T03:25:52.611390Z", + "scrape_date": "2025-12-07T03:25:24.252039Z", "stabilities": [ "unstable" ], @@ -3544,8 +3836,8 @@ "out" ], "outputs": { - "dist": "/nix/store/52a4fn7fjp49vfwjrp1wn78qxgg5d0h1-python3.13-vulture-2.14-dist", - "out": "/nix/store/pq7bmragdlpm8p5qb2k3qhk9qrpx24k5-python3.13-vulture-2.14" + "dist": "/nix/store/wf2fjr0w25d4lkxl0d21v23kllbl81gk-python3.12-vulture-2.14-dist", + "out": "/nix/store/nam3pwdp9wvaqyjzj5gfdii3dcq7708w-python3.12-vulture-2.14" }, "system": "x86_64-linux", "group": "toplevel", diff --git a/.flox/env/manifest.toml b/.flox/env/manifest.toml index bab9f242f..21fbbf633 100644 --- a/.flox/env/manifest.toml +++ b/.flox/env/manifest.toml @@ -7,7 +7,7 @@ pulumi-python.pkg-path = "pulumiPackages.pulumi-python" ruff.pkg-path = "ruff" ruff.version = "0.14.7" uv.pkg-path = "uv" -vulture.pkg-path = "python313Packages.vulture" +vulture.pkg-path = "python312Packages.vulture" yamllint.pkg-path = "yamllint" nushell.pkg-path = "nushell" fselect.pkg-path = "fselect" @@ -17,7 +17,8 @@ mask.pkg-path = "mask" bacon.pkg-path = "bacon" cargo-watch.pkg-path = "cargo-watch" cargo-nextest.pkg-path = "cargo-nextest" -cargo-tarpaulin.pkg-path = "cargo-tarpaulin" +cargo-llvm-cov.pkg-path = "cargo-llvm-cov" +cargo-llvm-cov.pkg-group = "rust-toolchain" cargo.pkg-path = "cargo" cargo.pkg-group = "rust-toolchain" rustc.pkg-path = "rustc" @@ -28,6 +29,8 @@ rustfmt.pkg-path = "rustfmt" rustfmt.pkg-group = "rust-toolchain" rust-lib-src.pkg-path = "rustPlatform.rustLibSrc" rust-lib-src.pkg-group = "rust-toolchain" +llvm.pkg-path = "llvm" +llvm.pkg-group = "rust-toolchain" libiconv.pkg-path = "libiconv" libiconv.systems = ["aarch64-darwin", "x86_64-darwin"] rust-analyzer.pkg-path = "rust-analyzer" @@ -42,6 +45,7 @@ pkgconf.pkg-path = "pkgconf" direnv.pkg-path = "direnv" jq.pkg-path = "jq" markdownlint-cli.pkg-path = "markdownlint-cli" +pre-commit.pkg-path = "pre-commit" [hook] on-activate = ''' diff --git a/.github/workflows/launch_infrastructure.yaml b/.github/workflows/launch_infrastructure.yaml index d2b733bd2..05703b997 100644 --- a/.github/workflows/launch_infrastructure.yaml +++ b/.github/workflows/launch_infrastructure.yaml @@ -49,16 +49,12 @@ jobs: with: role-to-assume: ${{ secrets.AWS_IAM_INFRASTRUCTURE_ROLE_ARN }} aws-region: ${{ secrets.AWS_REGION }} + - name: Set up Docker Buildx + if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule' + uses: docker/setup-buildx-action@v3 - name: Install Flox if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule' uses: flox/install-flox-action@v2 - - name: Conditionally download artifacts - if: (steps.changes.outputs.service == 'true' || github.event_name == 'schedule') && matrix.service == 'equitypricemodel' - uses: flox/activate-action@v1 - env: - AWS_S3_ARTIFACTS_BUCKET_NAME: ${{ secrets.AWS_S3_ARTIFACTS_BUCKET_NAME }} - with: - command: mask models artifacts download equitypricemodel - name: Build ${{ matrix.service }} image if: steps.changes.outputs.service == 'true' || github.event_name == 'schedule' uses: flox/activate-action@v1 diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml new file mode 100644 index 000000000..6e2bf0ae3 --- /dev/null +++ b/.github/workflows/run_code_checks.yaml @@ -0,0 +1,93 @@ +--- +name: Code checks +run-name: Code checks +on: + pull_request: +jobs: + run_rust_code_checks: + name: Run Rust code checks + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Free disk space + run: | + echo "Disk space before cleanup:" + df -h + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + echo "Disk space after cleanup:" + df -h + - name: Install Flox + uses: flox/install-flox-action@v2 + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: . + cache-on-failure: false + shared-key: rust-continuous-integration + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + - name: Run Rust code checks + uses: flox/activate-action@v1 + with: + command: mask development rust all + - name: Upload test coverage results + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}_test_coverage + path: .coverage_output/rust.xml + if-no-files-found: error + overwrite: true + run_python_code_checks: + name: Run Python code checks + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Install Flox + uses: flox/install-flox-action@v2 + - name: Run Python code checks + uses: flox/activate-action@v1 + with: + command: mask development python all + - name: Upload test coverage results + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}_test_coverage + path: .coverage_output/python.xml + if-no-files-found: error + overwrite: true + run_markdown_code_checks: + name: Run Markdown code checks + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Install Flox + uses: flox/install-flox-action@v2 + - name: Run Markdown code checks + uses: flox/activate-action@v1 + with: + command: mask development markdown all + upload_test_coverage: + needs: + - run_rust_code_checks + - run_python_code_checks + if: ${{ always() && (needs.run_rust_code_checks.result == 'success' || needs.run_python_code_checks.result == 'success') }} + name: Upload coverage to Coveralls + runs-on: ubuntu-latest + steps: + - name: Download test coverage results + uses: actions/download-artifact@v4 + with: + pattern: '*_test_coverage' + path: .coverage_output/ + merge-multiple: true + run-id: ${{ github.run_id }} + - name: Upload coverage to Coveralls + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + files: .coverage_output/python.xml .coverage_output/rust.xml diff --git a/.github/workflows/run_markdown_code_checks.yaml b/.github/workflows/run_markdown_code_checks.yaml deleted file mode 100644 index 4af91f889..000000000 --- a/.github/workflows/run_markdown_code_checks.yaml +++ /dev/null @@ -1,35 +0,0 @@ ---- -name: Markdown code checks -run-name: Markdown code checks -on: - pull_request: -jobs: - detect_changes: - runs-on: ubuntu-latest - name: Detect changes in Markdown files - permissions: - pull-requests: read - outputs: - markdown: ${{ steps.filter.outputs.markdown }} - steps: - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - markdown: - - '**/*.md' - - '**/*.markdown' - run_markdown_code_checks: - needs: detect_changes - if: ${{ needs.detect_changes.outputs.markdown == 'true' }} - name: Run Markdown code checks - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install Flox - uses: flox/install-flox-action@v2 - - name: Run Markdown code checks - uses: flox/activate-action@v1 - with: - command: mask development markdown all diff --git a/.github/workflows/run_python_code_checks.yaml b/.github/workflows/run_python_code_checks.yaml deleted file mode 100644 index 086ac08d5..000000000 --- a/.github/workflows/run_python_code_checks.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: Python code checks -run-name: Python code checks -on: - pull_request: -jobs: - detect_changes: - runs-on: ubuntu-latest - name: Detect changes in Python files - permissions: - pull-requests: read - outputs: - python: ${{ steps.filter.outputs.python }} - steps: - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - python: - - '**/*.py' - - '**/*.pyi' - - '**/pyproject.toml' - - '**/uv.lock' - run_python_code_checks: - needs: detect_changes - if: ${{ needs.detect_changes.outputs.python == 'true' }} - name: Run Python code checks - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install Flox - uses: flox/install-flox-action@v2 - - name: Run Python code checks - uses: flox/activate-action@v1 - with: - command: mask development python all - - name: Upload coverage to Coveralls - uses: coverallsapp/github-action@v2 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - file: coverage/.python.xml diff --git a/.github/workflows/run_rust_code_checks.yaml b/.github/workflows/run_rust_code_checks.yaml deleted file mode 100644 index a763c8d9d..000000000 --- a/.github/workflows/run_rust_code_checks.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -name: Rust code checks -run-name: Rust code checks -on: - pull_request: -jobs: - detect_changes: - runs-on: ubuntu-latest - name: Detect Rust code changes - permissions: - pull-requests: read - outputs: - rust: ${{ steps.filter.outputs.rust }} - steps: - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - rust: - - '**/*.rs' - - '**/Cargo.toml' - - '**/Cargo.lock' - run_rust_code_checks: - needs: detect_changes - if: ${{ needs.detect_changes.outputs.rust == 'true' }} - name: Run Rust code checks - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install Flox - uses: flox/install-flox-action@v2 - - name: Run Rust code checks - uses: flox/activate-action@v1 - with: - command: mask development rust all diff --git a/.gitignore b/.gitignore index ff8e5286a..698d96fd3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,8 @@ __pycache__/ .envrc .env.nu -coverage/ -.coverage +.coverage_output/ +.coverage* *.csv *.egg-info/ wandb/ @@ -22,3 +22,4 @@ data/ **/*.json .claude/tasks/ .scratchpad/ + diff --git a/CLAUDE.md b/CLAUDE.md index 078cf0322..89ce0deb2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,7 @@ This is a collection of guidelines and references. - Use Polars for [Python](https://docs.pola.rs/api/python/stable/reference/index.html) and [Rust](https://docs.rs/polars/latest/polars/) dataframes - Use `typing` module `cast` function for `tinygrad` method outputs when necessary with union types -- Write `pytest` functions for Python tests +- Write `pytest` functions for Python tests using plain functions, not class-based test organization - Ensure Rust and Python automated test suites achieve at least 90% line or statement coverage per service or library - Exclude generated code, third-party code, tooling boilerplate, and anything explicitly excluded in this repository from test coverage calculations diff --git a/Cargo.lock b/Cargo.lock index c09fa1779..485ede241 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,6 +81,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anyhow" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" + [[package]] name = "ar_archive_writer" version = "0.5.1" @@ -197,7 +203,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -250,7 +256,7 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" dependencies = [ - "bitflags", + "bitflags 2.10.0", ] [[package]] @@ -284,6 +290,16 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-channel" version = "2.5.0" @@ -361,9 +377,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.13" +version = "1.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c456581cb3c77fafcc8c67204a70680d40b61112d6da78c77bd31d945b65f1b5" +checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2" dependencies = [ "aws-credential-types", "aws-runtime", @@ -391,9 +407,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.11" +version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3" +checksum = "e26bbf46abc608f2dc61fd6cb3b7b0665497cc259a21520151ed98f8b37d2c79" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -413,9 +429,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.37.0" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c34dda4df7017c8db52132f0f8a2e0f8161649d15723ed63fc00c82d0f2081a" +checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" dependencies = [ "cc", "cmake", @@ -425,9 +441,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c635c2dc792cb4a11ce1a4f392a925340d1bdf499289b5ec1ec6810954eb43f5" +checksum = "b0f92058d22a46adf53ec57a6a96f34447daf02bff52e8fb956c66bcd5c6ac12" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -439,6 +455,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", + "bytes-utils", "fastrand", "http 0.2.12", "http 1.4.0", @@ -452,9 +469,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.122.0" +version = "1.123.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94c2ca0cba97e8e279eb6c0b2d0aa10db5959000e602ab2b7c02de6b85d4c19b" +checksum = "c018f22146966fdd493a664f62ee2483dff256b42a08c125ab6a084bde7b77fe" dependencies = [ "aws-credential-types", "aws-runtime", @@ -487,9 +504,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.93.0" +version = "1.94.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcb38bb33fc0a11f1ffc3e3e85669e0a11a37690b86f77e75306d8f369146a0" +checksum = "699da1961a289b23842d88fe2984c6ff68735fdf9bdcbc69ceaeb2491c9bf434" dependencies = [ "aws-credential-types", "aws-runtime", @@ -511,9 +528,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.95.0" +version = "1.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ada8ffbea7bd1be1f53df1dadb0f8fdb04badb13185b3321b929d1ee3caad09" +checksum = "e3e3a4cb3b124833eafea9afd1a6cc5f8ddf3efefffc6651ef76a03cbc6b4981" dependencies = [ "aws-credential-types", "aws-runtime", @@ -535,9 +552,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.97.0" +version = "1.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6443ccadc777095d5ed13e21f5c364878c9f5bad4e35187a6cdbd863b0afcad" +checksum = "89c4f19655ab0856375e169865c91264de965bd74c407c7f1e403184b1049409" dependencies = [ "aws-credential-types", "aws-runtime", @@ -560,9 +577,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.8" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efa49f3c607b92daae0c078d48a4571f599f966dce3caee5f1ea55c4d9073f99" +checksum = "68f6ae9b71597dc5fd115d52849d7a5556ad9265885ad3492ea8d73b93bbc46e" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -588,9 +605,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.11" +version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52eec3db979d18cb807fc1070961cc51d87d069abe9ab57917769687368a8c6c" +checksum = "3cba48474f1d6807384d06fec085b909f5807e16653c5af5c45dfe89539f0b70" dependencies = [ "futures-util", "pin-project-lite", @@ -599,9 +616,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.64.3" +version = "0.64.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddcf418858f9f3edd228acb8759d77394fed7531cce78d02bdda499025368439" +checksum = "a764fa7222922f6c0af8eea478b0ef1ba5ce1222af97e01f33ca5e957bd7f3b9" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -620,9 +637,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.18" +version = "0.60.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35b9c7354a3b13c66f60fe4616d6d1969c9fd36b1b5333a5dfb3ee716b33c588" +checksum = "1c0b3e587fbaa5d7f7e870544508af8ce82ea47cd30376e69e1e37c4ac746f79" dependencies = [ "aws-smithy-types", "bytes", @@ -631,9 +648,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.63.3" +version = "0.63.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630e67f2a31094ffa51b210ae030855cb8f3b7ee1329bdd8d085aaf61e8b97fc" +checksum = "af4a8a5fe3e4ac7ee871237c340bbce13e982d37543b65700f4419e039f5d78e" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -653,9 +670,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fb0abf49ff0cab20fd31ac1215ed7ce0ea92286ba09e2854b42ba5cabe7525" +checksum = "0709f0083aa19b704132684bc26d3c868e06bd428ccc4373b0b55c3e8748a58b" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -683,27 +700,27 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.62.3" +version = "0.62.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb96aa208d62ee94104645f7b2ecaf77bf27edf161590b6224bfbac2832f979" +checksum = "27b3a779093e18cad88bbae08dc4261e1d95018c4c5b9356a52bcae7c0b6e9bb" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0a46543fbc94621080b3cf553eb4cbbdc41dd9780a30c4756400f0139440a1d" +checksum = "4d3f39d5bb871aaf461d59144557f16d5927a5248a983a40654d9cf3b9ba183b" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.13" +version = "0.60.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cebbddb6f3a5bd81553643e9c7daf3cc3dc5b0b5f398ac668630e8a84e6fff0" +checksum = "05f76a580e3d8f8961e5d48763214025a2af65c2fa4cd1fb7f270a0e107a71b0" dependencies = [ "aws-smithy-types", "urlencoding", @@ -711,9 +728,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3df87c14f0127a0d77eb261c3bc45d5b4833e2a1f63583ebfb728e4852134ee" +checksum = "8fd3dfc18c1ce097cf81fced7192731e63809829c6cbf933c1ec47452d08e1aa" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -736,9 +753,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.11.3" +version = "1.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49952c52f7eebb72ce2a754d3866cc0f87b97d2a46146b79f80f3a93fb2b3716" +checksum = "8c55e0837e9b8526f49e0b9bfa9ee18ddee70e853f5bc09c5d11ebceddcb0fec" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -753,9 +770,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.4.3" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3a26048eeab0ddeba4b4f9d51654c79af8c3b32357dc5f336cee85ab331c33" +checksum = "576b0d6991c9c32bc14fc340582ef148311f924d41815f641a308b5d11e8e7cd" dependencies = [ "base64-simd", "bytes", @@ -779,18 +796,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.13" +version = "0.60.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57" +checksum = "b53543b4b86ed43f051644f704a98c7291b3618b67adf057ee77a366fa52fcaa" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.11" +version = "1.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164" +checksum = "6c50f3cdf47caa8d01f2be4a6663ea02418e892f9bbfd82c7b9a3a37eaccdd3a" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -873,6 +890,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -915,6 +938,12 @@ dependencies = [ "virtue", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.10.0" @@ -968,6 +997,56 @@ dependencies = [ "objc2", ] +[[package]] +name = "bollard" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-named-pipe", + "hyper-rustls 0.27.7", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.47.1-rc.27.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + [[package]] name = "borsh" version = "1.6.0" @@ -1157,6 +1236,15 @@ dependencies = [ "cc", ] +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "comfy-table" version = "7.1.2" @@ -1346,7 +1434,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags", + "bitflags 2.10.0", "crossterm_winapi", "libc", "parking_lot", @@ -1359,7 +1447,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags", + "bitflags 2.10.0", "parking_lot", "rustix 0.38.44", ] @@ -1417,8 +1505,18 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] @@ -1435,13 +1533,38 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.114", +] + [[package]] name = "darling_macro" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", "quote", "syn 2.0.114", ] @@ -1458,12 +1581,16 @@ dependencies = [ "duckdb", "http-body-util", "hyper 1.8.1", + "mockito", "polars", "reqwest", "sentry", "sentry-tower", "serde", "serde_json", + "serial_test", + "testcontainers", + "testcontainers-modules", "thiserror 2.0.18", "tokio", "tokio-test", @@ -1477,9 +1604,9 @@ dependencies = [ [[package]] name = "debug_unsafe" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85d3cef41d236720ed453e102153a53e4cc3d2fde848c0078a50cf249e8e3e5b" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" [[package]] name = "debugid" @@ -1503,11 +1630,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +checksum = "cc3dc5ad92c2e2d1c193bbbbdf2ea477cb81331de4f3103f267ca18368b988c4" dependencies = [ "powerfmt", + "serde_core", ] [[package]] @@ -1538,7 +1666,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" dependencies = [ - "bitflags", + "bitflags 2.10.0", "objc2", ] @@ -1553,6 +1681,17 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "docker_credential" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d89dfcba45b4afad7450a99b39e751590463e45c04728cf555d36bb66940de8" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", +] + [[package]] name = "duckdb" version = "1.4.4" @@ -1647,6 +1786,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "ethnum" version = "1.5.2" @@ -1947,6 +2097,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + [[package]] name = "gimli" version = "0.32.3" @@ -1982,7 +2145,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", @@ -2001,7 +2164,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", @@ -2233,6 +2396,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper 1.8.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -2288,7 +2466,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -2307,6 +2485,21 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -2412,6 +2605,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -2439,6 +2638,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.13.0" @@ -2519,6 +2729,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "lexical-core" version = "1.0.6" @@ -2578,9 +2794,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.180" +version = "0.2.181" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "459427e2af2b9c839b132acb702a1c654d95e10f8c326bfc2ad11310e458b1c5" [[package]] name = "libduckdb-sys" @@ -2588,6 +2804,7 @@ version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d78bacb8933586cee3b550c39b610d314f9b7a48701ac7a914a046165a4ad8da" dependencies = [ + "cc", "flate2", "pkg-config", "reqwest", @@ -2610,9 +2827,9 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags", + "bitflags 2.10.0", "libc", - "redox_syscall 0.7.0", + "redox_syscall 0.7.1", ] [[package]] @@ -2709,9 +2926,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" @@ -2749,6 +2966,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mockito" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90820618712cab19cfc46b274c6c22546a82affcb3c3bdf0f29e3db8e1bb92c0" +dependencies = [ + "assert-json-diff", + "bytes", + "colored", + "futures-core", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "log", + "pin-project-lite", + "rand 0.9.2", + "regex", + "serde_json", + "serde_urlencoded", + "similar", + "tokio", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -2772,7 +3014,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags", + "bitflags 2.10.0", "cfg-if", "cfg_aliases", "libc", @@ -2891,7 +3133,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" dependencies = [ - "bitflags", + "bitflags 2.10.0", "objc2", "objc2-foundation", ] @@ -2912,7 +3154,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.10.0", "dispatch2", "objc2", ] @@ -2923,7 +3165,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ - "bitflags", + "bitflags 2.10.0", "dispatch2", "objc2", "objc2-core-foundation", @@ -2956,7 +3198,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" dependencies = [ - "bitflags", + "bitflags 2.10.0", "objc2", "objc2-core-foundation", "objc2-core-graphics", @@ -2974,7 +3216,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags", + "bitflags 2.10.0", "block2", "libc", "objc2", @@ -2987,7 +3229,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ - "bitflags", + "bitflags 2.10.0", "objc2", "objc2-core-foundation", ] @@ -2998,7 +3240,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" dependencies = [ - "bitflags", + "bitflags 2.10.0", "objc2", "objc2-core-foundation", "objc2-foundation", @@ -3010,7 +3252,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d87d638e33c06f577498cbcc50491496a3ed4246998a7fbba7ccb98b1e7eab22" dependencies = [ - "bitflags", + "bitflags 2.10.0", "block2", "objc2", "objc2-cloud-kit", @@ -3051,7 +3293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -3091,7 +3333,7 @@ version = "0.10.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ - "bitflags", + "bitflags 2.10.0", "cfg-if", "foreign-types", "libc", @@ -3197,6 +3439,31 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn 2.0.114", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3307,7 +3574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32b4fed2343961b3eea3db2cee165540c3e1ad9d5782350cc55a9e76cf440148" dependencies = [ "atoi_simd", - "bitflags", + "bitflags 2.10.0", "bytemuck", "chrono", "chrono-tz", @@ -3374,7 +3641,7 @@ version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e77b1f08ef6dbb032bb1d0d3365464be950df9905f6827a95b24c4ca5518901d" dependencies = [ - "bitflags", + "bitflags 2.10.0", "boxcar", "bytemuck", "chrono", @@ -3382,7 +3649,7 @@ dependencies = [ "comfy-table", "either", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.13.0", "itoa", "num-traits", "polars-arrow", @@ -3439,7 +3706,7 @@ version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343931b818cf136349135ba11dbc18c27683b52c3477b1ba8ca606cf5ab1965c" dependencies = [ - "bitflags", + "bitflags 2.10.0", "hashbrown 0.15.5", "num-traits", "polars-arrow", @@ -3509,7 +3776,7 @@ dependencies = [ "chrono", "fallible-streaming-iterator", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.13.0", "itoa", "num-traits", "polars-arrow", @@ -3527,7 +3794,7 @@ version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fb6e2c6c2fa4ea0c660df1c06cf56960c81e7c2683877995bae3d4e3d408147" dependencies = [ - "bitflags", + "bitflags 2.10.0", "chrono", "either", "memchr", @@ -3577,14 +3844,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acf6062173fdc9ba05775548beb66e76643a148d9aeadc9984ed712bc4babd76" dependencies = [ "argminmax", - "base64", + "base64 0.22.1", "bytemuck", "chrono", "chrono-tz", "either", "hashbrown 0.15.5", "hex", - "indexmap", + "indexmap 2.13.0", "jsonpath_lib_polars_vendor", "libm", "memchr", @@ -3613,7 +3880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc1d769180dec070df0dc4b89299b364bf2cfe32b218ecc4ddd8f1a49ae60669" dependencies = [ "async-stream", - "base64", + "base64 0.22.1", "brotli", "bytemuck", "ethnum", @@ -3650,7 +3917,7 @@ version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cd3a2e33ae4484fe407ab2d2ba5684f0889d1ccf3ad6b844103c03638e6d0a0" dependencies = [ - "bitflags", + "bitflags 2.10.0", "bytemuck", "bytes", "chrono", @@ -3685,7 +3952,7 @@ version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18734f17e0e348724df3ae65f3ee744c681117c04b041cac969dfceb05edabc0" dependencies = [ - "bitflags", + "bitflags 2.10.0", "bytemuck", "polars-arrow", "polars-compute", @@ -3700,7 +3967,7 @@ version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e6c1ab13e04d5167661a9854ed1ea0482b2ed9b8a0f1118dabed7cd994a85e3" dependencies = [ - "indexmap", + "indexmap 2.13.0", "polars-error", "polars-utils", "serde", @@ -3713,7 +3980,7 @@ version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4e7766da02cc1d464994404d3e88a7a0ccd4933df3627c325480fbd9bbc0a11" dependencies = [ - "bitflags", + "bitflags 2.10.0", "hex", "polars-core", "polars-error", @@ -3737,7 +4004,7 @@ dependencies = [ "async-channel", "async-trait", "atomic-waker", - "bitflags", + "bitflags 2.10.0", "crossbeam-channel", "crossbeam-deque", "crossbeam-queue", @@ -3803,7 +4070,7 @@ dependencies = [ "flate2", "foldhash 0.1.5", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.13.0", "libc", "memmap2", "num-traits", @@ -3846,6 +4113,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.114", +] + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -3888,9 +4165,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa96cb91275ed31d6da3e983447320c4eb219ac180fa1679a0889ff32861e2d" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" dependencies = [ "ar_archive_writer", "cc", @@ -4088,7 +4365,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags", + "bitflags 2.10.0", ] [[package]] @@ -4131,22 +4408,31 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.10.0", ] [[package]] name = "redox_syscall" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" +checksum = "35985aa610addc02e24fc232012c86fd11f14111180f902b67e2d5331f8ebf2b" dependencies = [ - "bitflags", + "bitflags 2.10.0", ] [[package]] @@ -4219,7 +4505,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-channel", @@ -4378,7 +4664,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys 0.4.15", @@ -4391,7 +4677,7 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ - "bitflags", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys 0.11.0", @@ -4438,6 +4724,15 @@ dependencies = [ "security-framework 3.5.1", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -4478,9 +4773,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "same-file" @@ -4491,6 +4786,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scc" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" +dependencies = [ + "sdd", +] + [[package]] name = "schannel" version = "0.1.28" @@ -4509,6 +4813,30 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -4525,6 +4853,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sdd" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" + [[package]] name = "seahash" version = "4.1.0" @@ -4551,7 +4885,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags", + "bitflags 2.10.0", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -4564,7 +4898,7 @@ version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ - "bitflags", + "bitflags 2.10.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -4747,7 +5081,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ - "indexmap", + "indexmap 2.13.0", "itoa", "memchr", "serde", @@ -4766,6 +5100,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "serde_stacker" version = "0.1.14" @@ -4789,6 +5134,63 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.13.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "serial_test" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0b343e184fc3b7bb44dff0705fffcf4b3756ba6aff420dddd8b24ca145e555" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f50427f258fb77356e4cd4aa0e87e2bd2c66dbcee41dc405282cae2bfc26c83" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "sha1" version = "0.10.6" @@ -4885,6 +5287,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "siphasher" version = "1.0.2" @@ -4981,9 +5389,9 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" +checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" dependencies = [ "cc", "cfg-if", @@ -5025,6 +5433,29 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.114", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "strum" version = "0.26.3" @@ -5119,7 +5550,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.10.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -5153,17 +5584,55 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.24.0" +version = "3.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.1", "once_cell", "rustix 1.1.3", "windows-sys 0.61.2", ] +[[package]] +name = "testcontainers" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a4f01f39bb10fc2a5ab23eb0d888b1e2bb168c157f61a1b98e6c501c639c74" +dependencies = [ + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "docker_credential", + "either", + "etcetera", + "futures", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tokio-tar", + "tokio-util", + "url", +] + +[[package]] +name = "testcontainers-modules" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d43ed4e8f58424c3a2c6c56dbea6643c3c23e8666a34df13c54f0a184e6c707" +dependencies = [ + "testcontainers", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -5215,9 +5684,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.46" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", @@ -5236,9 +5705,9 @@ checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.26" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -5347,6 +5816,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "tokio-test" version = "0.4.5" @@ -5387,7 +5871,7 @@ version = "0.23.10+spec-1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" dependencies = [ - "indexmap", + "indexmap 2.13.0", "toml_datetime", "toml_parser", "winnow", @@ -5395,9 +5879,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.6+spec-1.1.0" +version = "1.0.7+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" +checksum = "247eaa3197818b831697600aadf81514e577e0cba5eab10f7e064e78ae154df1" dependencies = [ "winnow", ] @@ -5424,7 +5908,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags", + "bitflags 2.10.0", "bytes", "futures-util", "http 1.4.0", @@ -5534,9 +6018,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "537dd038a89878be9b64dd4bd1b260315c1bb94f4d784956b81e27a088d9a09e" [[package]] name = "unicode-normalization" @@ -5568,6 +6052,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -5586,7 +6076,7 @@ version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ - "base64", + "base64 0.22.1", "log", "native-tls", "once_cell", @@ -5655,7 +6145,7 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7df16e474ef958526d1205f6dda359fdfab79d9aa6d54bafcb92dcd07673dca" dependencies = [ - "darling", + "darling 0.20.11", "once_cell", "proc-macro-error2", "proc-macro2", @@ -5739,6 +6229,15 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.108" @@ -5798,6 +6297,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.13.0", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -5811,6 +6332,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.10.0", + "hashbrown 0.15.5", + "indexmap 2.13.0", + "semver", +] + [[package]] name = "web-sys" version = "0.3.85" @@ -5950,6 +6483,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -5986,6 +6528,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -6019,6 +6576,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -6031,6 +6594,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -6043,6 +6612,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -6067,6 +6642,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -6079,6 +6660,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -6091,6 +6678,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -6103,6 +6696,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -6129,6 +6728,88 @@ name = "wit-bindgen" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap 2.13.0", + "prettyplease", + "syn 2.0.114", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.114", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.10.0", + "indexmap 2.13.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.13.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] [[package]] name = "writeable" @@ -6192,18 +6873,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.38" +version = "0.8.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57cf3aa6855b23711ee9852dfc97dfaa51c45feaba5b645d0c777414d494a961" +checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.38" +version = "0.8.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a616990af1a287837c4fe6596ad77ef57948f787e46ce28e166facc0cc1cb75" +checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" dependencies = [ "proc-macro2", "quote", @@ -6279,7 +6960,7 @@ dependencies = [ "arbitrary", "crc32fast", "flate2", - "indexmap", + "indexmap 2.13.0", "memchr", "zopfli", ] @@ -6292,9 +6973,9 @@ checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c" [[package]] name = "zmij" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" +checksum = "4de98dfa5d5b7fef4ee834d0073d560c9ca7b6c46a71d058c48db7960f8cfaf7" [[package]] name = "zopfli" diff --git a/README.md b/README.md index abd5f544c..4e40e0396 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ > The open source capital management company -[![Python code checks](https://github.com/oscmcompany/fund/actions/workflows/run_python_code_checks.yaml/badge.svg)](https://github.com/oscmcompany/fund/actions/workflows/run_python_code_checks.yaml) [![Rust code checks](https://github.com/oscmcompany/fund/actions/workflows/run_rust_code_checks.yaml/badge.svg)](https://github.com/oscmcompany/fund/actions/workflows/run_rust_code_checks.yaml) +[![Code checks](https://github.com/oscmcompany/fund/actions/workflows/run_code_checks.yaml/badge.svg)](https://github.com/oscmcompany/fund/actions/workflows/run_code_checks.yaml) [![Test coverage](https://coveralls.io/repos/github/oscmcompany/fund/badge.svg?branch=master)](https://coveralls.io/github/oscmcompany/fund?branch=master) ## About @@ -24,6 +24,7 @@ brew install flox # https://flox.dev/ for more information flox activate uv venv source .venv/bin/activate +pre-commit install mask --help # see all available Mask tasks mask setup mask development python install diff --git a/applications/datamanager/Cargo.toml b/applications/datamanager/Cargo.toml index 16e3c27e7..1015a871f 100644 --- a/applications/datamanager/Cargo.toml +++ b/applications/datamanager/Cargo.toml @@ -3,6 +3,10 @@ name = "datamanager" version = "0.1.0" edition = "2021" +[lib] +name = "datamanager" +path = "src/lib.rs" + [[bin]] name = "datamanager" path = "src/main.rs" @@ -30,7 +34,7 @@ tracing-subscriber = { version = "0.3.20", features = ["env-filter", "fmt"] } aws-config = "1.5" aws-sdk-s3 = "1.112" aws-credential-types = "1.2.6" -duckdb = { version = "1.4.3", features = ["r2d2", "chrono"] } +duckdb = { version = "1.4.3", features = ["bundled", "r2d2", "chrono"] } validator = { version = "0.20", features = ["derive"] } thiserror = "2.0.3" sentry = { version = "0.35", features = ["tracing", "reqwest", "rustls"] } @@ -38,8 +42,11 @@ sentry-tower = { version = "0.35", features = ["http"] } [dev-dependencies] tokio-test = "0.4" -serde_json = "1.0" tower = { version = "0.5", features = ["util"] } hyper = { version = "1.0", features = ["full"] } http-body-util = "0.1" reqwest = { version = "0.12", features = ["json"] } +mockito = "1.7" +serial_test = "3.3" +testcontainers = "0.23" +testcontainers-modules = { version = "0.11", features = ["localstack"] } diff --git a/applications/datamanager/src/data.rs b/applications/datamanager/src/data.rs index 0b2f0ded8..d54ea9bf8 100644 --- a/applications/datamanager/src/data.rs +++ b/applications/datamanager/src/data.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use std::io::Cursor; use tracing::{debug, info, warn}; -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] pub struct EquityBar { pub ticker: String, pub timestamp: i64, @@ -34,16 +34,14 @@ pub fn create_equity_bar_dataframe(equity_bars_rows: Vec) -> Result equity_bars_rows.iter().map(|b| b.volume_weighted_average_price).collect::>(), "transactions" => equity_bars_rows.iter().map(|b| b.transactions).collect::>(), ) - .map_err(|e| { - warn!("Failed to create equity bar DataFrame: {}", e); - Error::Other(format!("Failed to create DataFrame: {}", e)) - })?; + .map_err(|e| Error::Other(format!("Failed to create equity bar DataFrame: {}", e)))?; debug!("Normalizing ticker column to uppercase"); let equity_bars_dataframe = equity_bars_dataframe .lazy() .with_columns([col("ticker").str().to_uppercase().alias("ticker")]) - .collect()?; + .collect() + .map_err(|e| Error::Other(format!("Failed to normalize ticker column: {}", e)))?; info!( "Created equity bar DataFrame: {} rows x {} columns", @@ -54,7 +52,7 @@ pub fn create_equity_bar_dataframe(equity_bars_rows: Vec) -> Result) -> Result< "quantile_50" => prediction_rows.iter().map(|p| p.quantile_50).collect::>(), "quantile_90" => prediction_rows.iter().map(|p| p.quantile_90).collect::>(), ) - .map_err(|e| { - warn!("Failed to create predictions DataFrame: {}", e); - Error::Other(format!("Failed to create DataFrame: {}", e)) - })?; + .map_err(|e| Error::Other(format!("Failed to create predictions DataFrame: {}", e)))?; debug!("Normalizing ticker column to uppercase"); let unfiltered_prediction_dataframe = prediction_dataframe .lazy() .with_columns([col("ticker").str().to_uppercase().alias("ticker")]) - .collect()?; + .collect() + .map_err(|e| Error::Other(format!("Failed to normalize ticker column: {}", e)))?; debug!( "Unfiltered predictions DataFrame has {} rows", @@ -108,7 +104,8 @@ pub fn create_predictions_dataframe(prediction_rows: Vec) -> Result< col("quantile_50"), col("quantile_90"), ]) - .collect()?; + .collect() + .map_err(|e| Error::Other(format!("Failed to filter predictions DataFrame: {}", e)))?; info!( "Created predictions DataFrame: {} rows x {} columns (filtered from {} input rows)", @@ -120,7 +117,7 @@ pub fn create_predictions_dataframe(prediction_rows: Vec) -> Result< Ok(filtered_prediction_dataframe) } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] pub struct Portfolio { pub ticker: String, pub timestamp: f64, @@ -141,11 +138,7 @@ pub fn create_portfolio_dataframe(portfolio_rows: Vec) -> Result portfolio_rows.iter().map(|p| p.side.as_str()).collect::>(), "dollar_amount" => portfolio_rows.iter().map(|p| p.dollar_amount).collect::>(), "action" => portfolio_rows.iter().map(|p| p.action.as_str()).collect::>(), - ) - .map_err(|e| { - warn!("Failed to create portfolio DataFrame: {}", e); - Error::Other(format!("Failed to create DataFrame: {}", e)) - })?; + )?; debug!("Normalizing ticker, side, and action columns to uppercase"); let portfolio_dataframe = portfolio_dataframe @@ -201,10 +194,7 @@ pub fn create_equity_details_dataframe(csv_content: String) -> Result Result, ) -> impl IntoResponse { info!("Sync date: {}", payload.date); + + let massive_api_key = state.massive.key.clone(); + let date = payload.date.format("%Y-%m-%d").to_string(); let url = format!( "{}/v2/aggs/grouped/locale/us/market/stocks/{}", @@ -134,7 +137,7 @@ pub async fn sync( .http_client .get(&url) .header("accept", "application/json") - .query(&[("adjusted", "true"), ("apiKey", state.massive.key.as_str())]) + .query(&[("adjusted", "true"), ("apiKey", massive_api_key.as_str())]) .send() .await { diff --git a/applications/datamanager/src/equity_details.rs b/applications/datamanager/src/equity_details.rs index 5dc8ceafc..bcf9faec6 100644 --- a/applications/datamanager/src/equity_details.rs +++ b/applications/datamanager/src/equity_details.rs @@ -15,34 +15,36 @@ pub async fn get(AxumState(state): AxumState) -> impl IntoResponse { Ok(dataframe) => { let mut buffer = Vec::new(); let mut writer = CsvWriter::new(&mut buffer); - match writer.finish(&mut dataframe.clone()) { - Ok(_) => { - let csv_content = String::from_utf8(buffer).unwrap_or_else(|_| { - info!("Failed to convert CSV buffer to UTF-8"); - String::new() - }); - - let mut response = csv_content.into_response(); - response.headers_mut().insert( - header::CONTENT_TYPE, - "text/csv; charset=utf-8".parse().unwrap_or_else(|_| { - info!("Failed to set Content-Type header"); - header::HeaderValue::from_static("text/csv") - }), - ); - *response.status_mut() = StatusCode::OK; - response - } + Ok(_) => {} Err(err) => { - info!("Failed to write DataFrame as CSV: {}", err); - ( + info!("Failed to write CSV: {}", err); + return ( StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to convert DataFrame to CSV: {}", err), + format!("Failed to write CSV: {}", err), ) - .into_response() + .into_response(); } } + + let csv_content = match String::from_utf8(buffer) { + Ok(content) => content, + Err(err) => { + info!("Failed to convert CSV to UTF-8: {}", err); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to convert CSV to UTF-8: {}", err), + ) + .into_response(); + } + }; + let mut response = csv_content.into_response(); + response.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("text/csv; charset=utf-8"), + ); + *response.status_mut() = StatusCode::OK; + response } Err(err) => { info!("Failed to fetch equity details from S3: {}", err); diff --git a/applications/datamanager/src/lib.rs b/applications/datamanager/src/lib.rs new file mode 100644 index 000000000..e84f0c2e9 --- /dev/null +++ b/applications/datamanager/src/lib.rs @@ -0,0 +1,11 @@ +pub mod data; +pub mod equity_bars; +pub mod equity_details; +pub mod errors; +pub mod health; +pub mod portfolios; +pub mod predictions; +pub mod router; +pub mod startup; +pub mod state; +pub mod storage; diff --git a/applications/datamanager/src/main.rs b/applications/datamanager/src/main.rs index 01b42d27d..3e668965c 100644 --- a/applications/datamanager/src/main.rs +++ b/applications/datamanager/src/main.rs @@ -1,58 +1,67 @@ -mod data; -mod equity_bars; -mod equity_details; -mod errors; -mod health; -mod portfolios; -mod predictions; -mod router; -mod state; -mod storage; - -use router::create_app; -use std::env; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +use datamanager::startup::{initialize_sentry, initialize_tracing, run_server}; + +async fn run_with_bind_address(bind_address: &str) -> i32 { + let _sentry_guard = initialize_sentry(); + initialize_tracing().expect("Failed to initialize tracing"); + + handle_server_result(run_server(bind_address).await) +} + +fn handle_server_result(server_result: Result<(), std::io::Error>) -> i32 { + match server_result { + Ok(_) => 0, + Err(error) => { + tracing::error!("Server error: {}", error); + 1 + } + } +} #[tokio::main] async fn main() { - let _sentry_guard = sentry::init(( - env::var("SENTRY_DSN").unwrap_or_default(), - sentry::ClientOptions { - release: sentry::release_name!(), - environment: Some( - env::var("ENVIRONMENT") - .unwrap_or_else(|_| "development".to_string()) - .into(), - ), - traces_sample_rate: 1.0, - ..Default::default() - }, - )); - - tracing_subscriber::registry() - .with( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| "datamanager=debug,tower_http=debug,axum=debug".into()), - ) - .with(tracing_subscriber::fmt::layer()) - .with( - sentry::integrations::tracing::layer().event_filter(|metadata| { - use sentry::integrations::tracing::EventFilter; - match metadata.level() { - &tracing::Level::ERROR | &tracing::Level::WARN => EventFilter::Event, - _ => EventFilter::Breadcrumb, - } - }), - ) - .init(); - - tracing::info!("Starting datamanager service"); - - let app = create_app().await; - let listener = tokio::net::TcpListener::bind("0.0.0.0:8080").await.unwrap(); - - if let Err(e) = axum::serve(listener, app).await { - tracing::error!("Server error: {}", e); - std::process::exit(1); + let exit_code = run_with_bind_address("0.0.0.0:8080").await; + + if exit_code != 0 { + std::process::exit(exit_code); + } +} + +#[cfg(test)] +mod tests { + use super::{handle_server_result, run_with_bind_address}; + use serial_test::serial; + + #[test] + fn test_handle_server_result_success() { + assert_eq!(handle_server_result(Ok(())), 0); + } + + #[test] + fn test_handle_server_result_error() { + let error = std::io::Error::new(std::io::ErrorKind::AddrNotAvailable, "bind failed"); + assert_eq!(handle_server_result(Err(error)), 1); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial] + async fn test_run_with_bind_address_returns_error_code_for_invalid_bind_address() { + // SAFETY: Environment variable mutation is safe here because: + // 1. Test is marked with #[serial] to prevent concurrent execution + // 2. Env vars are set synchronously before spawning async tasks + unsafe { + std::env::set_var("AWS_S3_DATA_BUCKET_NAME", "test-bucket"); + std::env::set_var("MASSIVE_BASE_URL", "http://test"); + std::env::set_var("MASSIVE_API_KEY", "test-key"); + } + + let exit_code = run_with_bind_address("invalid-address").await; + + assert_eq!(exit_code, 1); + + unsafe { + std::env::remove_var("AWS_S3_DATA_BUCKET_NAME"); + std::env::remove_var("MASSIVE_BASE_URL"); + std::env::remove_var("MASSIVE_API_KEY"); + } } } diff --git a/applications/datamanager/src/router.rs b/applications/datamanager/src/router.rs index a6f379674..d343b6557 100644 --- a/applications/datamanager/src/router.rs +++ b/applications/datamanager/src/router.rs @@ -13,7 +13,10 @@ use tower_http::trace::TraceLayer; pub async fn create_app() -> Router { let state = State::from_env().await; + create_app_with_state(state) +} +pub fn create_app_with_state(state: State) -> Router { Router::new() .route("/health", get(health::get_health)) .route("/predictions", post(predictions::save)) diff --git a/applications/datamanager/src/startup.rs b/applications/datamanager/src/startup.rs new file mode 100644 index 000000000..83e15b855 --- /dev/null +++ b/applications/datamanager/src/startup.rs @@ -0,0 +1,197 @@ +use crate::router::create_app; +use axum::Router; +use std::env; +use tokio::net::TcpListener; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + +pub fn initialize_sentry() -> sentry::ClientInitGuard { + sentry::init(( + env::var("SENTRY_DSN").unwrap_or_default(), + sentry::ClientOptions { + release: sentry::release_name!(), + environment: Some( + env::var("ENVIRONMENT") + .unwrap_or_else(|_| "development".to_string()) + .into(), + ), + traces_sample_rate: 1.0, + ..Default::default() + }, + )) +} + +pub fn initialize_tracing() -> Result<(), Box> { + tracing_subscriber::registry() + .with( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| "datamanager=debug,tower_http=debug,axum=debug".into()), + ) + .with(tracing_subscriber::fmt::layer()) + .with( + sentry::integrations::tracing::layer().event_filter(|metadata| { + use sentry::integrations::tracing::EventFilter; + match metadata.level() { + &tracing::Level::ERROR | &tracing::Level::WARN => EventFilter::Event, + _ => EventFilter::Breadcrumb, + } + }), + ) + .try_init()?; + Ok(()) +} + +pub async fn serve_app(listener: TcpListener, app: Router) -> std::io::Result<()> { + axum::serve(listener, app).await +} + +pub async fn run_server(bind_address: &str) -> std::io::Result<()> { + tracing::info!("Starting datamanager service"); + + let app = create_app().await; + let listener = TcpListener::bind(bind_address).await?; + + serve_app(listener, app).await +} + +#[cfg(test)] +mod tests { + use super::{initialize_sentry, initialize_tracing, run_server, serve_app}; + use aws_credential_types::Credentials; + use aws_sdk_s3::config::Region; + use reqwest::StatusCode; + use serial_test::serial; + use std::time::Duration; + + use crate::{ + router::create_app_with_state, + state::{MassiveSecrets, State}, + }; + + struct EnvironmentVariableGuard { + name: String, + original_value: Option, + } + + impl EnvironmentVariableGuard { + fn set(name: &str, value: &str) -> Self { + let original_value = std::env::var(name).ok(); + // SAFETY: Environment variable mutation is safe here because: + // 1. Tests using this guard are marked with #[serial] to prevent concurrent execution + // 2. Env vars are set synchronously before spawning async tasks + // 3. The Drop implementation ensures cleanup when guard goes out of scope + unsafe { + std::env::set_var(name, value); + } + + Self { + name: name.to_string(), + original_value, + } + } + } + + impl Drop for EnvironmentVariableGuard { + fn drop(&mut self) { + match self.original_value.as_ref() { + Some(value) => { + // SAFETY: See set() method - protected by #[serial] annotation + unsafe { + std::env::set_var(&self.name, value); + } + } + None => { + // SAFETY: See set() method - protected by #[serial] annotation + unsafe { + std::env::remove_var(&self.name); + } + } + } + } + } + + async fn create_test_state() -> State { + let credentials = + Credentials::new("test-access-key", "test-secret-key", None, None, "tests"); + + let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .credentials_provider(credentials) + .endpoint_url("http://127.0.0.1:9") + .load() + .await; + + let s3_config = aws_sdk_s3::config::Builder::from(&shared_config) + .force_path_style(true) + .build(); + + let s3_client = aws_sdk_s3::Client::from_conf(s3_config); + + State::new( + reqwest::Client::new(), + MassiveSecrets { + base: "http://127.0.0.1:1".to_string(), + key: "test-api-key".to_string(), + }, + s3_client, + "test-bucket".to_string(), + ) + } + + #[test] + #[serial] + fn test_initialize_observability_functions() { + let _environment_guard = EnvironmentVariableGuard::set("ENVIRONMENT", "test"); + let _sentry_guard = initialize_sentry(); + let _ = initialize_tracing(); + let _ = initialize_tracing(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial] + async fn test_serve_app_responds_on_health_route() { + let app = create_app_with_state(create_test_state().await); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + + let server_task = tokio::spawn(async move { serve_app(listener, app).await }); + + let client = reqwest::Client::new(); + let health_url = format!("http://{}/health", address); + + let mut healthy = false; + for _ in 0..20 { + if let Ok(response) = client.get(&health_url).send().await { + if response.status() == StatusCode::OK { + healthy = true; + break; + } + } + + tokio::time::sleep(Duration::from_millis(50)).await; + } + + server_task.abort(); + let _ = server_task.await; + + assert!(healthy); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial] + async fn test_run_server_returns_error_for_invalid_bind_address() { + let _region_guard = EnvironmentVariableGuard::set("AWS_REGION", "us-east-1"); + let _access_key_guard = + EnvironmentVariableGuard::set("AWS_ACCESS_KEY_ID", "test-access-key"); + let _secret_key_guard = + EnvironmentVariableGuard::set("AWS_SECRET_ACCESS_KEY", "test-secret-key"); + let _metadata_guard = EnvironmentVariableGuard::set("AWS_EC2_METADATA_DISABLED", "true"); + let _bucket_guard = EnvironmentVariableGuard::set("AWS_S3_DATA_BUCKET_NAME", "test-bucket"); + let _massive_base_guard = + EnvironmentVariableGuard::set("MASSIVE_BASE_URL", "http://127.0.0.1:1"); + let _massive_key_guard = EnvironmentVariableGuard::set("MASSIVE_API_KEY", "test-api-key"); + + let result = run_server("invalid-address").await; + + assert!(result.is_err()); + } +} diff --git a/applications/datamanager/src/state.rs b/applications/datamanager/src/state.rs index 0c74d6618..c422623fc 100644 --- a/applications/datamanager/src/state.rs +++ b/applications/datamanager/src/state.rs @@ -1,6 +1,6 @@ use aws_sdk_s3::Client as S3Client; use reqwest::Client as HTTPClient; -use tracing::{debug, error, info, warn}; +use tracing::{debug, info}; #[derive(Clone)] pub struct MassiveSecrets { @@ -37,54 +37,40 @@ impl State { let s3_client = S3Client::new(&config); - let bucket_name = match std::env::var("AWS_S3_DATA_BUCKET_NAME") { - Ok(name) => { - info!("Using S3 bucket from environment: {}", name); - name - } - Err(_) => { - let default_bucket = "oscm-data".to_string(); - error!( - "AWS_S3_DATA_BUCKET_NAME not set, using default: {}", - default_bucket - ); - default_bucket - } - }; + let bucket_name = + std::env::var("AWS_S3_DATA_BUCKET_NAME").unwrap_or_else(|_| "oscm-data".to_string()); + info!("Using S3 bucket: {}", bucket_name); - let massive_base = match std::env::var("MASSIVE_BASE_URL") { - Ok(url) => { - info!("Using Massive API base URL from environment: {}", url); - url - } - Err(_) => { - let default_url = "https://api.massive.io".to_string(); - error!("MASSIVE_BASE_URL not set, using default: {}", default_url); - default_url - } - }; + let massive_base_url = std::env::var("MASSIVE_BASE_URL") + .unwrap_or_else(|_| "https://api.massive.io".to_string()); + info!("Using Massive API base URL: {}", massive_base_url); - let massive_key = match std::env::var("MASSIVE_API_KEY") { - Ok(key) => { - debug!("MASSIVE_API_KEY loaded (length: {} chars)", key.len()); - key - } - Err(_) => { - warn!("MASSIVE_API_KEY not set - equity bar sync will not work"); - String::new() - } - }; + let massive_api_key = std::env::var("MASSIVE_API_KEY").unwrap_or_else(|_| String::new()); info!("Application state initialized successfully"); Self { http_client, massive: MassiveSecrets { - base: massive_base, - key: massive_key, + base: massive_base_url, + key: massive_api_key, }, s3_client, bucket_name, } } + + pub fn new( + http_client: HTTPClient, + massive: MassiveSecrets, + s3_client: S3Client, + bucket_name: String, + ) -> Self { + Self { + http_client, + massive, + s3_client, + bucket_name, + } + } } diff --git a/applications/datamanager/src/storage.rs b/applications/datamanager/src/storage.rs index b14528419..6b392e932 100644 --- a/applications/datamanager/src/storage.rs +++ b/applications/datamanager/src/storage.rs @@ -37,6 +37,62 @@ pub async fn write_predictions_dataframe_to_s3( write_dataframe_to_s3(state, dataframe, timestamp, "predictions".to_string()).await } +pub fn is_valid_ticker(ticker: &str) -> bool { + !ticker.is_empty() + && ticker.chars().any(|c| c.is_ascii_alphanumeric()) + && ticker + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-') +} + +pub fn format_s3_key(timestamp: &DateTime, dataframe_type: &str) -> String { + let year = timestamp.format("%Y"); + let month = timestamp.format("%m"); + let day = timestamp.format("%d"); + + format!( + "equity/{}/daily/year={}/month={}/day={}/data.parquet", + dataframe_type, year, month, day, + ) +} + +pub fn date_to_int(timestamp: &DateTime) -> Result { + timestamp + .format("%Y%m%d") + .to_string() + .parse::() + .map_err(|e| Error::Other(format!("Failed to convert date to integer: {}", e))) +} + +pub fn escape_sql_ticker(ticker: &str) -> String { + ticker.replace('\'', "''") +} + +pub fn sanitize_duckdb_config_value(value: &str) -> Result { + if value.is_empty() { + return Err(Error::Other("Configuration value cannot be empty".into())); + } + + // Reject SQL metacharacters + if value.contains('\'') || value.contains('"') || value.contains(';') || value.contains("--") { + let message = format!( + "Invalid configuration value contains SQL metacharacters: {}", + value + ); + error!("{}", message); + return Err(Error::Other(message)); + } + + // Reasonable length limit + if value.len() > 512 { + let message = format!("Configuration value too long: {} characters", value.len()); + error!("{}", message); + return Err(Error::Other(message)); + } + + Ok(value.to_string()) +} + async fn write_dataframe_to_s3( state: &State, dataframe: &DataFrame, @@ -45,14 +101,7 @@ async fn write_dataframe_to_s3( ) -> Result { info!("Uploading DataFrame to S3 as parquet"); - let year = timestamp.format("%Y"); - let month = timestamp.format("%m"); - let day = timestamp.format("%d"); - - let key = format!( - "equity/{}/daily/year={}/month={}/day={}/data.parquet", - dataframe_type, year, month, day, - ); + let key = format_s3_key(timestamp, &dataframe_type); let mut buffer = Vec::new(); { @@ -126,22 +175,50 @@ async fn create_duckdb_connection() -> Result { ); let session_token = credentials.session_token().unwrap_or_default(); - let s3_config = format!( - " - SET s3_region='{}'; - SET s3_url_style='path'; - SET s3_access_key_id='{}'; - SET s3_secret_access_key='{}'; - SET s3_session_token='{}'; - ", - region, - credentials.access_key_id(), - credentials.secret_access_key(), - session_token - ); + + let sanitized_region = sanitize_duckdb_config_value(®ion)?; + let sanitized_access_key = sanitize_duckdb_config_value(credentials.access_key_id())?; + let sanitized_secret_key = sanitize_duckdb_config_value(credentials.secret_access_key())?; + // Session token can be empty for static credentials (no temporary session) + let sanitized_session_token = if !session_token.is_empty() { + sanitize_duckdb_config_value(session_token)? + } else { + String::new() + }; + + let mut s3_configuration_statements = vec![ + format!("SET s3_region='{}';", sanitized_region), + "SET s3_url_style='path';".to_string(), + format!("SET s3_access_key_id='{}';", sanitized_access_key), + format!("SET s3_secret_access_key='{}';", sanitized_secret_key), + format!("SET s3_session_token='{}';", sanitized_session_token), + ]; + + if let Ok(duckdb_s3_endpoint) = std::env::var("DUCKDB_S3_ENDPOINT") { + debug!("Configuring DuckDB with custom S3 endpoint"); + let sanitized_endpoint = sanitize_duckdb_config_value(&duckdb_s3_endpoint)?; + s3_configuration_statements.push(format!("SET s3_endpoint='{}';", sanitized_endpoint)); + + let duckdb_s3_use_ssl = std::env::var("DUCKDB_S3_USE_SSL") + .unwrap_or_else(|_| "true".to_string()) + .to_lowercase(); + + if duckdb_s3_use_ssl != "true" && duckdb_s3_use_ssl != "false" { + let message = format!( + "Invalid DUCKDB_S3_USE_SSL: must be 'true' or 'false', got '{}'", + duckdb_s3_use_ssl + ); + error!("{}", message); + return Err(Error::Other(message)); + } + + s3_configuration_statements.push(format!("SET s3_use_ssl={};", duckdb_s3_use_ssl)); + } + + let s3_configuration_sql = s3_configuration_statements.join("\n"); debug!("Configuring DuckDB S3 settings"); - connection.execute_batch(&s3_config)?; + connection.execute_batch(&s3_configuration_sql)?; info!("DuckDB connection established with S3 access"); Ok(connection) @@ -157,7 +234,23 @@ pub async fn query_equity_bars_parquet_from_s3( let (start_timestamp, end_timestamp) = match (start_timestamp, end_timestamp) { (Some(start), Some(end)) => (start, end), - _ => { + (Some(start), None) => { + let end_date = chrono::Utc::now(); + info!( + "No end date specified, defaulting to now: {} to {}", + start, end_date + ); + (start, end_date) + } + (None, Some(end)) => { + let start_date = end - chrono::Duration::days(7); + info!( + "No start date specified, defaulting to 7 days before end: {} to {}", + start_date, end + ); + (start_date, end) + } + (None, None) => { let end_date = chrono::Utc::now(); let start_date = end_date - chrono::Duration::days(7); info!( @@ -179,16 +272,8 @@ pub async fn query_equity_bars_parquet_from_s3( info!("Using S3 glob pattern: {}", s3_glob); // Build date filter for hive partitions - let start_date_int = start_timestamp - .format("%Y%m%d") - .to_string() - .parse::() - .unwrap_or(0); - let end_date_int = end_timestamp - .format("%Y%m%d") - .to_string() - .parse::() - .unwrap_or(99999999); + let start_date_int = date_to_int(&start_timestamp)?; + let end_date_int = date_to_int(&end_timestamp)?; debug!( "Date range filter: {} to {} (as integers)", @@ -200,10 +285,7 @@ pub async fn query_equity_bars_parquet_from_s3( Some(ticker_list) if !ticker_list.is_empty() => { debug!("Validating {} tickers for query filter", ticker_list.len()); for ticker in ticker_list { - if !ticker - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-') - { + if !is_valid_ticker(ticker) { warn!("Invalid ticker format rejected: {}", ticker); return Err(Error::Other(format!("Invalid ticker format: {}", ticker))); } @@ -211,7 +293,7 @@ pub async fn query_equity_bars_parquet_from_s3( debug!("Ticker validation passed: {:?}", ticker_list); let ticker_values = ticker_list .iter() - .map(|t| format!("'{}'", t.replace('\'', "''"))) + .map(|t| format!("'{}'", escape_sql_ticker(t))) .collect::>() .join(", "); format!("AND ticker IN ({})", ticker_values) diff --git a/applications/datamanager/tests/common/mod.rs b/applications/datamanager/tests/common/mod.rs new file mode 100644 index 000000000..341c400b8 --- /dev/null +++ b/applications/datamanager/tests/common/mod.rs @@ -0,0 +1,273 @@ +#![allow(dead_code)] + +use aws_credential_types::Credentials; +use aws_sdk_s3::{config::Region, primitives::ByteStream, Client as S3Client}; +use axum::Router; +use std::{net::SocketAddr, sync::OnceLock, time::Duration}; +use testcontainers::runners::AsyncRunner; +use testcontainers_modules::localstack::LocalStack; +use tokio::{net::TcpListener, sync::oneshot, task::JoinHandle}; + +const TEST_BUCKET: &str = "test-bucket"; +const TEST_ACCESS_KEY: &str = "test"; +const TEST_SECRET_KEY: &str = "test"; +const TEST_REGION: &str = "us-east-1"; + +static LOCALSTACK_ENDPOINT: OnceLock = OnceLock::new(); +static TRACING_INIT: std::sync::Once = std::sync::Once::new(); + +pub struct EnvironmentVariableGuard { + name: String, + original_value: Option, +} + +impl EnvironmentVariableGuard { + pub fn set(name: &str, value: &str) -> Self { + let original_value = std::env::var(name).ok(); + unsafe { + std::env::set_var(name, value); + } + + Self { + name: name.to_string(), + original_value, + } + } + + pub fn remove(name: &str) -> Self { + let original_value = std::env::var(name).ok(); + unsafe { + std::env::remove_var(name); + } + + Self { + name: name.to_string(), + original_value, + } + } +} + +impl Drop for EnvironmentVariableGuard { + fn drop(&mut self) { + match self.original_value.as_ref() { + Some(value) => unsafe { + std::env::set_var(&self.name, value); + }, + None => unsafe { + std::env::remove_var(&self.name); + }, + } + } +} + +pub struct DuckDbEnvironmentGuard { + _guards: Vec, +} + +impl DuckDbEnvironmentGuard { + pub fn new(endpoint_host_port: &str) -> Self { + let guards = vec![ + EnvironmentVariableGuard::set("AWS_REGION", TEST_REGION), + EnvironmentVariableGuard::set("AWS_ACCESS_KEY_ID", TEST_ACCESS_KEY), + EnvironmentVariableGuard::set("AWS_SECRET_ACCESS_KEY", TEST_SECRET_KEY), + EnvironmentVariableGuard::set("AWS_EC2_METADATA_DISABLED", "true"), + EnvironmentVariableGuard::set("DUCKDB_S3_ENDPOINT", endpoint_host_port), + EnvironmentVariableGuard::set("DUCKDB_S3_USE_SSL", "false"), + ]; + Self { _guards: guards } + } +} + +pub fn initialize_test_tracing() { + TRACING_INIT.call_once(|| { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::TRACE) + .with_test_writer() + .try_init(); + }); +} + +pub async fn get_localstack_endpoint() -> String { + if let Some(endpoint) = LOCALSTACK_ENDPOINT.get() { + return endpoint.clone(); + } + + let container = LocalStack::default() + .start() + .await + .expect("Failed to start LocalStack container — is Docker running?"); + + let host = container.get_host().await.unwrap(); + let port = container.get_host_port_ipv4(4566).await.unwrap(); + let endpoint = format!("http://{}:{}", host, port); + + // INTENTIONAL LEAK: Container is leaked to keep it alive for entire test run. + // + // Rationale: + // - Tests use #[serial] for sequential execution within this process + // - All tests share the same LocalStack container for performance + // - Container cleanup happens automatically when process exits + // - Alternative (proper Drop cleanup) requires complex lifetime management + // across static OnceLock, creating more complexity than benefit + // + // Trade-off: Small memory leak during test execution vs architectural complexity + // Impact: Container memory is reclaimed when test process terminates + Box::leak(Box::new(container)); + + let _ = LOCALSTACK_ENDPOINT.set(endpoint.clone()); + endpoint +} + +pub async fn create_test_s3_client(endpoint_url: &str) -> S3Client { + let credentials = Credentials::new(TEST_ACCESS_KEY, TEST_SECRET_KEY, None, None, "tests"); + + let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .region(Region::new(TEST_REGION)) + .credentials_provider(credentials) + .endpoint_url(endpoint_url) + .load() + .await; + + let s3_config = aws_sdk_s3::config::Builder::from(&shared_config) + .force_path_style(true) + .build(); + + S3Client::from_conf(s3_config) +} + +/// Start LocalStack, create the test bucket, clean it, configure DuckDB env vars, +/// and return the endpoint URL and a ready-to-use S3 client. +pub async fn setup_test_bucket() -> (String, S3Client, DuckDbEnvironmentGuard) { + initialize_test_tracing(); + + let endpoint = get_localstack_endpoint().await; + let s3_client = create_test_s3_client(&endpoint).await; + + // Create bucket (ignore AlreadyExists / BucketAlreadyOwnedByYou) + let _ = s3_client.create_bucket().bucket(TEST_BUCKET).send().await; + + clean_bucket(&s3_client).await; + + let host_port = endpoint + .strip_prefix("http://") + .unwrap_or(&endpoint) + .to_string(); + let env_guard = DuckDbEnvironmentGuard::new(&host_port); + + (endpoint, s3_client, env_guard) +} + +pub async fn clean_bucket(s3_client: &S3Client) { + let mut continuation_token: Option = None; + + loop { + let mut request = s3_client.list_objects_v2().bucket(TEST_BUCKET); + if let Some(token) = &continuation_token { + request = request.continuation_token(token); + } + + let output = match request.send().await { + Ok(output) => output, + Err(_) => break, + }; + + let contents = output.contents(); + for object in contents { + if let Some(key) = object.key() { + let _ = s3_client + .delete_object() + .bucket(TEST_BUCKET) + .key(key) + .send() + .await; + } + } + + if output.is_truncated() == Some(true) { + continuation_token = output.next_continuation_token().map(|s| s.to_string()); + } else { + break; + } + } +} + +pub async fn put_test_object(s3_client: &S3Client, key: &str, bytes: Vec) { + s3_client + .put_object() + .bucket(TEST_BUCKET) + .key(key) + .body(ByteStream::from(bytes)) + .send() + .await + .expect("Failed to put test object"); +} + +pub fn test_bucket_name() -> String { + TEST_BUCKET.to_string() +} + +pub struct SpawnedAppServer { + pub base_url: String, + shutdown_sender: Option>, + server_handle: Option>, +} + +impl SpawnedAppServer { + pub async fn start(app: Router) -> Self { + initialize_test_tracing(); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_address = listener.local_addr().unwrap(); + let base_url = format!("http://{}", local_address); + + let (shutdown_sender, shutdown_receiver) = oneshot::channel::<()>(); + + let server_handle = tokio::spawn(async move { + let server = axum::serve(listener, app); + tokio::select! { + _ = server => {} + _ = shutdown_receiver => {} + } + }); + + wait_for_server_start(local_address).await; + + Self { + base_url, + shutdown_sender: Some(shutdown_sender), + server_handle: Some(server_handle), + } + } + + pub fn url(&self, path: &str) -> String { + if path.starts_with('/') { + format!("{}{}", self.base_url, path) + } else { + format!("{}/{}", self.base_url, path) + } + } +} + +impl Drop for SpawnedAppServer { + fn drop(&mut self) { + if let Some(shutdown_sender) = self.shutdown_sender.take() { + let _ = shutdown_sender.send(()); + } + + if let Some(server_handle) = self.server_handle.take() { + server_handle.abort(); + } + } +} + +async fn wait_for_server_start(address: SocketAddr) { + for _ in 0..50 { + if tokio::net::TcpStream::connect(address).await.is_ok() { + return; + } + + tokio::time::sleep(Duration::from_millis(10)).await; + } + + panic!("Server did not start listening on {}", address); +} diff --git a/applications/datamanager/tests/test_data.rs b/applications/datamanager/tests/test_data.rs new file mode 100644 index 000000000..5ba2e7339 --- /dev/null +++ b/applications/datamanager/tests/test_data.rs @@ -0,0 +1,677 @@ +mod common; + +use common::initialize_test_tracing; +use datamanager::data::{ + create_equity_bar_dataframe, create_equity_details_dataframe, create_portfolio_dataframe, + create_predictions_dataframe, EquityBar, Portfolio, Prediction, +}; +use polars::prelude::*; + +#[allow(dead_code)] +fn sample_equity_bar() -> EquityBar { + EquityBar { + ticker: "AAPL".to_string(), + timestamp: 1234567890, + open_price: Some(100.0), + high_price: Some(105.0), + low_price: Some(99.0), + close_price: Some(103.0), + volume: Some(1000000.0), + volume_weighted_average_price: Some(102.0), + transactions: Some(5000), + } +} + +#[allow(dead_code)] +fn sample_equity_bar_lowercase() -> EquityBar { + EquityBar { + ticker: "googl".to_string(), + timestamp: 1234567890, + open_price: Some(2000.0), + high_price: Some(2050.0), + low_price: Some(1990.0), + close_price: Some(2030.0), + volume: Some(500000.0), + volume_weighted_average_price: Some(2020.0), + transactions: Some(2500), + } +} + +#[allow(dead_code)] +fn sample_prediction() -> Prediction { + Prediction { + ticker: "AAPL".to_string(), + timestamp: 1234567890, + quantile_10: 95.0, + quantile_50: 100.0, + quantile_90: 105.0, + } +} + +#[allow(dead_code)] +fn sample_prediction_with_timestamp(timestamp: i64) -> Prediction { + Prediction { + ticker: "AAPL".to_string(), + timestamp, + quantile_10: 95.0, + quantile_50: 100.0, + quantile_90: 105.0, + } +} + +#[allow(dead_code)] +fn sample_portfolio() -> Portfolio { + Portfolio { + ticker: "AAPL".to_string(), + timestamp: 1234567890.0, + side: "long".to_string(), + dollar_amount: 10000.0, + action: "hold".to_string(), + } +} + +#[allow(dead_code)] +fn sample_portfolio_lowercase() -> Portfolio { + Portfolio { + ticker: "aapl".to_string(), + timestamp: 1234567890.0, + side: "short".to_string(), + dollar_amount: 5000.0, + action: "sell".to_string(), + } +} + +#[test] +fn test_create_equity_bar_dataframe_valid_data() { + initialize_test_tracing(); + let bars = vec![sample_equity_bar()]; + + let df = create_equity_bar_dataframe(bars).unwrap(); + + assert_eq!(df.height(), 1); + assert_eq!(df.width(), 9); + assert!(df.column("ticker").is_ok()); + assert!(df.column("timestamp").is_ok()); + assert!(df.column("open_price").is_ok()); + assert!(df.column("high_price").is_ok()); + assert!(df.column("low_price").is_ok()); + assert!(df.column("close_price").is_ok()); + assert!(df.column("volume").is_ok()); + assert!(df.column("volume_weighted_average_price").is_ok()); + assert!(df.column("transactions").is_ok()); +} + +#[test] +fn test_create_equity_bar_dataframe_uppercase_normalization() { + initialize_test_tracing(); + let bars = vec![sample_equity_bar_lowercase()]; + + let df = create_equity_bar_dataframe(bars).unwrap(); + + let ticker = df.column("ticker").unwrap().str().unwrap().get(0).unwrap(); + + assert_eq!(ticker, "GOOGL"); +} + +#[test] +fn test_create_equity_bar_dataframe_mixed_case_tickers() { + initialize_test_tracing(); + let bars = vec![sample_equity_bar(), sample_equity_bar_lowercase()]; + + let df = create_equity_bar_dataframe(bars).unwrap(); + + assert_eq!(df.height(), 2); + + let tickers = df + .column("ticker") + .unwrap() + .str() + .unwrap() + .into_iter() + .map(|t| t.unwrap()) + .collect::>(); + + assert_eq!(tickers, vec!["AAPL", "GOOGL"]); +} + +#[test] +fn test_create_equity_bar_dataframe_empty_vec() { + initialize_test_tracing(); + let bars: Vec = vec![]; + + let df = create_equity_bar_dataframe(bars).unwrap(); + + assert_eq!(df.height(), 0); + assert_eq!(df.width(), 9); +} + +#[test] +fn test_create_equity_bar_dataframe_with_none_prices() { + initialize_test_tracing(); + let bars = vec![EquityBar { + ticker: "TEST".to_string(), + timestamp: 1234567890, + open_price: None, + high_price: None, + low_price: None, + close_price: None, + volume: None, + volume_weighted_average_price: None, + transactions: None, + }]; + + let df = create_equity_bar_dataframe(bars).unwrap(); + + assert_eq!(df.height(), 1); + + let close_price = df.column("close_price").unwrap(); + assert_eq!(close_price.len(), 1); +} + +#[test] +fn test_create_equity_bar_dataframe_multiple_rows() { + initialize_test_tracing(); + let bars = vec![ + sample_equity_bar(), + sample_equity_bar(), + sample_equity_bar(), + ]; + + let df = create_equity_bar_dataframe(bars).unwrap(); + + assert_eq!(df.height(), 3); + assert_eq!(df.width(), 9); +} + +#[test] +fn test_create_predictions_dataframe_valid_data() { + initialize_test_tracing(); + let predictions = vec![sample_prediction()]; + + let df = create_predictions_dataframe(predictions).unwrap(); + + assert_eq!(df.height(), 1); + assert_eq!(df.width(), 5); + assert!(df.column("ticker").is_ok()); + assert!(df.column("timestamp").is_ok()); + assert!(df.column("quantile_10").is_ok()); + assert!(df.column("quantile_50").is_ok()); + assert!(df.column("quantile_90").is_ok()); +} + +#[test] +fn test_create_predictions_dataframe_uppercase_normalization() { + initialize_test_tracing(); + let predictions = vec![Prediction { + ticker: "aapl".to_string(), + timestamp: 1234567890, + quantile_10: 95.0, + quantile_50: 100.0, + quantile_90: 105.0, + }]; + + let df = create_predictions_dataframe(predictions).unwrap(); + + let ticker = df.column("ticker").unwrap().str().unwrap().get(0).unwrap(); + + assert_eq!(ticker, "AAPL"); +} + +#[test] +fn test_create_predictions_dataframe_deduplication() { + initialize_test_tracing(); + let predictions = vec![ + sample_prediction_with_timestamp(1000), + sample_prediction_with_timestamp(2000), + sample_prediction_with_timestamp(3000), + ]; + + let df = create_predictions_dataframe(predictions).unwrap(); + + assert_eq!(df.height(), 1); + + let timestamp = df.column("timestamp").unwrap().i64().unwrap().get(0); + assert_eq!(timestamp, Some(3000)); +} + +#[test] +fn test_create_predictions_dataframe_keeps_most_recent_per_ticker() { + initialize_test_tracing(); + let predictions = vec![ + Prediction { + ticker: "AAPL".to_string(), + timestamp: 1000, + quantile_10: 90.0, + quantile_50: 95.0, + quantile_90: 100.0, + }, + Prediction { + ticker: "AAPL".to_string(), + timestamp: 2000, + quantile_10: 95.0, + quantile_50: 100.0, + quantile_90: 105.0, + }, + Prediction { + ticker: "GOOGL".to_string(), + timestamp: 1500, + quantile_10: 1990.0, + quantile_50: 2000.0, + quantile_90: 2010.0, + }, + ]; + + let df = create_predictions_dataframe(predictions).unwrap(); + + assert_eq!(df.height(), 2); + + let aapl_rows = df + .clone() + .lazy() + .filter(polars::prelude::col("ticker").eq(polars::prelude::lit("AAPL"))) + .collect() + .unwrap(); + assert_eq!(aapl_rows.height(), 1); + + let aapl_timestamp = aapl_rows.column("timestamp").unwrap().i64().unwrap().get(0); + assert_eq!(aapl_timestamp, Some(2000)); +} + +#[test] +fn test_create_predictions_dataframe_empty_vec() { + initialize_test_tracing(); + let predictions: Vec = vec![]; + + let df = create_predictions_dataframe(predictions).unwrap(); + + assert_eq!(df.height(), 0); + assert_eq!(df.width(), 5); +} + +#[test] +fn test_create_predictions_dataframe_multiple_different_tickers() { + initialize_test_tracing(); + let predictions = vec![ + Prediction { + ticker: "AAPL".to_string(), + timestamp: 1000, + quantile_10: 95.0, + quantile_50: 100.0, + quantile_90: 105.0, + }, + Prediction { + ticker: "GOOGL".to_string(), + timestamp: 1000, + quantile_10: 1995.0, + quantile_50: 2000.0, + quantile_90: 2005.0, + }, + Prediction { + ticker: "MSFT".to_string(), + timestamp: 1000, + quantile_10: 295.0, + quantile_50: 300.0, + quantile_90: 305.0, + }, + ]; + + let df = create_predictions_dataframe(predictions).unwrap(); + + assert_eq!(df.height(), 3); +} + +#[test] +fn test_create_portfolio_dataframe_valid_data() { + initialize_test_tracing(); + let portfolios = vec![sample_portfolio()]; + + let df = create_portfolio_dataframe(portfolios).unwrap(); + + assert_eq!(df.height(), 1); + assert_eq!(df.width(), 5); + assert!(df.column("ticker").is_ok()); + assert!(df.column("timestamp").is_ok()); + assert!(df.column("side").is_ok()); + assert!(df.column("dollar_amount").is_ok()); + assert!(df.column("action").is_ok()); +} + +#[test] +fn test_create_portfolio_dataframe_uppercase_normalization() { + initialize_test_tracing(); + let portfolios = vec![sample_portfolio_lowercase()]; + + let df = create_portfolio_dataframe(portfolios).unwrap(); + + let ticker = df.column("ticker").unwrap().str().unwrap().get(0).unwrap(); + assert_eq!(ticker, "AAPL"); + + let side = df.column("side").unwrap().str().unwrap().get(0).unwrap(); + assert_eq!(side, "SHORT"); + + let action = df.column("action").unwrap().str().unwrap().get(0).unwrap(); + assert_eq!(action, "SELL"); +} + +#[test] +fn test_create_portfolio_dataframe_mixed_case() { + initialize_test_tracing(); + let portfolios = vec![ + Portfolio { + ticker: "aapl".to_string(), + timestamp: 1234567890.0, + side: "long".to_string(), + dollar_amount: 10000.0, + action: "buy".to_string(), + }, + Portfolio { + ticker: "GOOGL".to_string(), + timestamp: 1234567890.0, + side: "SHORT".to_string(), + dollar_amount: 5000.0, + action: "Sell".to_string(), + }, + ]; + + let df = create_portfolio_dataframe(portfolios).unwrap(); + + assert_eq!(df.height(), 2); + + let tickers = df + .column("ticker") + .unwrap() + .str() + .unwrap() + .into_iter() + .map(|t| t.unwrap()) + .collect::>(); + assert_eq!(tickers, vec!["AAPL", "GOOGL"]); + + let sides = df + .column("side") + .unwrap() + .str() + .unwrap() + .into_iter() + .map(|s| s.unwrap()) + .collect::>(); + assert_eq!(sides, vec!["LONG", "SHORT"]); + + let actions = df + .column("action") + .unwrap() + .str() + .unwrap() + .into_iter() + .map(|a| a.unwrap()) + .collect::>(); + assert_eq!(actions, vec!["BUY", "SELL"]); +} + +#[test] +fn test_create_portfolio_dataframe_empty_vec() { + initialize_test_tracing(); + let portfolios: Vec = vec![]; + + let df = create_portfolio_dataframe(portfolios).unwrap(); + + assert_eq!(df.height(), 0); + assert_eq!(df.width(), 5); +} + +// Tests for create_equity_details_dataframe + +#[test] +fn test_create_equity_details_dataframe_valid_csv() { + initialize_test_tracing(); + let csv_content = "ticker,sector,industry\nAAPL,Technology,Consumer Electronics\nGOOGL,Technology,Internet Services\n"; + + let df = create_equity_details_dataframe(csv_content.to_string()).unwrap(); + + assert_eq!(df.height(), 2); + assert_eq!(df.width(), 3); + assert!(df.column("ticker").is_ok()); + assert!(df.column("sector").is_ok()); + assert!(df.column("industry").is_ok()); +} + +#[test] +fn test_create_equity_details_dataframe_uppercase_normalization() { + initialize_test_tracing(); + let csv_content = "ticker,sector,industry\naapl,technology,consumer electronics\n"; + + let df = create_equity_details_dataframe(csv_content.to_string()).unwrap(); + + let ticker = df.column("ticker").unwrap().str().unwrap().get(0).unwrap(); + assert_eq!(ticker, "AAPL"); + + let sector = df.column("sector").unwrap().str().unwrap().get(0).unwrap(); + assert_eq!(sector, "TECHNOLOGY"); + + let industry = df + .column("industry") + .unwrap() + .str() + .unwrap() + .get(0) + .unwrap(); + assert_eq!(industry, "CONSUMER ELECTRONICS"); +} + +#[test] +fn test_create_equity_details_dataframe_with_nulls() { + initialize_test_tracing(); + let csv_content = "ticker,sector,industry\nAAPL,,\n"; + + let df = create_equity_details_dataframe(csv_content.to_string()).unwrap(); + + assert_eq!(df.height(), 1); + + let sector = df.column("sector").unwrap().str().unwrap().get(0).unwrap(); + assert_eq!(sector, "NOT AVAILABLE"); + + let industry = df + .column("industry") + .unwrap() + .str() + .unwrap() + .get(0) + .unwrap(); + assert_eq!(industry, "NOT AVAILABLE"); +} + +#[test] +fn test_create_equity_details_dataframe_extra_columns() { + initialize_test_tracing(); + let csv_content = + "ticker,sector,industry,extra_column\nAAPL,Technology,Consumer Electronics,Extra\n"; + + let df = create_equity_details_dataframe(csv_content.to_string()).unwrap(); + + assert_eq!(df.height(), 1); + assert_eq!(df.width(), 3); + assert!(df.column("extra_column").is_err()); +} + +#[test] +fn test_create_equity_details_dataframe_missing_ticker_column() { + initialize_test_tracing(); + let csv_content = "sector,industry\nTechnology,Consumer Electronics\n"; + + let result = create_equity_details_dataframe(csv_content.to_string()); + + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("missing required column")); +} + +#[test] +fn test_create_equity_details_dataframe_missing_sector_column() { + initialize_test_tracing(); + let csv_content = "ticker,industry\nAAPL,Consumer Electronics\n"; + + let result = create_equity_details_dataframe(csv_content.to_string()); + + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("missing required column")); +} + +#[test] +fn test_create_equity_details_dataframe_missing_industry_column() { + initialize_test_tracing(); + let csv_content = "ticker,sector\nAAPL,Technology\n"; + + let result = create_equity_details_dataframe(csv_content.to_string()); + + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("missing required column")); +} + +#[test] +fn test_create_equity_details_dataframe_empty_csv() { + initialize_test_tracing(); + let csv_content = "ticker,sector,industry\n"; + + let df = create_equity_details_dataframe(csv_content.to_string()).unwrap(); + + assert_eq!(df.height(), 0); + assert_eq!(df.width(), 3); +} + +#[test] +fn test_create_equity_details_dataframe_malformed_csv() { + initialize_test_tracing(); + let csv_content = + "ticker,sector,industry\nAAPL,Technology\nGOOGL,Technology,Internet Services,Extra\n"; + + let result = create_equity_details_dataframe(csv_content.to_string()); + + assert!(result.is_err()); + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("Polars") || err_msg.contains("parse") || err_msg.contains("column"), + "Expected parse error but got: {}", + err_msg + ); +} + +#[test] +fn test_equity_bar_dataframe_parquet_roundtrip() { + initialize_test_tracing(); + use std::io::Cursor; + + let original_bars = vec![sample_equity_bar()]; + let original_df = create_equity_bar_dataframe(original_bars.clone()).unwrap(); + + let mut buffer = Vec::new(); + ParquetWriter::new(&mut buffer) + .finish(&mut original_df.clone()) + .unwrap(); + + let cursor = Cursor::new(buffer); + let deserialized_df = ParquetReader::new(cursor).finish().unwrap(); + + assert_eq!(deserialized_df.width(), 9); + assert_eq!(deserialized_df.height(), 1); + + assert!(deserialized_df.column("ticker").is_ok()); + assert!(deserialized_df.column("timestamp").is_ok()); + assert!(deserialized_df.column("open_price").is_ok()); + assert!(deserialized_df.column("high_price").is_ok()); + assert!(deserialized_df.column("low_price").is_ok()); + assert!(deserialized_df.column("close_price").is_ok()); + assert!(deserialized_df.column("volume").is_ok()); + assert!(deserialized_df + .column("volume_weighted_average_price") + .is_ok()); + assert!(deserialized_df.column("transactions").is_ok()); + + let ticker_series = deserialized_df.column("ticker").unwrap(); + assert_eq!(ticker_series.str().unwrap().get(0).unwrap(), "AAPL"); +} + +#[test] +fn test_predictions_dataframe_parquet_roundtrip() { + initialize_test_tracing(); + use std::io::Cursor; + + let original_predictions = vec![sample_prediction()]; + let original_df = create_predictions_dataframe(original_predictions.clone()).unwrap(); + + let mut buffer = Vec::new(); + ParquetWriter::new(&mut buffer) + .finish(&mut original_df.clone()) + .unwrap(); + + let cursor = Cursor::new(buffer); + let deserialized_df = ParquetReader::new(cursor).finish().unwrap(); + + assert_eq!(deserialized_df.width(), 5); + assert_eq!(deserialized_df.height(), 1); + + assert!(deserialized_df.column("ticker").is_ok()); + assert!(deserialized_df.column("timestamp").is_ok()); + assert!(deserialized_df.column("quantile_10").is_ok()); + assert!(deserialized_df.column("quantile_50").is_ok()); + assert!(deserialized_df.column("quantile_90").is_ok()); + + let ticker_series = deserialized_df.column("ticker").unwrap(); + assert_eq!(ticker_series.str().unwrap().get(0).unwrap(), "AAPL"); +} + +#[test] +fn test_portfolio_dataframe_parquet_roundtrip() { + initialize_test_tracing(); + use std::io::Cursor; + + let original_portfolios = vec![sample_portfolio()]; + let original_df = create_portfolio_dataframe(original_portfolios.clone()).unwrap(); + + let mut buffer = Vec::new(); + ParquetWriter::new(&mut buffer) + .finish(&mut original_df.clone()) + .unwrap(); + + let cursor = Cursor::new(buffer); + let deserialized_df = ParquetReader::new(cursor).finish().unwrap(); + + assert_eq!(deserialized_df.width(), 5); + assert_eq!(deserialized_df.height(), 1); + + assert!(deserialized_df.column("ticker").is_ok()); + assert!(deserialized_df.column("timestamp").is_ok()); + assert!(deserialized_df.column("side").is_ok()); + assert!(deserialized_df.column("dollar_amount").is_ok()); + assert!(deserialized_df.column("action").is_ok()); + + let ticker_series = deserialized_df.column("ticker").unwrap(); + assert_eq!(ticker_series.str().unwrap().get(0).unwrap(), "AAPL"); +} + +#[test] +fn test_parquet_empty_dataframe_roundtrip() { + initialize_test_tracing(); + use std::io::Cursor; + + let empty_bars: Vec = vec![]; + let original_df = create_equity_bar_dataframe(empty_bars).unwrap(); + + let mut buffer = Vec::new(); + ParquetWriter::new(&mut buffer) + .finish(&mut original_df.clone()) + .unwrap(); + + let cursor = Cursor::new(buffer); + let deserialized_df = ParquetReader::new(cursor).finish().unwrap(); + + assert_eq!(deserialized_df.width(), 9); + assert_eq!(deserialized_df.height(), 0); +} diff --git a/applications/datamanager/tests/test_errors.rs b/applications/datamanager/tests/test_errors.rs new file mode 100644 index 000000000..8fa7552b4 --- /dev/null +++ b/applications/datamanager/tests/test_errors.rs @@ -0,0 +1,15 @@ +use datamanager::errors::Error; + +#[test] +fn test_error_display_formats_messages() { + let other_error = Error::Other("example message".to_string()); + assert_eq!(other_error.to_string(), "Other error: example message"); + + let connection = duckdb::Connection::open_in_memory().unwrap(); + let duckdb_error = connection + .execute_batch("SELECT * FROM missing_table") + .unwrap_err(); + let wrapped_error = Error::DuckDB(duckdb_error); + + assert!(wrapped_error.to_string().contains("DuckDB error")); +} diff --git a/applications/datamanager/tests/test_handlers.rs b/applications/datamanager/tests/test_handlers.rs new file mode 100644 index 000000000..d2ab22f1d --- /dev/null +++ b/applications/datamanager/tests/test_handlers.rs @@ -0,0 +1,720 @@ +mod common; + +use datamanager::{ + router::create_app_with_state, + state::{MassiveSecrets, State}, +}; +use mockito::{Matcher, Server}; +use polars::prelude::*; +use reqwest::StatusCode; +use serial_test::serial; + +use common::{ + create_test_s3_client, put_test_object, setup_test_bucket, test_bucket_name, + DuckDbEnvironmentGuard, EnvironmentVariableGuard, SpawnedAppServer, +}; + +async fn spawn_app( + endpoint: &str, + massive_base: String, +) -> (SpawnedAppServer, EnvironmentVariableGuard) { + let env_guard = EnvironmentVariableGuard::set("MASSIVE_API_KEY", "test-api-key"); + + let s3_client = create_test_s3_client(endpoint).await; + let state = State::new( + reqwest::Client::new(), + MassiveSecrets { + base: massive_base, + key: std::env::var("MASSIVE_API_KEY").unwrap(), + }, + s3_client, + test_bucket_name(), + ); + let app = create_app_with_state(state); + (SpawnedAppServer::start(app).await, env_guard) +} + +async fn spawn_app_with_unreachable_s3( + massive_base: String, +) -> (SpawnedAppServer, DuckDbEnvironmentGuard) { + // Point DuckDB env vars to the same unreachable endpoint so that + // DuckDB's httpfs also fails (DuckDB reads credentials and endpoint from env). + let env_guard = DuckDbEnvironmentGuard::new("127.0.0.1:9"); + + let unreachable_s3_client = create_test_s3_client("http://127.0.0.1:9").await; + let state = State::new( + reqwest::Client::new(), + MassiveSecrets { + base: massive_base, + key: "test-api-key".to_string(), + }, + unreachable_s3_client, + "test-bucket".to_string(), + ); + let app = create_app_with_state(state); + (SpawnedAppServer::start(app).await, env_guard) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_predictions_save_and_query_round_trip() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let (app, _env_guard) = spawn_app(&endpoint, "http://127.0.0.1:1".to_string()).await; + let client = reqwest::Client::new(); + + let save_payload = r#"{ + "data": [{ + "ticker": "AAPL", + "timestamp": 1735689600, + "quantile_10": 190.0, + "quantile_50": 200.0, + "quantile_90": 210.0 + }], + "timestamp": "2025-01-01T00:00:00Z" + }"#; + + let response = client + .post(app.url("/predictions")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(save_payload) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let encoded_query = urlencoding::encode("[{\"ticker\":\"AAPL\",\"timestamp\":1735689600.0}]"); + let response = client + .get(app.url(&format!( + "/predictions?tickers_and_timestamps={}", + encoded_query + ))) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = response.text().await.unwrap(); + assert!(body.contains("AAPL")); + assert!(body.contains("quantile_50")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_predictions_save_returns_internal_server_error_when_s3_upload_fails() { + let (app, _env_guard) = spawn_app_with_unreachable_s3("http://127.0.0.1:1".to_string()).await; + + let save_payload = r#"{ + "data": [{ + "ticker": "AAPL", + "timestamp": 1735689600, + "quantile_10": 190.0, + "quantile_50": 200.0, + "quantile_90": 210.0 + }], + "timestamp": "2025-01-01T00:00:00Z" + }"#; + + let response = reqwest::Client::new() + .post(app.url("/predictions")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(save_payload) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_predictions_query_returns_bad_request_for_invalid_url_encoding() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let (app, _env_guard) = spawn_app(&endpoint, "http://127.0.0.1:1".to_string()).await; + + let response = reqwest::Client::new() + .get(app.url("/predictions?tickers_and_timestamps=%")) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_predictions_query_returns_bad_request_for_invalid_json() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let (app, _env_guard) = spawn_app(&endpoint, "http://127.0.0.1:1".to_string()).await; + + let encoded = urlencoding::encode("not-json"); + let response = reqwest::Client::new() + .get(app.url(&format!("/predictions?tickers_and_timestamps={}", encoded))) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_predictions_query_returns_empty_json_array_when_no_rows_match() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let (app, _env_guard) = spawn_app(&endpoint, "http://127.0.0.1:1".to_string()).await; + let client = reqwest::Client::new(); + + let save_payload = r#"{ + "data": [{ + "ticker": "AAPL", + "timestamp": 1735689600, + "quantile_10": 190.0, + "quantile_50": 200.0, + "quantile_90": 210.0 + }], + "timestamp": "2025-01-01T00:00:00Z" + }"#; + + let response = client + .post(app.url("/predictions")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(save_payload) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let encoded = urlencoding::encode("[{\"ticker\":\"MSFT\",\"timestamp\":1735689600.0}]"); + let response = client + .get(app.url(&format!("/predictions?tickers_and_timestamps={}", encoded))) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.text().await.unwrap(), "[]"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_predictions_query_returns_internal_server_error_when_storage_query_fails() { + let (app, _env_guard) = spawn_app_with_unreachable_s3("http://127.0.0.1:1".to_string()).await; + + let encoded = urlencoding::encode("[{\"ticker\":\"AAPL\",\"timestamp\":1735689600.0}]"); + let response = reqwest::Client::new() + .get(app.url(&format!("/predictions?tickers_and_timestamps={}", encoded))) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_portfolios_save_and_get_round_trip() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let (app, _env_guard) = spawn_app(&endpoint, "http://127.0.0.1:1".to_string()).await; + let client = reqwest::Client::new(); + + let save_payload = r#"{ + "data": [{ + "ticker": "AAPL", + "timestamp": 1735689600.0, + "side": "long", + "dollar_amount": 10000.0, + "action": "buy" + }], + "timestamp": "2025-01-01T00:00:00Z" + }"#; + + let response = client + .post(app.url("/portfolios")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(save_payload) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let response = client + .get(app.url("/portfolios?timestamp=2025-01-01T00:00:00Z")) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = response.text().await.unwrap(); + assert!(body.contains("AAPL")); + assert!(body.contains("BUY")); + + let response = client.get(app.url("/portfolios")).send().await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_portfolios_save_returns_internal_server_error_when_s3_upload_fails() { + let (app, _env_guard) = spawn_app_with_unreachable_s3("http://127.0.0.1:1".to_string()).await; + + let save_payload = r#"{ + "data": [{ + "ticker": "AAPL", + "timestamp": 1735689600.0, + "side": "long", + "dollar_amount": 10000.0, + "action": "buy" + }], + "timestamp": "2025-01-01T00:00:00Z" + }"#; + + let response = reqwest::Client::new() + .post(app.url("/portfolios")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(save_payload) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_portfolios_get_returns_not_found_for_first_run_without_files() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let (app, _env_guard) = spawn_app(&endpoint, "http://127.0.0.1:1".to_string()).await; + + let response = reqwest::Client::new() + .get(app.url("/portfolios")) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_portfolios_get_returns_not_found_when_portfolio_file_is_empty() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let (app, _env_guard) = spawn_app(&endpoint, "http://127.0.0.1:1".to_string()).await; + let client = reqwest::Client::new(); + + let empty_save_payload = r#"{ + "data": [], + "timestamp": "2025-01-01T00:00:00Z" + }"#; + + let response = client + .post(app.url("/portfolios")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(empty_save_payload) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let response = client + .get(app.url("/portfolios?timestamp=2025-01-01T00:00:00Z")) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_details_get_returns_csv_content() { + let (endpoint, s3, _env_guard) = setup_test_bucket().await; + + put_test_object( + &s3, + "equity/details/categories.csv", + b"ticker,sector,industry\nAAPL,Technology,Consumer Electronics\n".to_vec(), + ) + .await; + + let (app, _env_guard) = spawn_app(&endpoint, "http://127.0.0.1:1".to_string()).await; + + let response = reqwest::Client::new() + .get(app.url("/equity-details")) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .unwrap() + .to_str() + .unwrap(); + assert!(content_type.contains("text/csv")); + + let body = response.text().await.unwrap(); + assert!(body.contains("AAPL")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_details_get_returns_internal_server_error_when_csv_is_missing() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let (app, _env_guard) = spawn_app(&endpoint, "http://127.0.0.1:1".to_string()).await; + + let response = reqwest::Client::new() + .get(app.url("/equity-details")) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_bars_sync_and_query_round_trip() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + + let mut massive_server = Server::new_async().await; + let _mock = massive_server + .mock("GET", "/v2/aggs/grouped/locale/us/market/stocks/2025-01-01") + .match_query(Matcher::AllOf(vec![ + Matcher::UrlEncoded("adjusted".into(), "true".into()), + Matcher::UrlEncoded("apiKey".into(), "test-api-key".into()), + ])) + .with_status(200) + .with_body( + r#"{ + "adjusted": true, + "queryCount": 1, + "request_id": "test", + "resultsCount": 1, + "status": "OK", + "results": [{ + "T": "AAPL", + "c": 105.0, + "h": 110.0, + "l": 99.0, + "n": 1000, + "o": 100.0, + "t": 1735689600, + "v": 2000000.0, + "vw": 104.0 + }] + }"#, + ) + .create_async() + .await; + + let (app, _env_guard) = spawn_app(&endpoint, massive_server.url()).await; + let client = reqwest::Client::new(); + + let response = client + .post(app.url("/equity-bars")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(r#"{"date":"2025-01-01T00:00:00Z"}"#) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let response = client + .get(app.url( + "/equity-bars?tickers=AAPL&start_timestamp=2025-01-01T00:00:00Z&end_timestamp=2025-01-01T00:00:00Z", + )) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .unwrap() + .to_str() + .unwrap(); + assert_eq!(content_type, "application/octet-stream"); + + let body = response.bytes().await.unwrap(); + let dataframe = ParquetReader::new(std::io::Cursor::new(body.to_vec())) + .finish() + .unwrap(); + assert_eq!(dataframe.height(), 1); + assert_eq!( + dataframe.column("ticker").unwrap().str().unwrap().get(0), + Some("AAPL") + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_bars_sync_returns_no_content_when_api_has_no_results() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + + let mut massive_server = Server::new_async().await; + let _mock = massive_server + .mock("GET", "/v2/aggs/grouped/locale/us/market/stocks/2025-01-01") + .match_query(Matcher::AllOf(vec![ + Matcher::UrlEncoded("adjusted".into(), "true".into()), + Matcher::UrlEncoded("apiKey".into(), "test-api-key".into()), + ])) + .with_status(200) + .with_body(r#"{"status":"OK","resultsCount":0}"#) + .create_async() + .await; + + let (app, _env_guard) = spawn_app(&endpoint, massive_server.url()).await; + + let response = reqwest::Client::new() + .post(app.url("/equity-bars")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(r#"{"date":"2025-01-01T00:00:00Z"}"#) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NO_CONTENT); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_bars_sync_returns_internal_server_error_for_invalid_json() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + + let mut massive_server = Server::new_async().await; + let _mock = massive_server + .mock("GET", "/v2/aggs/grouped/locale/us/market/stocks/2025-01-01") + .match_query(Matcher::AllOf(vec![ + Matcher::UrlEncoded("adjusted".into(), "true".into()), + Matcher::UrlEncoded("apiKey".into(), "test-api-key".into()), + ])) + .with_status(200) + .with_body("not-json") + .create_async() + .await; + + let (app, _env_guard) = spawn_app(&endpoint, massive_server.url()).await; + + let response = reqwest::Client::new() + .post(app.url("/equity-bars")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(r#"{"date":"2025-01-01T00:00:00Z"}"#) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_bars_sync_returns_bad_gateway_for_unparseable_results() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + + let mut massive_server = Server::new_async().await; + let _mock = massive_server + .mock("GET", "/v2/aggs/grouped/locale/us/market/stocks/2025-01-01") + .match_query(Matcher::AllOf(vec![ + Matcher::UrlEncoded("adjusted".into(), "true".into()), + Matcher::UrlEncoded("apiKey".into(), "test-api-key".into()), + ])) + .with_status(200) + .with_body( + r#"{ + "status": "OK", + "results": [{"T":"AAPL"}] + }"#, + ) + .create_async() + .await; + + let (app, _env_guard) = spawn_app(&endpoint, massive_server.url()).await; + + let response = reqwest::Client::new() + .post(app.url("/equity-bars")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(r#"{"date":"2025-01-01T00:00:00Z"}"#) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_bars_sync_returns_internal_server_error_when_api_request_fails() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let (app, _env_guard) = spawn_app(&endpoint, "http://127.0.0.1:1".to_string()).await; + + let response = reqwest::Client::new() + .post(app.url("/equity-bars")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(r#"{"date":"2025-01-01T00:00:00Z"}"#) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_bars_query_returns_internal_server_error_for_invalid_ticker() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let (app, _env_guard) = spawn_app(&endpoint, "http://127.0.0.1:1".to_string()).await; + + let response = reqwest::Client::new() + .get(app.url( + "/equity-bars?tickers=AAPL;DROP&start_timestamp=2025-01-01T00:00:00Z&end_timestamp=2025-01-01T00:00:00Z", + )) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_bars_query_without_ticker_filter_returns_data() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + + let mut massive_server = Server::new_async().await; + let _mock = massive_server + .mock("GET", "/v2/aggs/grouped/locale/us/market/stocks/2025-01-01") + .match_query(Matcher::AllOf(vec![ + Matcher::UrlEncoded("adjusted".into(), "true".into()), + Matcher::UrlEncoded("apiKey".into(), "test-api-key".into()), + ])) + .with_status(200) + .with_body(r#"{"adjusted":true,"queryCount":1,"request_id":"t","resultsCount":1,"status":"OK","results":[{"T":"AAPL","c":105.0,"h":110.0,"l":99.0,"n":1000,"o":100.0,"t":1735689600,"v":2000000.0,"vw":104.0}]}"#) + .create_async() + .await; + + let (app, _env_guard) = spawn_app(&endpoint, massive_server.url()).await; + let client = reqwest::Client::new(); + + let response = client + .post(app.url("/equity-bars")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(r#"{"date":"2025-01-01T00:00:00Z"}"#) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + // Query without tickers — covers "No tickers specified" branch + let response = client + .get(app.url( + "/equity-bars?start_timestamp=2025-01-01T00:00:00Z&end_timestamp=2025-01-01T00:00:00Z", + )) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = response.bytes().await.unwrap(); + let dataframe = ParquetReader::new(std::io::Cursor::new(body.to_vec())) + .finish() + .unwrap(); + assert_eq!(dataframe.height(), 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_bars_query_with_empty_tickers_param_returns_data() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + + let mut massive_server = Server::new_async().await; + let _mock = massive_server + .mock("GET", "/v2/aggs/grouped/locale/us/market/stocks/2025-01-01") + .match_query(Matcher::AllOf(vec![ + Matcher::UrlEncoded("adjusted".into(), "true".into()), + Matcher::UrlEncoded("apiKey".into(), "test-api-key".into()), + ])) + .with_status(200) + .with_body(r#"{"adjusted":true,"queryCount":1,"request_id":"t","resultsCount":1,"status":"OK","results":[{"T":"AAPL","c":105.0,"h":110.0,"l":99.0,"n":1000,"o":100.0,"t":1735689600,"v":2000000.0,"vw":104.0}]}"#) + .create_async() + .await; + + let (app, _env_guard) = spawn_app(&endpoint, massive_server.url()).await; + let client = reqwest::Client::new(); + + let response = client + .post(app.url("/equity-bars")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(r#"{"date":"2025-01-01T00:00:00Z"}"#) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + // Query with empty tickers — covers "Ticker list was empty" branch + let response = client + .get(app.url( + "/equity-bars?tickers=&start_timestamp=2025-01-01T00:00:00Z&end_timestamp=2025-01-01T00:00:00Z", + )) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_bars_sync_returns_internal_server_error_for_api_error_status() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + + let mut massive_server = Server::new_async().await; + let _mock = massive_server + .mock("GET", "/v2/aggs/grouped/locale/us/market/stocks/2025-01-01") + .match_query(Matcher::AllOf(vec![ + Matcher::UrlEncoded("adjusted".into(), "true".into()), + Matcher::UrlEncoded("apiKey".into(), "test-api-key".into()), + ])) + .with_status(500) + .with_body("Internal Server Error") + .create_async() + .await; + + let (app, _env_guard) = spawn_app(&endpoint, massive_server.url()).await; + + let response = reqwest::Client::new() + .post(app.url("/equity-bars")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(r#"{"date":"2025-01-01T00:00:00Z"}"#) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_equity_bars_sync_returns_bad_gateway_when_s3_upload_fails() { + let mut massive_server = Server::new_async().await; + let _mock = massive_server + .mock("GET", "/v2/aggs/grouped/locale/us/market/stocks/2025-01-01") + .match_query(Matcher::AllOf(vec![ + Matcher::UrlEncoded("adjusted".into(), "true".into()), + Matcher::UrlEncoded("apiKey".into(), "test-api-key".into()), + ])) + .with_status(200) + .with_body(r#"{"adjusted":true,"queryCount":1,"request_id":"t","resultsCount":1,"status":"OK","results":[{"T":"AAPL","c":105.0,"h":110.0,"l":99.0,"n":1000,"o":100.0,"t":1735689600,"v":2000000.0,"vw":104.0}]}"#) + .create_async() + .await; + + // Working Massive API but broken S3 → parse succeeds, upload fails + let (app, _env_guard) = spawn_app_with_unreachable_s3(massive_server.url()).await; + + let response = reqwest::Client::new() + .post(app.url("/equity-bars")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(r#"{"date":"2025-01-01T00:00:00Z"}"#) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_portfolios_get_returns_internal_server_error_when_storage_query_fails() { + // DuckDB connects to unreachable S3 → connection error (not "not found") + let (app, _env_guard) = spawn_app_with_unreachable_s3("http://127.0.0.1:1".to_string()).await; + + let response = reqwest::Client::new() + .get(app.url("/portfolios")) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); +} diff --git a/applications/datamanager/tests/test_state_and_health.rs b/applications/datamanager/tests/test_state_and_health.rs new file mode 100644 index 000000000..8f51ae2aa --- /dev/null +++ b/applications/datamanager/tests/test_state_and_health.rs @@ -0,0 +1,108 @@ +mod common; + +use datamanager::{ + router::create_app_with_state, + state::{MassiveSecrets, State}, +}; +use reqwest::StatusCode; +use serial_test::serial; + +use common::{ + create_test_s3_client, setup_test_bucket, test_bucket_name, EnvironmentVariableGuard, + SpawnedAppServer, +}; + +async fn create_state_for_endpoint(endpoint: &str, bucket_name: &str) -> State { + let s3_client = create_test_s3_client(endpoint).await; + + State::new( + reqwest::Client::new(), + MassiveSecrets { + base: "http://127.0.0.1:1".to_string(), + key: "test-key".to_string(), + }, + s3_client, + bucket_name.to_string(), + ) +} + +async fn spawn_server_for_state(state: State) -> SpawnedAppServer { + let app = create_app_with_state(state); + SpawnedAppServer::start(app).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_health_route_returns_ok() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state_for_endpoint(&endpoint, &test_bucket_name()).await; + let app_server = spawn_server_for_state(state).await; + + let response = reqwest::Client::new() + .get(app_server.url("/health")) + .send() + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_router_returns_not_found_for_unknown_route() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state_for_endpoint(&endpoint, &test_bucket_name()).await; + let app_server = spawn_server_for_state(state).await; + + let response = reqwest::Client::new() + .get(app_server.url("/missing")) + .send() + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_state_new_sets_all_fields() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state_for_endpoint(&endpoint, "custom-bucket").await; + + assert_eq!(state.massive.base, "http://127.0.0.1:1"); + assert_eq!(state.massive.key, "test-key"); + assert_eq!(state.bucket_name, "custom-bucket"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_state_from_env_uses_defaults_when_variables_are_missing() { + let _aws_bucket_guard = EnvironmentVariableGuard::remove("AWS_S3_DATA_BUCKET_NAME"); + let _massive_base_guard = EnvironmentVariableGuard::remove("MASSIVE_BASE_URL"); + let _massive_key_guard = EnvironmentVariableGuard::remove("MASSIVE_API_KEY"); + let _region_guard = EnvironmentVariableGuard::set("AWS_REGION", "us-east-1"); + let _metadata_guard = EnvironmentVariableGuard::set("AWS_EC2_METADATA_DISABLED", "true"); + + let state = State::from_env().await; + + assert_eq!(state.bucket_name, "oscm-data"); + assert_eq!(state.massive.base, "https://api.massive.io"); + assert!(state.massive.key.is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_state_from_env_uses_environment_values() { + let _aws_bucket_guard = EnvironmentVariableGuard::set("AWS_S3_DATA_BUCKET_NAME", "env-bucket"); + let _massive_base_guard = + EnvironmentVariableGuard::set("MASSIVE_BASE_URL", "https://massive.example"); + let _massive_key_guard = EnvironmentVariableGuard::set("MASSIVE_API_KEY", "env-api-key"); + let _region_guard = EnvironmentVariableGuard::set("AWS_REGION", "us-east-1"); + let _metadata_guard = EnvironmentVariableGuard::set("AWS_EC2_METADATA_DISABLED", "true"); + + let state = State::from_env().await; + + assert_eq!(state.bucket_name, "env-bucket"); + assert_eq!(state.massive.base, "https://massive.example"); + assert_eq!(state.massive.key, "env-api-key"); +} diff --git a/applications/datamanager/tests/test_storage.rs b/applications/datamanager/tests/test_storage.rs new file mode 100644 index 000000000..910ce4e04 --- /dev/null +++ b/applications/datamanager/tests/test_storage.rs @@ -0,0 +1,481 @@ +mod common; + +use chrono::{TimeZone, Utc}; +use datamanager::{ + data::{ + create_equity_bar_dataframe, create_portfolio_dataframe, create_predictions_dataframe, + EquityBar, Portfolio, Prediction, + }, + state::{MassiveSecrets, State}, + storage::{ + date_to_int, escape_sql_ticker, format_s3_key, is_valid_ticker, + query_equity_bars_parquet_from_s3, query_portfolio_dataframe_from_s3, + query_predictions_dataframe_from_s3, read_equity_details_dataframe_from_s3, + sanitize_duckdb_config_value, write_equity_bars_dataframe_to_s3, + write_portfolio_dataframe_to_s3, write_predictions_dataframe_to_s3, PredictionQuery, + }, +}; +use polars::prelude::*; +use serial_test::serial; +use std::io::Cursor; + +use common::{create_test_s3_client, put_test_object, setup_test_bucket, test_bucket_name}; + +fn sample_prediction() -> Prediction { + Prediction { + ticker: "AAPL".to_string(), + timestamp: 1_735_689_600, + quantile_10: 190.0, + quantile_50: 200.0, + quantile_90: 210.0, + } +} + +fn sample_portfolio() -> Portfolio { + Portfolio { + ticker: "AAPL".to_string(), + timestamp: 1_735_689_600.0, + side: "LONG".to_string(), + dollar_amount: 10_000.0, + action: "BUY".to_string(), + } +} + +fn sample_equity_bar() -> EquityBar { + EquityBar { + ticker: "AAPL".to_string(), + timestamp: 1_735_689_600, + open_price: Some(100.0), + high_price: Some(110.0), + low_price: Some(99.0), + close_price: Some(105.0), + volume: Some(2_000_000.0), + volume_weighted_average_price: Some(104.0), + transactions: Some(1_000), + } +} + +fn fixed_date_time() -> chrono::DateTime { + Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).unwrap() +} + +async fn create_state(endpoint: &str) -> State { + let s3_client = create_test_s3_client(endpoint).await; + + State::new( + reqwest::Client::new(), + MassiveSecrets { + base: "http://127.0.0.1:1".to_string(), + key: "test-api-key".to_string(), + }, + s3_client, + test_bucket_name(), + ) +} + +#[test] +fn test_is_valid_ticker() { + assert!(is_valid_ticker("AAPL")); + assert!(is_valid_ticker("BRK.B")); + assert!(is_valid_ticker("BTC-USD")); + + assert!(!is_valid_ticker("")); + assert!(!is_valid_ticker("AAPL$")); + assert!(!is_valid_ticker("AAPL;DROP")); +} + +#[test] +fn test_format_s3_key() { + let timestamp = fixed_date_time(); + let key = format_s3_key(×tamp, "predictions"); + + assert_eq!( + key, + "equity/predictions/daily/year=2025/month=01/day=01/data.parquet" + ); +} + +#[test] +fn test_date_to_int() { + let timestamp = fixed_date_time(); + assert_eq!(date_to_int(×tamp).unwrap(), 20250101); +} + +#[test] +fn test_escape_sql_ticker() { + assert_eq!(escape_sql_ticker("AAPL"), "AAPL"); + assert_eq!(escape_sql_ticker("O'Reilly"), "O''Reilly"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_write_and_query_predictions_round_trip() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + let timestamp = fixed_date_time(); + + let predictions_dataframe = create_predictions_dataframe(vec![sample_prediction()]).unwrap(); + let s3_key = write_predictions_dataframe_to_s3(&state, &predictions_dataframe, ×tamp) + .await + .unwrap(); + + assert_eq!( + s3_key, + "equity/predictions/daily/year=2025/month=01/day=01/data.parquet" + ); + + let query_results = query_predictions_dataframe_from_s3( + &state, + vec![PredictionQuery { + ticker: "AAPL".to_string(), + timestamp: timestamp.timestamp() as f64, + }], + ) + .await + .unwrap(); + + assert_eq!(query_results.height(), 1); + assert_eq!( + query_results + .column("ticker") + .unwrap() + .str() + .unwrap() + .get(0), + Some("AAPL") + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_query_predictions_returns_empty_dataframe_when_no_rows_match() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + let timestamp = fixed_date_time(); + + let predictions_dataframe = create_predictions_dataframe(vec![sample_prediction()]).unwrap(); + write_predictions_dataframe_to_s3(&state, &predictions_dataframe, ×tamp) + .await + .unwrap(); + + let query_results = query_predictions_dataframe_from_s3( + &state, + vec![PredictionQuery { + ticker: "MSFT".to_string(), + timestamp: timestamp.timestamp() as f64, + }], + ) + .await + .unwrap(); + + assert_eq!(query_results.height(), 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_query_predictions_errors_when_query_positions_are_empty() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + + let result = query_predictions_dataframe_from_s3(&state, vec![]).await; + + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("No positions provided")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_write_and_query_portfolio_round_trip() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + let timestamp = fixed_date_time(); + + let portfolio_dataframe = create_portfolio_dataframe(vec![sample_portfolio()]).unwrap(); + write_portfolio_dataframe_to_s3(&state, &portfolio_dataframe, ×tamp) + .await + .unwrap(); + + let query_results = query_portfolio_dataframe_from_s3(&state, Some(timestamp)) + .await + .unwrap(); + + assert_eq!(query_results.height(), 1); + assert_eq!( + query_results + .column("ticker") + .unwrap() + .str() + .unwrap() + .get(0), + Some("AAPL") + ); + assert_eq!( + query_results + .column("action") + .unwrap() + .str() + .unwrap() + .get(0), + Some("BUY") + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_query_portfolio_without_timestamp_uses_latest_partition() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + + let old_timestamp = Utc.with_ymd_and_hms(2024, 12, 31, 0, 0, 0).unwrap(); + let new_timestamp = fixed_date_time(); + + let old_portfolio = Portfolio { + ticker: "OLD".to_string(), + ..sample_portfolio() + }; + let new_portfolio = Portfolio { + ticker: "NEW".to_string(), + ..sample_portfolio() + }; + + let old_dataframe = create_portfolio_dataframe(vec![old_portfolio]).unwrap(); + let new_dataframe = create_portfolio_dataframe(vec![new_portfolio]).unwrap(); + + write_portfolio_dataframe_to_s3(&state, &old_dataframe, &old_timestamp) + .await + .unwrap(); + write_portfolio_dataframe_to_s3(&state, &new_dataframe, &new_timestamp) + .await + .unwrap(); + + let query_results = query_portfolio_dataframe_from_s3(&state, None) + .await + .unwrap(); + + assert_eq!(query_results.height(), 1); + assert_eq!( + query_results + .column("ticker") + .unwrap() + .str() + .unwrap() + .get(0), + Some("NEW") + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_query_portfolio_falls_back_when_action_column_is_missing() { + let (endpoint, s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + let timestamp = fixed_date_time(); + + let key = format_s3_key(×tamp, "portfolios"); + + let mut dataframe = df!( + "ticker" => vec!["AAPL"], + "timestamp" => vec![1_735_689_600.0], + "side" => vec!["LONG"], + "dollar_amount" => vec![10_000.0], + ) + .unwrap(); + + let mut parquet_bytes = Vec::new(); + ParquetWriter::new(&mut parquet_bytes) + .finish(&mut dataframe) + .unwrap(); + + put_test_object(&s3, &key, parquet_bytes).await; + + let query_results = query_portfolio_dataframe_from_s3(&state, Some(timestamp)) + .await + .unwrap(); + + assert_eq!(query_results.height(), 1); + assert_eq!( + query_results + .column("action") + .unwrap() + .str() + .unwrap() + .get(0), + Some("UNSPECIFIED") + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_write_and_query_equity_bars_round_trip() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + let timestamp = fixed_date_time(); + + let bars_dataframe = create_equity_bar_dataframe(vec![sample_equity_bar()]).unwrap(); + write_equity_bars_dataframe_to_s3(&state, &bars_dataframe, ×tamp) + .await + .unwrap(); + + let parquet_bytes = query_equity_bars_parquet_from_s3( + &state, + Some(vec!["AAPL".to_string()]), + Some(timestamp), + Some(timestamp), + ) + .await + .unwrap(); + + let cursor = Cursor::new(parquet_bytes); + let result_dataframe = ParquetReader::new(cursor).finish().unwrap(); + + assert_eq!(result_dataframe.height(), 1); + assert_eq!( + result_dataframe + .column("ticker") + .unwrap() + .str() + .unwrap() + .get(0), + Some("AAPL") + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_query_equity_bars_rejects_invalid_ticker_format() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + let timestamp = fixed_date_time(); + + let result = query_equity_bars_parquet_from_s3( + &state, + Some(vec!["AAPL;DROP".to_string()]), + Some(timestamp), + Some(timestamp), + ) + .await; + + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("Invalid ticker format")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_read_equity_details_dataframe_from_s3_success() { + let (endpoint, s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + + put_test_object( + &s3, + "equity/details/categories.csv", + b"ticker,sector,industry\nAAPL,Technology,Consumer Electronics\n".to_vec(), + ) + .await; + + let dataframe = read_equity_details_dataframe_from_s3(&state).await.unwrap(); + + assert_eq!(dataframe.height(), 1); + assert_eq!( + dataframe.column("ticker").unwrap().str().unwrap().get(0), + Some("AAPL") + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_read_equity_details_dataframe_from_s3_returns_error_for_invalid_utf8() { + let (endpoint, s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + + put_test_object(&s3, "equity/details/categories.csv", vec![0xff, 0xfe, 0xfd]).await; + + let result = read_equity_details_dataframe_from_s3(&state).await; + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("UTF-8")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_query_equity_bars_without_date_range_uses_defaults() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + + // Use fixed date to avoid flakiness from midnight rollover + let test_date = fixed_date_time(); + let bars_dataframe = create_equity_bar_dataframe(vec![sample_equity_bar()]).unwrap(); + write_equity_bars_dataframe_to_s3(&state, &bars_dataframe, &test_date) + .await + .unwrap(); + + // Query with explicit date range around test_date to ensure deterministic results + let parquet_bytes = query_equity_bars_parquet_from_s3( + &state, + Some(vec!["AAPL".to_string()]), + Some(test_date - chrono::Duration::days(1)), + Some(test_date + chrono::Duration::days(1)), + ) + .await + .unwrap(); + + let cursor = Cursor::new(parquet_bytes); + let result = ParquetReader::new(cursor).finish().unwrap(); + assert!(result.height() >= 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn test_query_equity_bars_without_ticker_filter_returns_all() { + let (endpoint, _s3, _env_guard) = setup_test_bucket().await; + let state = create_state(&endpoint).await; + let timestamp = fixed_date_time(); + + let bars_dataframe = create_equity_bar_dataframe(vec![ + sample_equity_bar(), + EquityBar { + ticker: "GOOGL".to_string(), + ..sample_equity_bar() + }, + ]) + .unwrap(); + + write_equity_bars_dataframe_to_s3(&state, &bars_dataframe, ×tamp) + .await + .unwrap(); + + // Query with None tickers — covers "No ticker filter applied" path + let parquet_bytes = + query_equity_bars_parquet_from_s3(&state, None, Some(timestamp), Some(timestamp)) + .await + .unwrap(); + + let cursor = Cursor::new(parquet_bytes); + let result = ParquetReader::new(cursor).finish().unwrap(); + + assert_eq!(result.height(), 2); +} + +#[test] +fn test_sanitize_duckdb_config_value_valid() { + assert!(sanitize_duckdb_config_value("localhost:4566").is_ok()); + assert!(sanitize_duckdb_config_value("https://s3.amazonaws.com").is_ok()); + assert!(sanitize_duckdb_config_value("true").is_ok()); + assert!(sanitize_duckdb_config_value("false").is_ok()); + assert!(sanitize_duckdb_config_value("http://127.0.0.1:9000").is_ok()); +} + +#[test] +fn test_sanitize_duckdb_config_value_rejects_injection() { + assert!(sanitize_duckdb_config_value("'; DROP TABLE users; --").is_err()); + assert!(sanitize_duckdb_config_value("localhost'; --").is_err()); + assert!(sanitize_duckdb_config_value("\"malicious\"").is_err()); + assert!(sanitize_duckdb_config_value("").is_err()); + assert!(sanitize_duckdb_config_value(&"a".repeat(513)).is_err()); + assert!(sanitize_duckdb_config_value("value;another").is_err()); +} diff --git a/maskfile.md b/maskfile.md index 6f7f540f7..3d38cbb59 100644 --- a/maskfile.md +++ b/maskfile.md @@ -71,11 +71,12 @@ fi image_reference="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com/oscmcompany/${application_name}-${stage_name}" cache_reference="${image_reference}:buildcache" -# Use GHA backend for Cargo caching when running in GitHub Actions +# Use GHA backend for caching when running in GitHub Actions if [ -n "${GITHUB_ACTIONS:-}" ]; then - echo "Running in GitHub Actions - using hybrid cache (gha + registry)" - cache_from_arguments="--cache-from type=gha --cache-from type=registry,ref=${cache_reference}" - cache_to_arguments="--cache-to type=gha,mode=max --cache-to type=registry,ref=${cache_reference},mode=max" + scope="${application_name}-${stage_name}" + echo "Running in GitHub Actions - using hybrid cache (gha + registry) with scope: ${scope}" + cache_from_arguments="--cache-from type=gha,scope=${scope} --cache-from type=registry,ref=${cache_reference}" + cache_to_arguments="--cache-to type=gha,scope=${scope},mode=max --cache-to type=registry,ref=${cache_reference},mode=max" else echo "Running locally - using registry cache only" cache_from_arguments="--cache-from type=registry,ref=${cache_reference}" @@ -83,7 +84,11 @@ else fi echo "Setting up Docker Buildx" -docker buildx create --use --name psf-builder 2>/dev/null || docker buildx use psf-builder || (echo "Using default buildx builder" && docker buildx use default) +if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "Using buildx builder configured by docker/setup-buildx-action" +else + docker buildx create --use --name oscm-builder 2>/dev/null || docker buildx use oscm-builder || (echo "Using default buildx builder" && docker buildx use default) +fi echo "Logging into ECR (to pull cache if available)" aws ecr get-login-password --region ${aws_region} | docker login \ @@ -120,17 +125,37 @@ if [ -z "$aws_region" ]; then exit 1 fi -image_reference="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com/oscmcompany/${application_name}-${stage_name}" +repository_name="oscm/${application_name}-${stage_name}" +image_reference="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com/${repository_name}" +commit_hash=$(git rev-parse --short HEAD) echo "Logging into ECR" aws ecr get-login-password --region ${aws_region} | docker login \ --username AWS \ --password-stdin ${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com > /dev/null +echo "Checking if image for commit ${commit_hash} already exists in ECR" +existing_tag="NONE" +if image_digest=$(aws ecr describe-images \ + --repository-name "${repository_name}" \ + --image-ids "imageTag=git-${commit_hash}" \ + --query 'imageDetails[0].imageDigest' \ + --output text 2>/dev/null); then + existing_tag="${image_digest}" +fi + +if [ "$existing_tag" != "NONE" ] && [ "$existing_tag" != "None" ] && [ -n "$existing_tag" ]; then + echo "Image for commit ${commit_hash} already exists in ECR, skipping push" + echo "Image pushed: ${application_name} ${stage_name} (cached)" + exit 0 +fi + echo "Pushing image" -docker push ${image_reference}:latest +docker tag "${image_reference}:latest" "${image_reference}:git-${commit_hash}" +docker push "${image_reference}:latest" +docker push "${image_reference}:git-${commit_hash}" -echo "Image pushed: ${application_name} ${stage_name}" +echo "Image pushed: ${application_name} ${stage_name} (commit: ${commit_hash})" ``` ### stack @@ -363,16 +388,60 @@ echo "Rust linting completed successfully" #### test -> Run Rust tests +> Run Rust tests with coverage reporting ```bash set -euo pipefail -echo "Running Rust tests" +echo "Running Rust tests with coverage" + +echo "Checking Docker availability for integration tests" +if [[ "${RUN_INTEGRATION_TESTS:-0}" == "1" ]]; then + echo "RUN_INTEGRATION_TESTS=1 - enforcing Docker availability" + if command -v docker >/dev/null 2>&1; then + if ! docker info >/dev/null 2>&1; then + echo "Error: Docker is installed but daemon is not running" + echo "Integration tests requiring Docker will fail" + echo "Start Docker with: open -a Docker (macOS) or sudo systemctl start docker (Linux)" + exit 1 + fi + echo "Docker daemon is running" + else + echo "Error: Docker is not installed" + echo "Integration tests requiring Docker will fail" + echo "Install Docker from: https://docs.docker.com/get-docker/" + exit 1 + fi +else + if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + echo "Docker is available - integration tests can run" + else + echo "Warning: Docker is not available or daemon is not running" + echo "Integration tests requiring Docker may be skipped or fail" + echo "To enforce Docker for integration tests, set RUN_INTEGRATION_TESTS=1" + fi +fi -cargo test --workspace --verbose +mkdir -p .coverage_output -echo "Rust tests completed successfully" +if ! command -v cargo-llvm-cov >/dev/null 2>&1; then + echo "cargo-llvm-cov not available - running tests without coverage" + cargo test --workspace --verbose +elif ! command -v llvm-cov >/dev/null 2>&1 || ! command -v llvm-profdata >/dev/null 2>&1; then + echo "LLVM tools (llvm-cov or llvm-profdata) not available - running tests without coverage" + cargo test --workspace --verbose +else + export LLVM_COV=$(which llvm-cov) + export LLVM_PROFDATA=$(which llvm-profdata) + if cargo llvm-cov --workspace --verbose \ + --cobertura \ + --output-path .coverage_output/rust.xml; then + echo "Rust tests with coverage completed successfully" + else + echo "cargo llvm-cov failed - check test output above" + exit 1 + fi +fi ``` #### all @@ -384,7 +453,7 @@ set -euo pipefail echo "Running Rust development checks" -mask development rust update +# mask development rust update # Temporarily removing for continuous integration speed mask development rust check @@ -481,9 +550,14 @@ uvx ty check ```bash set -euo pipefail -echo "Running Python tests" +echo "Running Python tests with coverage" + +mkdir -p .coverage_output -uv run coverage run --parallel-mode -m pytest && uv run coverage combine && uv run coverage report && uv run coverage xml -o coverage/.python.xml +uv run coverage run --parallel-mode -m pytest \ + && uv run coverage combine \ + && uv run coverage report \ + && uv run coverage xml -o .coverage_output/python.xml echo "Python tests completed successfully" ``` diff --git a/pyproject.toml b/pyproject.toml index 5d99766c3..f3df451b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "numpy>=1.26.4", "tinygrad>=0.10.3", "requests>=2.32.5", + "mypy-boto3-s3>=1.42.37", ] [tool.uv.sources] @@ -55,7 +56,7 @@ show_missing = true skip_covered = true [tool.coverage.xml] -output = "coverage/.python.xml" +output = ".coverage_output/python.xml" [tool.ruff.lint] select = [ diff --git a/tools/sync_equity_categories.py b/tools/sync_equity_categories.py index e4a6184b1..eb1131a5a 100644 --- a/tools/sync_equity_categories.py +++ b/tools/sync_equity_categories.py @@ -23,18 +23,18 @@ logger = structlog.get_logger() -POLYGON_BASE_URL = "https://api.polygon.io" +MASSIVE_BASE_URL = os.getenv("MASSIVE_BASE_URL", "https://api.massive.io") -# Polygon ticker types: CS (Common Stock), ADRC/ADRP/ADRS (ADR variants) +# Massive ticker types: CS (Common Stock), ADRC/ADRP/ADRS (ADR variants) EQUITY_TYPES = {"CS", "ADRC", "ADRP", "ADRS"} def fetch_all_tickers(api_key: str) -> list[dict]: - """Fetch all US stock tickers from Polygon API with pagination.""" - logger.info("Fetching tickers from Polygon API") + """Fetch all US stock tickers from Massive API with pagination.""" + logger.info("Fetching tickers from Massive API") all_tickers = [] - url = f"{POLYGON_BASE_URL}/v3/reference/tickers" + url = f"{MASSIVE_BASE_URL}/v3/reference/tickers" params = { "market": "stocks", "active": "true", @@ -80,7 +80,7 @@ def extract_categories(tickers: list[dict]) -> pl.DataFrame: if ticker_data.get("type") not in EQUITY_TYPES: continue - # Try to get sector/industry from various fields Polygon provides + # Try to get sector/industry from various fields Massive provides sector = ticker_data.get("sector", "") industry = ticker_data.get("industry", "") @@ -144,7 +144,7 @@ def sync_equity_categories( tickers = fetch_all_tickers(api_key) categories = extract_categories(tickers) - s3_client = boto3.client("s3") + s3_client = cast("S3Client", boto3.client("s3")) return upload_categories_to_s3(s3_client, bucket_name, categories) diff --git a/uv.lock b/uv.lock index 933caa4db..a186b9970 100644 --- a/uv.lock +++ b/uv.lock @@ -736,6 +736,7 @@ source = { virtual = "." } dependencies = [ { name = "fastapi" }, { name = "internal" }, + { name = "mypy-boto3-s3" }, { name = "numpy" }, { name = "requests" }, { name = "sagemaker", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'win32'" }, @@ -756,6 +757,7 @@ dev = [ requires-dist = [ { name = "fastapi", specifier = ">=0.121.0" }, { name = "internal", editable = "libraries/python" }, + { name = "mypy-boto3-s3", specifier = ">=1.42.37" }, { name = "numpy", specifier = ">=1.26.4" }, { name = "requests", specifier = ">=2.32.5" }, { name = "sagemaker", specifier = ">=2.256.0" }, @@ -1459,6 +1461,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] +[[package]] +name = "mypy-boto3-s3" +version = "1.42.37" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e6/41/44066f4cd3421bacb6aad4ec7b1da8d0f8858560e526166db64d95fa7ad7/mypy_boto3_s3-1.42.37.tar.gz", hash = "sha256:628a4652f727870a07e1c3854d6f30dc545a7dd5a4b719a2c59c32a95d92e4c1", size = 76317, upload-time = "2026-01-28T20:51:52.971Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/06/cb6050ecd72f5fa449bac80ad1a4711719367c4f545201317f36e3999784/mypy_boto3_s3-1.42.37-py3-none-any.whl", hash = "sha256:7c118665f3f583dbfde1013ce47908749f9d2a760f28f59ec65732306ee9cec9", size = 83439, upload-time = "2026-01-28T20:51:49.99Z" }, +] + [[package]] name = "mypy-extensions" version = "1.1.0" @@ -3020,6 +3031,7 @@ dependencies = [ { name = "typing-extensions" }, ] wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/54/a2ba279afcca44bbd320d4e73675b282fcee3d81400ea1b53934efca6462/torch-2.10.0-2-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:13ec4add8c3faaed8d13e0574f5cd4a323c11655546f91fbe6afa77b57423574", size = 79498202, upload-time = "2026-02-10T21:44:52.603Z" }, { url = "https://files.pythonhosted.org/packages/cc/af/758e242e9102e9988969b5e621d41f36b8f258bb4a099109b7a4b4b50ea4/torch-2.10.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5fd4117d89ffd47e3dcc71e71a22efac24828ad781c7e46aaaf56bf7f2796acf", size = 145996088, upload-time = "2026-01-21T16:24:44.171Z" }, { url = "https://files.pythonhosted.org/packages/23/8e/3c74db5e53bff7ed9e34c8123e6a8bfef718b2450c35eefab85bb4a7e270/torch-2.10.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:787124e7db3b379d4f1ed54dd12ae7c741c16a4d29b49c0226a89bea50923ffb", size = 915711952, upload-time = "2026-01-21T16:23:53.503Z" }, { url = "https://files.pythonhosted.org/packages/6e/01/624c4324ca01f66ae4c7cd1b74eb16fb52596dce66dbe51eff95ef9e7a4c/torch-2.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:2c66c61f44c5f903046cc696d088e21062644cbe541c7f1c4eaae88b2ad23547", size = 113757972, upload-time = "2026-01-21T16:24:39.516Z" },