diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 859e36de5..b4ed28be7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -45,7 +45,6 @@ updates: default-days: 14 directories: - "/" - - "/crates/instrumentation-build/example" groups: cargo: applies-to: version-updates diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7711d2527..27c24cfa1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -57,12 +57,10 @@ jobs: - run: pixi run cargo fmt --all -- --check - run: pixi run cargo clippy --workspace --all-targets --all-features --locked -- -D warnings - run: pixi run cargo test --workspace --all-features --locked --all-targets + - run: pixi run cargo test -p quent-store --no-default-features --features io-ndjson --locked - run: pixi run cargo build --workspace --all-features --locked --release - # The instrumentation-build example is its own workspace (callback-only, - # Serialize-free), excluded above; build and run it separately so it stays - # covered. - - run: pixi run cargo clippy --manifest-path crates/instrumentation-build/example/Cargo.toml --all-targets --locked -- -D warnings - - run: pixi run cargo run --manifest-path crates/instrumentation-build/example/Cargo.toml --locked + - run: pixi run cargo run -p quent-instrumentation-build-example --locked + - run: pixi run cargo run -p quent-store-build-example --locked # Regression gate for `quent-open` backward compatibility: build a viewer for # a sidecar pinning the previous quent commit (the PR's base commit, or the diff --git a/Cargo.lock b/Cargo.lock index 3b8ca5cfd..7eb93dc88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2778,6 +2778,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "quent-instrumentation-build-example" +version = "0.1.0" +dependencies = [ + "quent-instrumentation", + "quent-instrumentation-build", + "quent-yaml", + "serde", +] + [[package]] name = "quent-io" version = "0.1.0" @@ -3223,6 +3233,46 @@ dependencies = [ "uuid", ] +[[package]] +name = "quent-store" +version = "0.1.0" +dependencies = [ + "quent-build-info", + "quent-events", + "quent-io", + "serde", + "tempfile", + "thiserror", + "tracing", + "uuid", +] + +[[package]] +name = "quent-store-build" +version = "0.1.0" +dependencies = [ + "prettyplease 0.3.0", + "quent-instrumentation-build", + "quent-schema", + "quote", + "syn 3.0.3", + "tempfile", + "thiserror", +] + +[[package]] +name = "quent-store-build-example" +version = "0.1.0" +dependencies = [ + "quent-events", + "quent-instrumentation-build-example", + "quent-store", + "quent-store-build", + "quent-yaml", + "serde", + "tempfile", +] + [[package]] name = "quent-time" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f867c30cc..ef0cdf7ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,9 @@ [workspace] resolver = "3" members = [ + # Legacy PoC crates, scheduled for removal + "crates/model", + "crates/model-macros", # Application-agnostic crates "crates/analyzer", "crates/dynamic-attributes", @@ -18,8 +21,6 @@ members = [ "crates/io/postcard", "crates/io/types", "crates/instrumentation", - "crates/model", - "crates/model-macros", "crates/open", "crates/stdlib", "crates/time", @@ -68,15 +69,13 @@ members = [ "crates/fsm", "crates/resource", "crates/instrumentation-build", + "crates/instrumentation-build/example", "crates/yaml", + "crates/store", + "crates/store-build", + "crates/store-build/example" ] -# The instrumentation-build example is deliberately its own workspace (see its -# Cargo.toml). Kept out of this one so `--workspace --all-features` doesn't -# unify the filesystem/collector exporters (and their `serde` bound) into its -# callback-only, `Serialize`-free graph. -exclude = ["crates/instrumentation-build/example"] - # default-members excludes any crate that activates `quent-time/__test-clock-override`. # `cargo build` and `cargo test` (no -p, no --workspace) skip those crates, preserving # the zero-cost guarantee for everything else. Use `cargo build -p ` to opt in. diff --git a/crates/instrumentation-build/example/Cargo.lock b/crates/instrumentation-build/example/Cargo.lock deleted file mode 100644 index 9859f6a81..000000000 --- a/crates/instrumentation-build/example/Cargo.lock +++ /dev/null @@ -1,918 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "getrandom 0.3.4", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "annotate-snippets" -version = "0.12.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1" -dependencies = [ - "anstyle", - "memchr", - "unicode-width", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "arraydeque" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" - -[[package]] -name = "async-trait" -version = "0.1.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "bytes" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "convert_case" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "encoding_rs_io" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83" -dependencies = [ - "encoding_rs", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "futures-core" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" - -[[package]] -name = "futures-sink" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" - -[[package]] -name = "futures-task" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" - -[[package]] -name = "futures-util" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" -dependencies = [ - "futures-core", - "futures-task", - "pin-project-lite", - "slab", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi 5.3.0", - "wasip2", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi 6.0.0", -] - -[[package]] -name = "granit-parser" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03f81ad4732830d85cfd417a9f62cde6dadda4354d37d078a6084a19560aa2d" -dependencies = [ - "arraydeque", - "smallvec", -] - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", - "serde", - "serde_core", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "js-sys" -version = "0.3.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" - -[[package]] -name = "log" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "nohash-hasher" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" - -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "petgraph" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap", - "serde", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "prettyplease" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" -dependencies = [ - "proc-macro2", - "syn 3.0.3", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quent-build-info" -version = "0.1.0" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "quent-constraints" -version = "0.1.0" -dependencies = [ - "petgraph", - "quent-schema", - "rustc-hash", -] - -[[package]] -name = "quent-dynamic-attributes" -version = "0.1.0" -dependencies = [ - "thiserror", -] - -[[package]] -name = "quent-events" -version = "0.1.0" -dependencies = [ - "quent-build-info", - "quent-dynamic-attributes", - "quent-time", - "uuid", -] - -[[package]] -name = "quent-fsm" -version = "0.1.0" -dependencies = [ - "petgraph", - "quent-constraints", - "quent-schema", - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "quent-instrumentation" -version = "0.1.0" -dependencies = [ - "async-trait", - "quent-build-info", - "quent-dynamic-attributes", - "quent-events", - "quent-io", - "quent-io-callback", - "thiserror", - "tokio", - "tokio-util", - "tracing", - "uuid", -] - -[[package]] -name = "quent-instrumentation-build" -version = "0.1.0" -dependencies = [ - "convert_case", - "prettyplease", - "proc-macro2", - "quent-constraints", - "quent-ref-target", - "quent-schema", - "quote", - "syn 3.0.3", - "thiserror", -] - -[[package]] -name = "quent-instrumentation-build-example" -version = "0.1.0" -dependencies = [ - "quent-instrumentation", - "quent-instrumentation-build", - "quent-yaml", -] - -[[package]] -name = "quent-io" -version = "0.1.0" -dependencies = [ - "async-trait", - "quent-events", - "quent-io-types", - "uuid", -] - -[[package]] -name = "quent-io-callback" -version = "0.1.0" -dependencies = [ - "async-trait", - "quent-events", - "quent-io-types", - "uuid", -] - -[[package]] -name = "quent-io-types" -version = "0.1.0" -dependencies = [ - "async-trait", - "quent-events", - "thiserror", - "tracing", - "uuid", -] - -[[package]] -name = "quent-ref-target" -version = "0.1.0" -dependencies = [ - "quent-constraints", - "quent-schema", - "thiserror", -] - -[[package]] -name = "quent-ref-tree" -version = "0.1.0" -dependencies = [ - "petgraph", - "quent-constraints", - "quent-ref-target", - "quent-schema", - "rustc-hash", - "thiserror", -] - -[[package]] -name = "quent-resource" -version = "0.1.0" -dependencies = [ - "indexmap", - "quent-constraints", - "quent-fsm", - "quent-schema", - "rustc-hash", - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "quent-schema" -version = "0.1.0" -dependencies = [ - "indexmap", - "rustc-hash", - "serde", - "smallvec", - "thiserror", -] - -[[package]] -name = "quent-time" -version = "0.1.0" -dependencies = [ - "thiserror", -] - -[[package]] -name = "quent-yaml" -version = "0.1.0" -dependencies = [ - "indexmap", - "quent-constraints", - "quent-fsm", - "quent-ref-target", - "quent-ref-tree", - "quent-resource", - "quent-schema", - "serde", - "serde-saphyr", - "thiserror", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde-saphyr" -version = "0.0.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd22781911de0ca6debda95f073c8f18bec65d1a94f1fa9573f3102e514cea4" -dependencies = [ - "ahash", - "annotate-snippets", - "base64", - "encoding_rs_io", - "getrandom 0.3.4", - "granit-parser", - "nohash-hasher", - "num-traits", - "serde_core", - "smallvec", - "zmij", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" -dependencies = [ - "serde", -] - -[[package]] -name = "syn" -version = "2.0.118" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thiserror" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "tokio" -version = "1.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" -dependencies = [ - "pin-project-lite", - "tokio-macros", -] - -[[package]] -name = "tokio-macros" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.118", -] - -[[package]] -name = "tokio-util" -version = "0.7.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.118", -] - -[[package]] -name = "tracing-core" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "nu-ansi-term", - "sharded-slab", - "smallvec", - "thread_local", - "tracing-core", - "tracing-log", -] - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unicode-segmentation" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" - -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - -[[package]] -name = "uuid" -version = "1.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" -dependencies = [ - "getrandom 0.4.3", - "js-sys", - "serde_core", - "wasm-bindgen", -] - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn 2.0.118", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "zerocopy" -version = "0.8.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.118", -] - -[[package]] -name = "zmij" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/crates/instrumentation-build/example/Cargo.toml b/crates/instrumentation-build/example/Cargo.toml index 02fdb4f43..51b7b2754 100644 --- a/crates/instrumentation-build/example/Cargo.toml +++ b/crates/instrumentation-build/example/Cargo.toml @@ -1,8 +1,3 @@ -# Its own workspace, excluded from the repo workspace, so its exporter feature -# graph stays callback-only (`Serialize`-free) instead of being unified with the -# repo's filesystem/collector exporters under `--workspace --all-features`. -[workspace] - [package] name = "quent-instrumentation-build-example" version = "0.1.0" @@ -10,7 +5,8 @@ edition = "2024" publish = false [dependencies] -quent-instrumentation = { path = "../../instrumentation", features = ["io-callback"] } +quent-instrumentation = { path = "../../instrumentation", features = ["io-callback", "io-ndjson"] } +serde = { version = "1", features = ["derive"] } [build-dependencies] quent-instrumentation-build = { path = ".." } diff --git a/crates/instrumentation-build/example/build.rs b/crates/instrumentation-build/example/build.rs index a149d4127..258c890d3 100644 --- a/crates/instrumentation-build/example/build.rs +++ b/crates/instrumentation-build/example/build.rs @@ -21,6 +21,8 @@ fn main() -> Result<(), Box> { // Schema -> generated Rust instrumentation source. let opts = Options { + // Filesystem exporters serialize generated events. + serde: true, // Generate `DemoEvent`, which lets one typed callback receive events // from every entity in the model. umbrella_event: true, diff --git a/crates/instrumentation-build/example/src/lib.rs b/crates/instrumentation-build/example/src/lib.rs new file mode 100644 index 000000000..83f2f1e71 --- /dev/null +++ b/crates/instrumentation-build/example/src/lib.rs @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Emits a small generated instrumentation model through selectable exporters. + +use std::path::PathBuf; + +use quent_instrumentation::{ + EventCallback, ExporterOptions, FileSystemExporterOptions, FileSystemFormat, +}; + +use demo::{Connection, Context, Demo, DemoEvent, Handle, Observer, Query, Server, Uuid}; + +#[allow(unused)] +mod demo { + include!(concat!(env!("OUT_DIR"), "/demo.rs")); +} + +/// Emits the demo events through a debug-printing callback. +pub fn run_with_debug_print() -> Result> { + let context = Context::try_new(EventCallback::::new(|event| { + println!("{event:?}") + }))?; + emit_events(context) +} + +/// Exports the demo events as NDJSON and returns their context ID. +pub fn run_with_ndjson( + root_export_path: impl Into, +) -> Result> { + let context = Context::try_new(ExporterOptions::FileSystem(FileSystemExporterOptions::new( + FileSystemFormat::Ndjson, + root_export_path.into(), + )))?; + emit_events(context) +} + +fn emit_events(context: Context) -> Result> { + // The context builds one exporter pipeline per entity event type and + // exposes the corresponding typed observers. + let context_id = context.id(); + + // `observer.handle()` creates a fresh entity instance to emit events for. + let mut server = context.observer::().handle(); + server.booted()?; + + let observer: Observer = context.observer::(); + // Once-cardinality events take `&mut self` and may fire only once, tracked + // by the handle, hence it is mut: + let mut conn: Handle = observer.handle(); + + // One method per entity event: + conn.opened( + demo::Endpoint { + host: "localhost".to_owned(), + port: 8080, + }, + Uuid::nil(), + // A handle can deal out a reference to the entity it represents: + server.as_entity_ref(), + )?; + conn.data(1234, None)?; + + // A `dynamic` schema field maps to `DynamicAttributes`, which are + // dynamically-typed key-value pairs: + let mut extra = demo::DynamicAttributes::new(); + extra.add_string("peer_agent", "curl/8.4"); + extra.add_u64("chunk_index", 3); + extra.add_bool("compressed", true); + conn.data( + 5678, + Some(demo::Meta { + tags: vec!["tls".to_string(), "keepalive".to_string()], + extra, + }), + )?; + + // `as_entity_ref_with` produces an entity ref that also carries data: + conn.routed(server.as_entity_ref_with(demo::Route { hops: 3 }))?; + + // An FSM entity's events are transitions into its states. + // Their cardinality is derived from the topology at build time. + // + // FSMs will get typestate pattern handles in the future, also see + // https://github.com/rapidsai/quent/issues/416 + let mut query = context.observer::().handle(); + query.submitted("select 1".to_owned(), conn.as_entity_ref())?; + query.running(10)?; + query.ready(true)?; + + conn.closed()?; + + // A once-event returns an error if emitted again. + assert!(conn.closed_emitted()); + assert!(conn.closed().is_err()); + + Ok(context_id) +} diff --git a/crates/instrumentation-build/example/src/main.rs b/crates/instrumentation-build/example/src/main.rs index 724ff71ba..ed396a042 100644 --- a/crates/instrumentation-build/example/src/main.rs +++ b/crates/instrumentation-build/example/src/main.rs @@ -1,78 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use quent_instrumentation::EventCallback; - -use crate::demo::{Connection, Context, Demo, DemoEvent, Handle, Observer, Query, Server, Uuid}; - -#[allow(unused)] -mod demo { - include!(concat!(env!("OUT_DIR"), "/demo.rs")); -} - fn main() -> Result<(), Box> { - // The context builds one exporter pipeline per entity event type and - // exposes the corresponding typed observers. - let context: Context = Context::try_new(println_exporter())?; - - // `observer.handle()` creates a fresh entity instance to events emit for. - let mut server = context.observer::().handle(); - server.booted()?; - - let observer: Observer = context.observer::(); - // Once-cardinality events take `&mut self` and may fire only once, tracked - // by the handle, hence it is mut: - let mut conn: Handle = observer.handle(); - - // One method per entity event: - conn.opened( - demo::Endpoint { - host: "localhost".to_owned(), - port: 8080, - }, - Uuid::nil(), - // A handle can deal out a reference to the entity it represents: - server.as_entity_ref(), - )?; - conn.data(1234, None)?; - - // A `dynamic` schema field maps to `DynamicAttributes`, which are - // dynamically-typed key-value pairs: - let mut extra = demo::DynamicAttributes::new(); - extra.add_string("peer_agent", "curl/8.4"); - extra.add_u64("chunk_index", 3); - extra.add_bool("compressed", true); - conn.data( - 5678, - Some(demo::Meta { - tags: vec!["tls".to_string(), "keepalive".to_string()], - extra, - }), - )?; - - // `as_entity_ref_with` produces an entity ref that also carries data: - conn.routed(server.as_entity_ref_with(demo::Route { hops: 3 }))?; - - // An FSM entity's events are transitions into its states. - // Their cardinality is derived from the topology at build time. - // - // FSMs will get typestate pattern handles in the future, also see - // https://github.com/rapidsai/quent/issues/416 - let mut query = context.observer::().handle(); - query.submitted("select 1".to_owned(), conn.as_entity_ref())?; - query.running(10)?; - query.ready(true)?; - - conn.closed()?; - - // A once-event returns an error if emitted again. - assert!(conn.closed_emitted()); - assert!(conn.closed().is_err()); - + quent_instrumentation_build_example::run_with_debug_print()?; Ok(()) } - -/// Return a callback that debug-prints each emitted event. -fn println_exporter() -> EventCallback { - EventCallback::new(|event| println!("{event:?}")) -} diff --git a/crates/instrumentation-build/src/lib.rs b/crates/instrumentation-build/src/lib.rs index 15e7254db..b905574b1 100644 --- a/crates/instrumentation-build/src/lib.rs +++ b/crates/instrumentation-build/src/lib.rs @@ -54,8 +54,9 @@ mod runtime; use std::path::PathBuf; -use quent_constraints::{BaseConstraintsError, Report, validate}; -use quent_schema::{Path, Schema}; +use convert_case::Case; +use quent_constraints::{BaseConstraintsError, Report}; +use quent_schema::{Entity, Path, Schema}; use quote::quote; /// Options controlling event and instrumentation source generation. @@ -166,19 +167,39 @@ pub struct GenerateInfo { pub warnings: Vec, } -/// Generate event source and, when enabled, instrumentation source for `schema`. -pub fn generate(schema: &Schema, opts: &Options) -> Result { +/// Validates the schema requirements shared by generated event models. +/// +/// Returns constraint names without registered validators as warnings. +pub fn validate_schema(schema: &Schema) -> Result, GenerateError> { let Report { base_constraints, unregistered_constraints, - results: _, // unused for now, but built-in constraints go here later - // and will add to either errors or warnings. - } = validate::<()>(schema); - - let warnings = unregistered_constraints; + results: _, + } = quent_constraints::validate::<()>(schema); - // Fail if base constraints aren't met. base_constraints?; + Ok(unregistered_constraints) +} + +/// Returns the model path generated for `schema` relative to the generated module root. +pub fn generated_model_path(schema: &Schema) -> proc_macro2::TokenStream { + let model = common::raw_ident(common::to_case(schema.name(), Case::Pascal)); + quote! { #model } +} + +/// Returns the entity marker path generated relative to the generated module root. +pub fn generated_entity_path(entity: &Entity) -> proc_macro2::TokenStream { + common::relative_type_path(entity.path(), &[], "") +} + +/// Returns the entity event path generated relative to the generated module root. +pub fn generated_entity_event_path(entity: &Entity) -> proc_macro2::TokenStream { + common::relative_type_path(entity.path(), &[], "Event") +} + +/// Generate event source and, when enabled, instrumentation source for `schema`. +pub fn generate(schema: &Schema, opts: &Options) -> Result { + let warnings = validate_schema(schema)?; let file_name = opts .file_name diff --git a/crates/instrumentation/src/lib.rs b/crates/instrumentation/src/lib.rs index 725b6ab07..89ab5dae3 100644 --- a/crates/instrumentation/src/lib.rs +++ b/crates/instrumentation/src/lib.rs @@ -32,6 +32,8 @@ pub use quent_dynamic_attributes::DynamicAttributes; pub use quent_events as events; pub use quent_events::{AnyEntity, EntityEvent, EntityRef, Event, Model, ModelEvents}; pub use quent_io::{ExporterOptions, ExporterProvider}; +#[cfg(any(feature = "io-ndjson", feature = "io-msgpack", feature = "io-postcard"))] +pub use quent_io::{FileSystemExporterOptions, FileSystemFormat}; pub use uuid::Uuid; /// A caller-supplied typed event sink, selected via the `io-callback` feature. diff --git a/crates/store-build/Cargo.toml b/crates/store-build/Cargo.toml new file mode 100644 index 000000000..4e70ad2ad --- /dev/null +++ b/crates/store-build/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "quent-store-build" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[dependencies] +prettyplease.workspace = true +quent-instrumentation-build = { path = "../instrumentation-build" } +quent-schema = { path = "../schema" } +quote = "1" +syn.workspace = true +thiserror.workspace = true + +[dev-dependencies] +quent-schema = { path = "../schema", features = ["test-utils"] } +tempfile = "3" diff --git a/crates/store-build/example/Cargo.toml b/crates/store-build/example/Cargo.toml new file mode 100644 index 000000000..67f6997dc --- /dev/null +++ b/crates/store-build/example/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "quent-store-build-example" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +quent-events = { path = "../../events", features = ["serde"] } +quent-instrumentation-build-example = { path = "../../instrumentation-build/example" } +quent-store = { path = "../../store" } +serde = { version = "1", features = ["derive"] } +tempfile = "3" + +[build-dependencies] +quent-store-build = { path = ".." } +quent-yaml = { path = "../../yaml" } diff --git a/crates/store-build/example/build.rs b/crates/store-build/example/build.rs new file mode 100644 index 000000000..6d9f20ad1 --- /dev/null +++ b/crates/store-build/example/build.rs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Generates store types from the schema shared with the instrumentation example. + +use std::path::Path; + +use quent_store_build::{Options, generate}; + +fn main() -> Result<(), Box> { + let model = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../instrumentation-build/example/model.yaml"); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed={}", model.display()); + + let parsed = quent_yaml::parse_from_file(&model)?; + for warning in &parsed.warnings { + println!("cargo:warning={warning}"); + } + + let options = Options { + // Generate `DemoEvent` so the example can load all model events through + // one iterator. Entity-specific loading does not require this option. + umbrella_event: true, + ..Options::default() + }; + let generated = generate(&parsed.schema, &options)?; + if !generated.warnings.is_empty() { + println!("cargo:warning= {}", generated.warnings.join("\n")); + } + println!( + "cargo:warning=stored-event retrieval API written to {}", + generated.path.display() + ); + Ok(()) +} diff --git a/crates/store-build/example/src/main.rs b/crates/store-build/example/src/main.rs new file mode 100644 index 000000000..57737a7b4 --- /dev/null +++ b/crates/store-build/example/src/main.rs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Runs instrumentation and loads its filesystem-exported events. + +use demo::{Demo, Query}; +use quent_store::event::filesystem::Store; +use quent_store::event::{EntityEventStore, ModelEventStore}; + +#[allow(unused_imports)] +mod demo { + include!(concat!(env!("OUT_DIR"), "/demo.rs")); +} + +fn main() -> Result<(), Box> { + let output = tempfile::tempdir()?; + let context_id = quent_instrumentation_build_example::run_with_ndjson(output.path())?; + + let store = Store::::new(output.path()); + + println!("--- Query events ---"); + + // Load events for one entity type. + for event in store.entity_events::(context_id)? { + println!("{:?}", event?); + } + + println!("\n--- All model events ---"); + + // Load all model events as `DemoEvent`. + for event in store.events(context_id)? { + println!("{:?}", event?); + } + + Ok(()) +} diff --git a/crates/store-build/src/lib.rs b/crates/store-build/src/lib.rs new file mode 100644 index 000000000..3d149b686 --- /dev/null +++ b/crates/store-build/src/lib.rs @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Generates schema-based typed APIs for retrieving stored events. +//! +//! Add `quent-store-build` to `[build-dependencies]`, call [`generate`] from +//! `build.rs`, and include the generated file from Cargo's `OUT_DIR`. +//! The crate including that source needs normal dependencies on `quent-store`, +//! serde-enabled `quent-events`, and derive-enabled `serde`. +//! +//! ```ignore +//! // build.rs +//! use quent_store_build::{Options, generate}; +//! +//! fn main() -> Result<(), Box> { +//! let schema = todo!("load a quent_schema::Schema"); +//! generate(&schema, &Options::default())?; +//! Ok(()) +//! } +//! ``` +//! +//! ```ignore +//! // src/lib.rs +//! mod model { +//! include!(concat!(env!("OUT_DIR"), "/demo.rs")); +//! } +//! ``` +//! +//! ```toml +//! [build-dependencies] +//! quent-store-build = { path = "../quent/crates/store-build" } +//! +//! [dependencies] +//! quent-events = { path = "../quent/crates/events", features = ["serde"] } +//! quent-store = { path = "../quent/crates/store" } +//! serde = { version = "1", features = ["derive"] } +//! ``` + +use std::path::PathBuf; + +use quent_schema::Schema; +use quote::quote; + +/// Options controlling stored-event retrieval source generation. +/// +/// Generated event and record types always derive `serde::Serialize` and +/// `serde::Deserialize`. +pub struct Options { + /// Derive [`Debug`](std::fmt::Debug) on generated event and record types. + pub debug: bool, + + /// Additional derives applied to every generated event payload enum. + pub event_derives: &'static [&'static str], + + /// Additional derives applied to every generated record struct. + pub record_derives: &'static [&'static str], + + /// Generate a model-wide umbrella event and model-wide filesystem loading support. + /// + /// The consuming crate must enable at least one `quent-store` `io-*` feature. + pub umbrella_event: bool, + + /// Directory the generated file is written into. + pub out_dir: PathBuf, + + /// File name to write; defaults to the lowercase schema name with a `.rs` extension. + pub file_name: Option, +} + +impl Default for Options { + fn default() -> Self { + Self { + debug: true, + event_derives: Default::default(), + record_derives: Default::default(), + umbrella_event: false, + out_dir: PathBuf::from(std::env::var("OUT_DIR").unwrap_or_default()), + file_name: None, + } + } +} + +/// An error from generating stored-event retrieval source. +#[derive(Debug, thiserror::Error)] +pub enum GenerateError { + #[error(transparent)] + EventModel(#[from] quent_instrumentation_build::GenerateError), + #[error("generated stored-event retrieval code did not form a valid Rust file")] + InvalidGeneratedCode(#[source] syn::Error), + #[error("failed to write generated stored-event retrieval source")] + Io(#[from] std::io::Error), +} + +/// Information about generated stored-event retrieval source. +pub struct GenerateInfo { + /// Path of the generated Rust source file. + pub path: PathBuf, + /// Constraint names without registered validators. + pub warnings: Vec, +} + +/// Generates event types and their typed stored-event retrieval API. +/// +/// # Errors +/// +/// Returns an error when the schema cannot be generated or the output cannot be written. +pub fn generate(schema: &Schema, opts: &Options) -> Result { + let warnings = quent_instrumentation_build::validate_schema(schema)?; + let file_name = opts + .file_name + .clone() + .unwrap_or_else(|| format!("{}.rs", schema.name().to_string().to_lowercase())); + let path = opts.out_dir.join(file_name); + std::fs::write(&path, generate_str(schema, opts)?)?; + Ok(GenerateInfo { path, warnings }) +} + +/// Returns stored-event retrieval source for `schema`. +/// +/// # Errors +/// +/// Returns an error when event generation fails or the combined output is not valid Rust. +pub fn generate_str(schema: &Schema, opts: &Options) -> Result { + let event_opts = quent_instrumentation_build::Options { + instrumentation: false, + debug: opts.debug, + serde: true, + event_derives: opts.event_derives, + record_derives: opts.record_derives, + umbrella_event: opts.umbrella_event, + ..quent_instrumentation_build::Options::default() + }; + let events = quent_instrumentation_build::generate_str(schema, &event_opts)?; + let events = + syn::parse_str::(&events).map_err(GenerateError::InvalidGeneratedCode)?; + + let model = quent_instrumentation_build::generated_model_path(schema); + let stored_model = if opts.umbrella_event { + let streams = schema.entities().map(|entity| { + let event = quent_instrumentation_build::generated_entity_event_path(entity); + quote! { + ::quent_store::event::filesystem::EventStream::new( + <#event as ::quent_events::EntityEvent>::NAME, + ::quent_store::event::filesystem::import_event_files::<#model, #event>, + ) + } + }); + quote! { + impl ::quent_store::event::filesystem::Model for #model { + fn event_streams( + ) -> &'static [::quent_store::event::filesystem::EventStream] { + static STREAMS: &[ + ::quent_store::event::filesystem::EventStream<#model> + ] = &[ + #(#streams,)* + ]; + STREAMS + } + } + } + } else { + quote! {} + }; + let entities = schema.entities().map(|entity| { + let marker = quent_instrumentation_build::generated_entity_path(entity); + quote! { + impl ::quent_store::event::StoredEntity<#model> for #marker {} + } + }); + + let file = syn::parse2::(quote! { + #events + + #stored_model + + #(#entities)* + }) + .map_err(GenerateError::InvalidGeneratedCode)?; + + Ok(prettyplease::unparse(&file)) +} + +#[cfg(test)] +mod tests { + use quent_schema::builder::{AnnotationsBuilder, EntityBuilder, SchemaBuilder}; + use quent_schema::test_utils::{entity, event}; + + use super::*; + + #[test] + fn generates_nested_retrieval_apis_with_optional_model_loading() { + let schema = SchemaBuilder::try_new("Demo") + .unwrap() + .with_entity(entity("Foo::Query", [event("created", [])])) + .with_entity(entity("Foo::Nested::Task", [event("created", [])])) + .build() + .unwrap(); + + let default_source = generate_str(&schema, &Options::default()).unwrap(); + let opts = Options { + umbrella_event: true, + ..Options::default() + }; + let umbrella_source = generate_str(&schema, &opts).unwrap(); + + assert!(default_source.contains("event::StoredEntity for foo::Query")); + assert!(default_source.contains("event::StoredEntity for foo::nested::Task")); + assert!(!default_source.contains("filesystem::Model for Demo")); + assert!(umbrella_source.contains("impl ::quent_store::event::filesystem::Model for Demo")); + assert_eq!(umbrella_source.matches("import_event_files::<").count(), 2); + } + + #[test] + fn generate_returns_unregistered_constraint_warnings() { + let annotations = AnnotationsBuilder::new() + .with_constraint("example.unknown.v0.1.0", None) + .build() + .unwrap(); + let query = EntityBuilder::try_new("Query") + .unwrap() + .with_event(event("created", [])) + .with_annotations(annotations) + .build() + .unwrap(); + let schema = SchemaBuilder::try_new("Demo") + .unwrap() + .with_entity(query) + .build() + .unwrap(); + let output = tempfile::tempdir().unwrap(); + let options = Options { + out_dir: output.path().to_owned(), + ..Options::default() + }; + + let generated = generate(&schema, &options).unwrap(); + + assert_eq!(generated.warnings, ["example.unknown.v0.1.0"]); + } +} diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml new file mode 100644 index 000000000..565b73ab0 --- /dev/null +++ b/crates/store/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "quent-store" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[features] +default = ["io-ndjson", "io-msgpack", "io-postcard"] +io-ndjson = ["quent-io/ndjson"] +io-msgpack = ["quent-io/msgpack"] +io-postcard = ["quent-io/postcard"] + +[dependencies] +quent-build-info = { path = "../build-info" } +quent-events = { path = "../events" } +quent-io = { path = "../io", default-features = false } +serde.workspace = true +thiserror.workspace = true +tracing.workspace = true +uuid.workspace = true + +[dev-dependencies] +tempfile = "3" diff --git a/crates/store/src/event/filesystem.rs b/crates/store/src/event/filesystem.rs new file mode 100644 index 000000000..13c75938f --- /dev/null +++ b/crates/store/src/event/filesystem.rs @@ -0,0 +1,417 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Filesystem-backed event storage. + +use std::marker::PhantomData; +use std::path::{Path, PathBuf}; + +use quent_build_info::{ArtifactInfo, SIDECAR_FILE_NAME}; +use quent_events::{EntityEvent, Event, Model as EventModel, ModelEvents}; +use quent_io::ImporterProvider; +use quent_io::filesystem::{Format, importer}; +use serde::de::DeserializeOwned; +use uuid::Uuid; + +use super::{ + EntityEventLoader, EntityEventStore, EventIterator, ModelEventLoader, ModelEventStore, + StoredEntity, +}; + +/// Result returned by filesystem event stores. +pub type Result = std::result::Result; + +/// An error encountered while loading filesystem events. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("context `{0}` was not found")] + ContextNotFound(Uuid), + #[error("context path `{0}` is not a directory")] + ContextNotDirectory(PathBuf), + #[error("context model `{actual}` does not match expected model `{expected}`")] + ModelMismatch { expected: String, actual: String }, + #[error("event file `{path}` requires the `{feature}` feature for `{format}` data")] + DisabledFormat { + path: PathBuf, + format: String, + feature: &'static str, + }, + #[error("failed to {operation} `{path}`: {source}")] + Io { + operation: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("failed to import events from `{path}`: {source}")] + Importer { + path: PathBuf, + #[source] + source: quent_io::ImporterError, + }, +} + +/// Associates a generated model with its filesystem entity-event streams. +#[doc(hidden)] +pub trait Model: ModelEvents { + /// Returns the streams generated from the model schema. + fn event_streams() -> &'static [EventStream] + where + Self: Sized; +} + +type ImportFn = + fn(Vec) -> Result::UmbrellaEvent, Error>>; + +/// Describes one entity-event stream in a generated analysis model. +#[doc(hidden)] +pub struct EventStream { + entity: &'static str, + import: ImportFn, +} + +impl EventStream { + /// Creates a generated entity-event stream descriptor. + #[doc(hidden)] + pub const fn new(entity: &'static str, import: ImportFn) -> Self { + Self { entity, import } + } +} + +/// Identifies an event file and the importer required to decode it. +#[doc(hidden)] +pub struct EventFile { + format: Format, + path: PathBuf, +} + +/// Imports files containing entity events and converts them to the model umbrella type. +#[doc(hidden)] +pub fn import_event_files( + files: Vec, +) -> Result> +where + M: ModelEvents, + E: DeserializeOwned + Into + 'static, + M::UmbrellaEvent: 'static, +{ + Ok(Box::new(import_files::(files).map(|event| { + event.map(|event| Event::new(event.id, event.timestamp, event.data.into())) + }))) +} + +/// Loads model events from filesystem exporter output. +pub struct Store { + root: PathBuf, + model: PhantomData M>, +} + +impl Store { + /// Creates a store rooted at an exporter output directory. + pub fn new(root: impl Into) -> Self { + Self { + root: root.into(), + model: PhantomData, + } + } + + /// Returns the exporter output directory. + pub fn root(&self) -> &Path { + &self.root + } +} + +impl EntityEventStore for Store { + type Error = Error; +} + +impl EntityEventLoader for Store +where + M: EventModel, + E: StoredEntity, + E::Event: DeserializeOwned + 'static, +{ + type Error = Error; + + fn load_entity_events(&self, context_id: Uuid) -> Result> { + let context = self.context(context_id)?; + Ok(import_files::(event_files( + &context, + E::Event::NAME, + )?)) + } +} + +impl ModelEventStore for Store {} + +impl ModelEventLoader for Store +where + M: EventModel + Model + 'static, +{ + type Error = Error; + + fn load_model_events( + &self, + context_id: Uuid, + ) -> Result> { + let context = self.context(context_id)?; + let mut streams = Vec::new(); + for descriptor in M::event_streams() { + let files = event_files(&context, descriptor.entity)?; + streams.push((descriptor.import)(files)?); + } + Ok(Box::new(streams.into_iter().flatten())) + } +} + +impl Store +where + M: EventModel, +{ + /// Returns the context directory after verifying that it exists and belongs to `M`. + fn context(&self, context_id: Uuid) -> Result { + let context = self.root.join(context_id.to_string()); + match std::fs::metadata(&context) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => return Err(Error::ContextNotDirectory(context)), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + return Err(Error::ContextNotFound(context_id)); + } + Err(source) => { + return Err(Error::Io { + operation: "inspect context path", + path: context, + source, + }); + } + } + let artifact = ArtifactInfo::read_sidecar(&context).map_err(|source| Error::Io { + operation: "read context metadata from", + path: context.join(SIDECAR_FILE_NAME), + source, + })?; + if artifact.model.name != M::NAME { + return Err(Error::ModelMismatch { + expected: M::NAME.to_owned(), + actual: artifact.model.name, + }); + } + Ok(context) + } +} + +/// Imports event files in their supplied order and yields importer failures as iterator items. +fn import_files(files: Vec) -> EventIterator +where + T: DeserializeOwned + 'static, +{ + Box::new(files.into_iter().flat_map(|file| { + let stream = || { + let path = file.path; + let importer = importer::Options { + format: file.format, + path: path.clone(), + } + .create_importer() + .map_err(|source| Error::Importer { + path: path.clone(), + source, + })?; + Ok::<_, Error>(Box::new(importer.map(move |event| { + event.map_err(|source| Error::Importer { + path: path.clone(), + source, + }) + })) as EventIterator) + }; + + stream().unwrap_or_else(|error| Box::new(std::iter::once(Err(error)))) + })) +} + +/// Returns recognized event files for `entity` in path order. +/// +/// A missing or non-directory entity path produces an empty list. A recognized format whose +/// feature is disabled produces an error. +fn event_files(context: &Path, entity: &str) -> Result> { + let directory = context.join(entity); + match std::fs::metadata(&directory) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => return Ok(Vec::new()), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(source) => { + return Err(Error::Io { + operation: "inspect event directory", + path: directory, + source, + }); + } + } + + let entries = std::fs::read_dir(&directory).map_err(|source| Error::Io { + operation: "read event directory", + path: directory.clone(), + source, + })?; + let mut paths = Vec::new(); + for entry in entries { + let entry = entry.map_err(|source| Error::Io { + operation: "read entry in event directory", + path: directory.clone(), + source, + })?; + let path = entry.path(); + let file_type = entry.file_type().map_err(|source| Error::Io { + operation: "inspect event file", + path: path.clone(), + source, + })?; + if file_type.is_file() { + paths.push(path); + } + } + paths.sort(); + + let mut files = Vec::new(); + for path in paths { + let Some(extension) = path.extension().and_then(|value| value.to_str()) else { + tracing::debug!( + path = %path.display(), + "ignoring file with unsupported event format" + ); + continue; + }; + match Format::try_from(extension) { + Ok(format) => files.push(EventFile { format, path }), + Err(_) => { + let normalized = extension.to_ascii_lowercase(); + let Some(feature) = format_feature(&normalized) else { + tracing::debug!( + path = %path.display(), + "ignoring file with unsupported event format" + ); + continue; + }; + return Err(Error::DisabledFormat { + path, + format: normalized, + feature, + }); + } + } + } + Ok(files) +} + +fn format_feature(extension: &str) -> Option<&'static str> { + match extension { + "ndjson" => Some("io-ndjson"), + "msgpack" => Some("io-msgpack"), + "postcard" => Some("io-postcard"), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use quent_build_info::ModelInfo; + use quent_events::Model as EventModel; + #[cfg(feature = "io-ndjson")] + use serde::Deserialize; + + use super::*; + + struct TestModel; + + impl EventModel for TestModel { + const NAME: &'static str = "Test"; + } + + #[test] + fn validates_context_and_model() { + let root = tempfile::tempdir().unwrap(); + let store = Store::::new(root.path()); + + let missing = Uuid::from_u128(1); + assert!(matches!( + store.context(missing), + Err(Error::ContextNotFound(id)) if id == missing + )); + + let mismatch = Uuid::from_u128(2); + let context = root.path().join(mismatch.to_string()); + fs::create_dir(&context).unwrap(); + let mut model = ModelInfo::unknown(); + model.name = "Other".to_owned(); + ArtifactInfo::new(model).write_sidecar(&context).unwrap(); + assert!(matches!( + store.context(mismatch), + Err(Error::ModelMismatch { actual, .. }) if actual == "Other" + )); + } + + #[cfg(feature = "io-ndjson")] + #[test] + fn returns_event_files_in_path_order() { + let root = tempfile::tempdir().unwrap(); + let entity = root.path().join("Alpha"); + fs::create_dir(&entity).unwrap(); + for name in ["charlie.ndjson", "alpha.ndjson", "bravo.ndjson"] { + fs::write(entity.join(name), b"").unwrap(); + } + + let paths = event_files(root.path(), "Alpha") + .unwrap() + .into_iter() + .map(|file| file.path.file_name().unwrap().to_owned()) + .collect::>(); + + assert_eq!(paths, ["alpha.ndjson", "bravo.ndjson", "charlie.ndjson"]); + } + + #[test] + fn rejects_event_files_for_disabled_formats() { + if Format::try_from("msgpack").is_ok() { + return; + } + + let root = tempfile::tempdir().unwrap(); + let entity = root.path().join("Alpha"); + fs::create_dir(&entity).unwrap(); + let path = entity.join("events.msgpack"); + fs::write(&path, b"").unwrap(); + + assert!(matches!( + event_files(root.path(), "Alpha"), + Err(Error::DisabledFormat { + path: error_path, + format, + feature: "io-msgpack", + }) if error_path == path && format == "msgpack" + )); + } + + #[cfg(feature = "io-ndjson")] + #[test] + fn reports_import_failures_during_iteration() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("events.ndjson"); + fs::write(&path, b"not json\n").unwrap(); + + #[derive(Deserialize)] + struct TestEvent; + + let mut events = import_files::(vec![EventFile { + format: Format::Ndjson, + path: path.clone(), + }]); + + assert!(matches!( + events.next(), + Some(Err(Error::Importer { path: error_path, .. })) if error_path == path + )); + assert!(events.next().is_none()); + } +} diff --git a/crates/store/src/event/mod.rs b/crates/store/src/event/mod.rs new file mode 100644 index 000000000..81ecf8a15 --- /dev/null +++ b/crates/store/src/event/mod.rs @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Typed access to fully materialized model events. + +use quent_events::{Entity, Event, ModelEvents}; +use uuid::Uuid; + +#[cfg(any(feature = "io-ndjson", feature = "io-msgpack", feature = "io-postcard"))] +pub mod filesystem; + +/// An iterator yielding owned [`Event`](Event) values or read failures. +pub type EventIterator = Box, E>>>; + +/// The result of creating an [`EventIterator`]. +pub type EventIteratorResult = Result, E>; + +/// Loads stored events as owned values with payloads typed for an entity in model `M`. +pub trait EntityEventStore { + /// Error returned when events cannot be loaded. + type Error; + + /// Loads events for entity type `E` without an ordering guarantee. + fn entity_events( + &self, + context_id: Uuid, + ) -> EventIteratorResult>::Error> + where + E: StoredEntity, + Self: EntityEventLoader>::Error>, + { + self.load_entity_events(context_id) + } +} + +/// Loads model-wide stored events as owned values with umbrella-event payloads. +/// +/// Generated models support this trait only when +/// `quent_store_build::Options::umbrella_event` is enabled. +pub trait ModelEventStore: EntityEventStore { + /// Loads every event stored for `context_id` without an ordering guarantee. + fn events( + &self, + context_id: Uuid, + ) -> EventIteratorResult>::Error> + where + Self: ModelEventLoader>::Error>, + { + self.load_model_events(context_id) + } +} + +/// Loads one concrete entity event type for an [`EntityEventStore`]. +#[doc(hidden)] +pub trait EntityEventLoader { + /// Error returned when events cannot be loaded. + type Error; + + /// Loads events for `E` without an ordering guarantee. + fn load_entity_events(&self, context_id: Uuid) -> EventIteratorResult; +} + +/// Loads umbrella events for a [`ModelEventStore`]. +#[doc(hidden)] +pub trait ModelEventLoader { + /// Error returned when events cannot be loaded. + type Error; + + /// Loads model events without an ordering guarantee. + fn load_model_events( + &self, + context_id: Uuid, + ) -> EventIteratorResult; +} + +/// Marks an entity as belonging to analysis model `M`. +#[doc(hidden)] +pub trait StoredEntity: Entity {} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs new file mode 100644 index 000000000..0ec4ba673 --- /dev/null +++ b/crates/store/src/lib.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Typed access to stored data. + +pub mod event;