From 6b588b24844d0c74f06f63f11ac3a5f042e52e45 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 08:40:38 -0300 Subject: [PATCH 01/28] feat(queue): scaffold queue worker crate --- queue/Cargo.lock | 2325 ++++++++++++++++++++++++++++++++++++ queue/Cargo.toml | 35 + queue/README.md | 3 + queue/iii.worker.yaml | 12 + queue/src/boot.rs | 1 + queue/src/config.rs | 1 + queue/src/configuration.rs | 1 + queue/src/functions.rs | 1 + queue/src/lib.rs | 12 + queue/src/main.rs | 3 + queue/src/manifest.rs | 1 + queue/src/store.rs | 1 + queue/src/trigger.rs | 1 + queue/tests/.gitkeep | 1 + 14 files changed, 2398 insertions(+) create mode 100644 queue/Cargo.lock create mode 100644 queue/Cargo.toml create mode 100644 queue/README.md create mode 100644 queue/iii.worker.yaml create mode 100644 queue/src/boot.rs create mode 100644 queue/src/config.rs create mode 100644 queue/src/configuration.rs create mode 100644 queue/src/functions.rs create mode 100644 queue/src/lib.rs create mode 100644 queue/src/main.rs create mode 100644 queue/src/manifest.rs create mode 100644 queue/src/store.rs create mode 100644 queue/src/trigger.rs create mode 100644 queue/tests/.gitkeep diff --git a/queue/Cargo.lock b/queue/Cargo.lock new file mode 100644 index 000000000..30beba797 --- /dev/null +++ b/queue/Cargo.lock @@ -0,0 +1,2325 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "iii-helpers" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09daa7c14a9e4c1f7077c4a181918d207e3f05cdfea8b2d7781bbeb80caf4c5d" +dependencies = [ + "futures-util", + "opentelemetry", + "opentelemetry-http", + "opentelemetry_sdk", + "reqwest", + "schemars", + "serde", + "serde_json", + "sysinfo", + "tokio", + "tokio-tungstenite", + "tracing", + "uuid", +] + +[[package]] +name = "iii-queue" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "clap", + "iii-sdk", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "serial_test", + "tokio", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "iii-sdk" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5957568413b9a5178c11bf91b20909e93e568b187e259e941ba6009a7ec3f5c1" +dependencies = [ + "async-trait", + "futures-util", + "hostname", + "iii-helpers", + "reqwest", + "schemars", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-tungstenite", + "tracing", + "uuid", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "opentelemetry" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", + "tracing", +] + +[[package]] +name = "opentelemetry-http" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustls" +version = "0.23.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "serial_test" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "699f4197115b8a7e7ff19c9a315a4bd6fffec26cc4626ef45ecaea389e081c6d" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/queue/Cargo.toml b/queue/Cargo.toml new file mode 100644 index 000000000..5a604fded --- /dev/null +++ b/queue/Cargo.toml @@ -0,0 +1,35 @@ +[workspace] + +[package] +name = "iii-queue" +version = "0.1.0" +edition = "2021" +description = "Durable queue worker for iii - durable:subscriber trigger type and queue service functions" +license = "Apache-2.0" +repository = "https://github.com/iii-hq/workers" +publish = false + +[lib] +name = "iii_queue" +path = "src/lib.rs" + +[[bin]] +name = "queue" +path = "src/main.rs" + +[dependencies] +iii-sdk = "=0.20.0" +async-trait = "0.1" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_yaml = "0.9" +anyhow = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } +clap = { version = "4", features = ["derive"] } +schemars = "0.8" +uuid = { version = "1", features = ["v4"] } + +[dev-dependencies] +serial_test = "3" diff --git a/queue/README.md b/queue/README.md new file mode 100644 index 000000000..5d840799d --- /dev/null +++ b/queue/README.md @@ -0,0 +1,3 @@ +# queue worker + +Durable function queues for iii. The complete registry migration documentation lands later in this branch. diff --git a/queue/iii.worker.yaml b/queue/iii.worker.yaml new file mode 100644 index 000000000..05a9429b6 --- /dev/null +++ b/queue/iii.worker.yaml @@ -0,0 +1,12 @@ +iii: v1 +name: queue +language: rust +deploy: binary +manifest: Cargo.toml +bin: queue +description: Durable function queues - registers the `durable:subscriber` trigger type and the queue/DLQ service functions. +config: + adapter: + name: builtin +dependencies: + configuration: "^0.19.0" diff --git a/queue/src/boot.rs b/queue/src/boot.rs new file mode 100644 index 000000000..615bc2dee --- /dev/null +++ b/queue/src/boot.rs @@ -0,0 +1 @@ +//! Worker boot and shutdown wiring. diff --git a/queue/src/config.rs b/queue/src/config.rs new file mode 100644 index 000000000..983971e9a --- /dev/null +++ b/queue/src/config.rs @@ -0,0 +1 @@ +//! Worker configuration. diff --git a/queue/src/configuration.rs b/queue/src/configuration.rs new file mode 100644 index 000000000..8650a73b7 --- /dev/null +++ b/queue/src/configuration.rs @@ -0,0 +1 @@ +//! Configuration worker integration. diff --git a/queue/src/functions.rs b/queue/src/functions.rs new file mode 100644 index 000000000..2d38d2066 --- /dev/null +++ b/queue/src/functions.rs @@ -0,0 +1 @@ +//! Queue and DLQ service function registration. diff --git a/queue/src/lib.rs b/queue/src/lib.rs new file mode 100644 index 000000000..380c95789 --- /dev/null +++ b/queue/src/lib.rs @@ -0,0 +1,12 @@ +/// The trigger type this worker registers. Always `durable:subscriber` -- +/// the built-in `iii-queue` worker must be removed from the engine config +/// before this worker can boot (see [`boot::start`]'s guard). +pub const TRIGGER_TYPE: &str = "durable:subscriber"; + +pub mod boot; +pub mod config; +pub mod configuration; +pub mod functions; +pub mod manifest; +pub mod store; +pub mod trigger; diff --git a/queue/src/main.rs b/queue/src/main.rs new file mode 100644 index 000000000..284177ae8 --- /dev/null +++ b/queue/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("queue worker - wiring lands in a later task on this branch"); +} diff --git a/queue/src/manifest.rs b/queue/src/manifest.rs new file mode 100644 index 000000000..8655b32bd --- /dev/null +++ b/queue/src/manifest.rs @@ -0,0 +1 @@ +//! Worker manifest generation. diff --git a/queue/src/store.rs b/queue/src/store.rs new file mode 100644 index 000000000..d039e69de --- /dev/null +++ b/queue/src/store.rs @@ -0,0 +1 @@ +//! Queue store implementations. diff --git a/queue/src/trigger.rs b/queue/src/trigger.rs new file mode 100644 index 000000000..5fd72f268 --- /dev/null +++ b/queue/src/trigger.rs @@ -0,0 +1 @@ +//! Durable subscriber trigger handling. diff --git a/queue/tests/.gitkeep b/queue/tests/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/queue/tests/.gitkeep @@ -0,0 +1 @@ + From f50685f19a65a026f6379400ad73fef4ecab5a27 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 08:41:42 -0300 Subject: [PATCH 02/28] feat(queue): worker config with pluggable transport adapter entry --- queue/src/config.rs | 84 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/queue/src/config.rs b/queue/src/config.rs index 983971e9a..b1ca1863e 100644 --- a/queue/src/config.rs +++ b/queue/src/config.rs @@ -1 +1,83 @@ -//! Worker configuration. +//! Worker configuration: which queue transport adapter this worker uses. +//! Mirrors the builtin's QueueModuleConfig (engine/src/workers/queue/config.rs) +//! with the default in-process transport named `builtin`. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +pub const DEFAULT_ADAPTER: &str = "builtin"; + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AdapterEntry { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub config: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct QueueConfig { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub adapter: Option, +} + +impl QueueConfig { + pub fn effective_adapter_name(&self) -> &str { + self.adapter + .as_ref() + .map(|a| a.name.as_str()) + .unwrap_or(DEFAULT_ADAPTER) + } + + pub fn normalized(&self) -> Self { + self.clone() + } + + pub fn json_schema() -> Value { + serde_json::to_value(schemars::schema_for!(QueueConfig)).unwrap_or(Value::Null) + } + + pub fn to_json(&self) -> Value { + serde_json::to_value(self).unwrap_or(Value::Null) + } + + pub fn from_json(value: &Value) -> Result { + serde_json::from_value(value.clone()) + .map_err(|e| format!("invalid queue configuration: {e}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_config_uses_builtin_adapter() { + let c = QueueConfig::default(); + assert_eq!(c.effective_adapter_name(), "builtin"); + } + + #[test] + fn deserializes_adapter_entry() { + let c: QueueConfig = + serde_yaml::from_str("{adapter: {name: file_based, config: {path: './data/queue'}}}") + .unwrap(); + assert_eq!(c.effective_adapter_name(), "file_based"); + assert_eq!(c.adapter.unwrap().config.unwrap()["path"], "./data/queue"); + } + + #[test] + fn rejects_unknown_fields() { + let r: Result = serde_yaml::from_str("{adapter: {name: builtin}, wat: 1}"); + assert!(r.is_err(), "deny_unknown_fields must reject 'wat'"); + } + + #[test] + fn json_roundtrip() { + let c: QueueConfig = serde_yaml::from_str("{adapter: {name: file_based}}").unwrap(); + let back = QueueConfig::from_json(&c.to_json()).unwrap(); + assert_eq!(back.effective_adapter_name(), "file_based"); + } +} From f23b4f2754517ceb4f72677ff855a49cdcc27ec6 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 08:46:35 -0300 Subject: [PATCH 03/28] feat(queue): port in-memory and file-backed queue store with retry/DLQ parity --- queue/src/store.rs | 600 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 600 insertions(+) diff --git a/queue/src/store.rs b/queue/src/store.rs index d039e69de..4b956f5a6 100644 --- a/queue/src/store.rs +++ b/queue/src/store.rs @@ -1 +1,601 @@ //! Queue store implementations. +//! +//! The engine builtin stores waiting job ids, delayed job ids, and DLQ entries +//! separately. This worker keeps the same behavioral model behind a smaller +//! store interface: normal queues contain both ready and delayed jobs, DLQs are +//! per topic, and file-backed mode persists the full snapshot on every +//! mutation. Retry backoff mirrors the builtin's exponential curve: +//! `backoff_ms * 2^(attempts - 1)`. + +use std::collections::{HashMap, VecDeque}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tokio::sync::Mutex; +use uuid::Uuid; + +const STORE_FILE_NAME: &str = "queue_store.json"; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Job { + pub id: String, + pub payload: Value, + pub attempts: u32, + pub enqueued_at_ms: u64, + #[serde(default)] + pub(crate) ready_at_ms: u64, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TopicStats { + pub depth: u64, + pub dlq_depth: u64, + pub delivered: u64, + pub failed: u64, +} + +#[async_trait] +pub trait QueueStore: Send + Sync + 'static { + async fn enqueue(&self, topic: &str, payload: Value) -> anyhow::Result; + async fn dequeue(&self, topic: &str) -> Option; + async fn ack(&self, topic: &str, job_id: &str); + async fn nack(&self, topic: &str, job: Job, max_retries: u32, backoff_ms: u64); + async fn list_topics(&self) -> Vec; + async fn topic_stats(&self, topic: &str) -> TopicStats; + async fn dlq_topics(&self) -> Vec<(String, u64)>; + async fn dlq_messages(&self, topic: &str, limit: u64) -> Vec; + async fn redrive_dlq(&self, topic: &str) -> u64; + async fn redrive_dlq_message(&self, topic: &str, job_id: &str) -> bool; + async fn discard_dlq_message(&self, topic: &str, job_id: &str) -> bool; +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +struct StoreData { + queues: HashMap>, + dlqs: HashMap>, + stats: HashMap, +} + +#[derive(Debug)] +struct SharedStore { + inner: Mutex, + file_dir: Option, +} + +#[derive(Debug, Clone)] +pub struct InMemoryStore { + shared: Arc, +} + +#[derive(Debug, Clone)] +pub struct FileStore { + shared: Arc, +} + +impl InMemoryStore { + pub fn new() -> Self { + Self { + shared: Arc::new(SharedStore { + inner: Mutex::new(StoreData::default()), + file_dir: None, + }), + } + } +} + +impl Default for InMemoryStore { + fn default() -> Self { + Self::new() + } +} + +impl FileStore { + pub async fn open(path: impl AsRef, _save_interval_ms: u64) -> anyhow::Result { + let dir = path.as_ref().to_path_buf(); + std::fs::create_dir_all(&dir)?; + let data = load_snapshot(&dir).await?; + Ok(Self { + shared: Arc::new(SharedStore { + inner: Mutex::new(data), + file_dir: Some(dir), + }), + }) + } +} + +#[async_trait] +impl QueueStore for InMemoryStore { + async fn enqueue(&self, topic: &str, payload: Value) -> anyhow::Result { + enqueue(&self.shared, topic, payload).await + } + + async fn dequeue(&self, topic: &str) -> Option { + dequeue(&self.shared, topic).await + } + + async fn ack(&self, topic: &str, job_id: &str) { + ack(&self.shared, topic, job_id).await; + } + + async fn nack(&self, topic: &str, job: Job, max_retries: u32, backoff_ms: u64) { + nack(&self.shared, topic, job, max_retries, backoff_ms).await; + } + + async fn list_topics(&self) -> Vec { + list_topics(&self.shared).await + } + + async fn topic_stats(&self, topic: &str) -> TopicStats { + topic_stats(&self.shared, topic).await + } + + async fn dlq_topics(&self) -> Vec<(String, u64)> { + dlq_topics(&self.shared).await + } + + async fn dlq_messages(&self, topic: &str, limit: u64) -> Vec { + dlq_messages(&self.shared, topic, limit).await + } + + async fn redrive_dlq(&self, topic: &str) -> u64 { + redrive_dlq(&self.shared, topic).await + } + + async fn redrive_dlq_message(&self, topic: &str, job_id: &str) -> bool { + redrive_dlq_message(&self.shared, topic, job_id).await + } + + async fn discard_dlq_message(&self, topic: &str, job_id: &str) -> bool { + discard_dlq_message(&self.shared, topic, job_id).await + } +} + +#[async_trait] +impl QueueStore for FileStore { + async fn enqueue(&self, topic: &str, payload: Value) -> anyhow::Result { + enqueue(&self.shared, topic, payload).await + } + + async fn dequeue(&self, topic: &str) -> Option { + dequeue(&self.shared, topic).await + } + + async fn ack(&self, topic: &str, job_id: &str) { + ack(&self.shared, topic, job_id).await; + } + + async fn nack(&self, topic: &str, job: Job, max_retries: u32, backoff_ms: u64) { + nack(&self.shared, topic, job, max_retries, backoff_ms).await; + } + + async fn list_topics(&self) -> Vec { + list_topics(&self.shared).await + } + + async fn topic_stats(&self, topic: &str) -> TopicStats { + topic_stats(&self.shared, topic).await + } + + async fn dlq_topics(&self) -> Vec<(String, u64)> { + dlq_topics(&self.shared).await + } + + async fn dlq_messages(&self, topic: &str, limit: u64) -> Vec { + dlq_messages(&self.shared, topic, limit).await + } + + async fn redrive_dlq(&self, topic: &str) -> u64 { + redrive_dlq(&self.shared, topic).await + } + + async fn redrive_dlq_message(&self, topic: &str, job_id: &str) -> bool { + redrive_dlq_message(&self.shared, topic, job_id).await + } + + async fn discard_dlq_message(&self, topic: &str, job_id: &str) -> bool { + discard_dlq_message(&self.shared, topic, job_id).await + } +} + +async fn enqueue(shared: &SharedStore, topic: &str, payload: Value) -> anyhow::Result { + let now = now_ms(); + let job = Job { + id: Uuid::new_v4().to_string(), + payload, + attempts: 0, + enqueued_at_ms: now, + ready_at_ms: now, + }; + let id = job.id.clone(); + + let snapshot = { + let mut data = shared.inner.lock().await; + data.queues + .entry(topic.to_string()) + .or_default() + .push_back(job); + data.stats.entry(topic.to_string()).or_default().depth = + data.queues.get(topic).map_or(0, |q| q.len() as u64); + data.clone() + }; + persist_if_needed(shared, &snapshot).await?; + Ok(id) +} + +async fn dequeue(shared: &SharedStore, topic: &str) -> Option { + let now = now_ms(); + let result = { + let mut data = shared.inner.lock().await; + let queue = data.queues.get_mut(topic)?; + let index = queue.iter().position(|job| job.ready_at_ms <= now)?; + let job = queue.remove(index); + if queue.is_empty() { + data.queues.remove(topic); + } + let depth = data.queues.get(topic).map_or(0, |q| q.len() as u64); + data.stats.entry(topic.to_string()).or_default().depth = depth; + (job, data.clone()) + }; + + let _ = persist_if_needed(shared, &result.1).await; + result.0 +} + +async fn ack(shared: &SharedStore, topic: &str, _job_id: &str) { + let snapshot = { + let mut data = shared.inner.lock().await; + data.stats.entry(topic.to_string()).or_default().delivered += 1; + data.clone() + }; + let _ = persist_if_needed(shared, &snapshot).await; +} + +async fn nack(shared: &SharedStore, topic: &str, mut job: Job, max_retries: u32, backoff_ms: u64) { + job.attempts = job.attempts.saturating_add(1); + let snapshot = { + let mut data = shared.inner.lock().await; + if job.attempts >= max_retries { + data.dlqs.entry(topic.to_string()).or_default().push(job); + let dlq_depth = data.dlqs.get(topic).map_or(0, |q| q.len() as u64); + let stats = data.stats.entry(topic.to_string()).or_default(); + stats.failed += 1; + stats.dlq_depth = dlq_depth; + } else { + let delay = exponential_backoff_ms(backoff_ms, job.attempts); + job.ready_at_ms = now_ms().saturating_add(delay); + data.queues + .entry(topic.to_string()) + .or_default() + .push_back(job); + data.stats.entry(topic.to_string()).or_default().depth = + data.queues.get(topic).map_or(0, |q| q.len() as u64); + } + data.clone() + }; + let _ = persist_if_needed(shared, &snapshot).await; +} + +async fn list_topics(shared: &SharedStore) -> Vec { + let data = shared.inner.lock().await; + let mut topics = data + .queues + .keys() + .chain(data.dlqs.keys()) + .chain(data.stats.keys()) + .cloned() + .collect::>(); + topics.sort(); + topics.dedup(); + topics +} + +async fn topic_stats(shared: &SharedStore, topic: &str) -> TopicStats { + let data = shared.inner.lock().await; + let mut stats = data.stats.get(topic).cloned().unwrap_or_default(); + stats.depth = data.queues.get(topic).map_or(0, |q| q.len() as u64); + stats.dlq_depth = data.dlqs.get(topic).map_or(0, |q| q.len() as u64); + stats +} + +async fn dlq_topics(shared: &SharedStore) -> Vec<(String, u64)> { + let data = shared.inner.lock().await; + let mut topics = data + .dlqs + .iter() + .filter_map(|(topic, jobs)| { + if jobs.is_empty() { + None + } else { + Some((topic.clone(), jobs.len() as u64)) + } + }) + .collect::>(); + topics.sort_by(|a, b| a.0.cmp(&b.0)); + topics +} + +async fn dlq_messages(shared: &SharedStore, topic: &str, limit: u64) -> Vec { + let data = shared.inner.lock().await; + data.dlqs + .get(topic) + .map(|jobs| jobs.iter().take(limit as usize).cloned().collect()) + .unwrap_or_default() +} + +async fn redrive_dlq(shared: &SharedStore, topic: &str) -> u64 { + let snapshot_and_count = { + let mut data = shared.inner.lock().await; + let mut jobs = data.dlqs.remove(topic).unwrap_or_default(); + let count = jobs.len() as u64; + for job in &mut jobs { + job.attempts = 0; + job.ready_at_ms = now_ms(); + } + data.queues + .entry(topic.to_string()) + .or_default() + .extend(jobs.into_iter()); + let depth = data.queues.get(topic).map_or(0, |q| q.len() as u64); + let stats = data.stats.entry(topic.to_string()).or_default(); + stats.depth = depth; + stats.dlq_depth = 0; + (data.clone(), count) + }; + let _ = persist_if_needed(shared, &snapshot_and_count.0).await; + snapshot_and_count.1 +} + +async fn redrive_dlq_message(shared: &SharedStore, topic: &str, job_id: &str) -> bool { + let snapshot_and_found = { + let mut data = shared.inner.lock().await; + let Some(dlq) = data.dlqs.get_mut(topic) else { + return false; + }; + let Some(index) = dlq.iter().position(|job| job.id == job_id) else { + return false; + }; + let mut job = dlq.remove(index); + job.attempts = 0; + job.ready_at_ms = now_ms(); + data.queues + .entry(topic.to_string()) + .or_default() + .push_back(job); + let depth = data.queues.get(topic).map_or(0, |q| q.len() as u64); + let dlq_depth = data.dlqs.get(topic).map_or(0, |q| q.len() as u64); + let stats = data.stats.entry(topic.to_string()).or_default(); + stats.depth = depth; + stats.dlq_depth = dlq_depth; + (data.clone(), true) + }; + let _ = persist_if_needed(shared, &snapshot_and_found.0).await; + snapshot_and_found.1 +} + +async fn discard_dlq_message(shared: &SharedStore, topic: &str, job_id: &str) -> bool { + let snapshot_and_found = { + let mut data = shared.inner.lock().await; + let Some(dlq) = data.dlqs.get_mut(topic) else { + return false; + }; + let Some(index) = dlq.iter().position(|job| job.id == job_id) else { + return false; + }; + dlq.remove(index); + let dlq_depth = data.dlqs.get(topic).map_or(0, |q| q.len() as u64); + data.stats.entry(topic.to_string()).or_default().dlq_depth = dlq_depth; + (data.clone(), true) + }; + let _ = persist_if_needed(shared, &snapshot_and_found.0).await; + snapshot_and_found.1 +} + +async fn load_snapshot(dir: &Path) -> anyhow::Result { + let path = dir.join(STORE_FILE_NAME); + if !path.exists() { + return Ok(StoreData::default()); + } + let bytes = std::fs::read(path)?; + Ok(serde_json::from_slice(&bytes)?) +} + +async fn persist_if_needed(shared: &SharedStore, snapshot: &StoreData) -> anyhow::Result<()> { + let Some(dir) = &shared.file_dir else { + return Ok(()); + }; + std::fs::create_dir_all(dir)?; + let path = dir.join(STORE_FILE_NAME); + let tmp = dir.join(format!("{STORE_FILE_NAME}.tmp")); + let bytes = serde_json::to_vec_pretty(snapshot)?; + std::fs::write(&tmp, bytes)?; + std::fs::rename(tmp, path)?; + Ok(()) +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time is before UNIX_EPOCH") + .as_millis() as u64 +} + +fn exponential_backoff_ms(backoff_ms: u64, attempts: u32) -> u64 { + backoff_ms.saturating_mul(2_u64.saturating_pow(attempts.saturating_sub(1))) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::future::Future; + use std::pin::Pin; + use tokio::time::{sleep, Duration}; + + type StoreFuture = Pin> + Send>>; + + fn temp_store_dir() -> PathBuf { + let dir = std::env::temp_dir().join(format!("queue_store_{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + async fn for_each_backend(test: F) + where + F: Fn(Arc) -> StoreFuture, + { + test(Arc::new(InMemoryStore::new())).await; + let dir = temp_store_dir(); + let store = Arc::new(FileStore::open(&dir, 5).await.unwrap()); + test(store).await; + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn enqueue_dequeue_ack_removes_ready_job() { + for_each_backend(|store| { + Box::pin(async move { + let id = store.enqueue("demo", json!({"x": 1})).await.unwrap(); + let job = store.dequeue("demo").await.unwrap(); + assert_eq!(job.id, id); + assert_eq!(job.payload, json!({"x": 1})); + store.ack("demo", &job.id).await; + assert!(store.dequeue("demo").await.is_none()); + assert_eq!(store.topic_stats("demo").await.delivered, 1); + store + }) + }) + .await; + } + + #[tokio::test] + async fn nack_requeues_after_exponential_backoff() { + for_each_backend(|store| { + Box::pin(async move { + store.enqueue("demo", json!("job")).await.unwrap(); + let job = store.dequeue("demo").await.unwrap(); + store.nack("demo", job, 3, 30).await; + assert!(store.dequeue("demo").await.is_none()); + sleep(Duration::from_millis(35)).await; + let retry = store.dequeue("demo").await.unwrap(); + assert_eq!(retry.attempts, 1); + store + }) + }) + .await; + } + + #[tokio::test] + async fn exhausted_nack_moves_to_dlq() { + for_each_backend(|store| { + Box::pin(async move { + store.enqueue("demo", json!("job")).await.unwrap(); + let job = store.dequeue("demo").await.unwrap(); + store.nack("demo", job, 1, 1).await; + assert!(store.dequeue("demo").await.is_none()); + let messages = store.dlq_messages("demo", 10).await; + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].attempts, 1); + let stats = store.topic_stats("demo").await; + assert_eq!(stats.dlq_depth, 1); + assert_eq!(stats.failed, 1); + store + }) + }) + .await; + } + + #[tokio::test] + async fn redrive_dlq_moves_all_back_with_attempts_reset() { + for_each_backend(|store| { + Box::pin(async move { + for n in 0..2 { + store.enqueue("demo", json!(n)).await.unwrap(); + let job = store.dequeue("demo").await.unwrap(); + store.nack("demo", job, 1, 1).await; + } + assert_eq!(store.redrive_dlq("demo").await, 2); + assert!(store.dlq_messages("demo", 10).await.is_empty()); + let first = store.dequeue("demo").await.unwrap(); + let second = store.dequeue("demo").await.unwrap(); + assert_eq!(first.attempts, 0); + assert_eq!(second.attempts, 0); + store + }) + }) + .await; + } + + #[tokio::test] + async fn redrive_single_dlq_message_operates_by_id() { + for_each_backend(|store| { + Box::pin(async move { + store.enqueue("demo", json!("a")).await.unwrap(); + let job = store.dequeue("demo").await.unwrap(); + store.nack("demo", job, 1, 1).await; + let id = store.dlq_messages("demo", 10).await[0].id.clone(); + assert!(!store.redrive_dlq_message("demo", "missing").await); + assert!(store.redrive_dlq_message("demo", &id).await); + assert_eq!(store.dequeue("demo").await.unwrap().id, id); + store + }) + }) + .await; + } + + #[tokio::test] + async fn discard_single_dlq_message_operates_by_id() { + for_each_backend(|store| { + Box::pin(async move { + store.enqueue("demo", json!("a")).await.unwrap(); + let job = store.dequeue("demo").await.unwrap(); + store.nack("demo", job, 1, 1).await; + let id = store.dlq_messages("demo", 10).await[0].id.clone(); + assert!(!store.discard_dlq_message("demo", "missing").await); + assert!(store.discard_dlq_message("demo", &id).await); + assert!(store.dlq_messages("demo", 10).await.is_empty()); + store + }) + }) + .await; + } + + #[tokio::test] + async fn topics_and_stats_reflect_depths() { + for_each_backend(|store| { + Box::pin(async move { + store.enqueue("demo", json!("ready")).await.unwrap(); + store.enqueue("demo", json!("still-ready")).await.unwrap(); + store.enqueue("other", json!("ready")).await.unwrap(); + let job = store.dequeue("demo").await.unwrap(); + store.nack("demo", job, 1, 1).await; + assert_eq!(store.list_topics().await, vec!["demo", "other"]); + assert_eq!(store.dlq_topics().await, vec![("demo".to_string(), 1)]); + let demo = store.topic_stats("demo").await; + assert_eq!(demo.depth, 1); + assert_eq!(demo.dlq_depth, 1); + store + }) + }) + .await; + } + + #[tokio::test] + async fn file_store_message_survives_restart() { + let dir = temp_store_dir(); + { + let store = FileStore::open(&dir, 5).await.unwrap(); + store + .enqueue("demo", json!({"survives": true})) + .await + .unwrap(); + } + let reopened = FileStore::open(&dir, 5).await.unwrap(); + let job = reopened.dequeue("demo").await.unwrap(); + assert_eq!(job.payload, json!({"survives": true})); + let _ = std::fs::remove_dir_all(dir); + } +} From ada7bdfa9b24f98e4c78505edf71a0e3fe2252da Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 08:48:58 -0300 Subject: [PATCH 04/28] feat(queue): durable:subscriber consumer loops with ack/nack/DLQ parity --- queue/src/trigger.rs | 445 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 445 insertions(+) diff --git a/queue/src/trigger.rs b/queue/src/trigger.rs index 5fd72f268..65c54299d 100644 --- a/queue/src/trigger.rs +++ b/queue/src/trigger.rs @@ -1 +1,446 @@ //! Durable subscriber trigger handling. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use iii_sdk::errors::Error; +use iii_sdk::protocol::TriggerRequest; +use iii_sdk::trigger::{TriggerConfig, TriggerHandler}; +use iii_sdk::IIIClient; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; + +use crate::store::QueueStore; + +pub const DEFAULT_MAX_RETRIES: u32 = 3; +pub const DEFAULT_BACKOFF_MS: u64 = 1000; +const IDLE_POLL_MS: u64 = 50; + +#[async_trait] +pub trait Invoker: Send + Sync + 'static { + async fn call(&self, function_id: &str, payload: Value) -> Result, String>; +} + +#[derive(Clone)] +pub struct IiiInvoker { + iii: Arc, +} + +impl IiiInvoker { + pub fn new(iii: Arc) -> Self { + Self { iii } + } +} + +#[async_trait] +impl Invoker for IiiInvoker { + async fn call(&self, function_id: &str, payload: Value) -> Result, String> { + self.iii + .trigger(TriggerRequest { + function_id: function_id.to_string(), + payload, + action: None, + timeout_ms: None, + }) + .await + .map(Some) + .map_err(|e| e.to_string()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct SubscriberSpec { + /// Queue/topic name to consume. + #[serde(alias = "topic")] + pub queue: String, + #[serde(default, alias = "maxRetries", skip_serializing_if = "Option::is_none")] + pub max_retries: Option, + #[serde( + default, + alias = "backoffDelayMs", + skip_serializing_if = "Option::is_none" + )] + pub backoff_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub condition_function_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_config: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct SubscriberQueueConfig { + #[serde(default, alias = "maxRetries", skip_serializing_if = "Option::is_none")] + pub max_retries: Option, + #[serde( + default, + alias = "backoffDelayMs", + skip_serializing_if = "Option::is_none" + )] + pub backoff_delay_ms: Option, +} + +#[derive(Debug, Clone)] +pub struct RegisteredSubscriber { + pub trigger_id: String, + pub function_id: String, + pub spec: SubscriberSpec, +} + +struct Consumer { + registration: RegisteredSubscriber, + handle: JoinHandle<()>, +} + +#[derive(Clone)] +pub struct QueueTriggerHandler { + store: Arc, + invoker: Arc, + consumers: Arc>>, +} + +impl QueueTriggerHandler { + pub fn new(store: Arc, invoker: Arc) -> Self { + Self { + store, + invoker, + consumers: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub async fn registrations(&self) -> Vec { + self.consumers + .lock() + .await + .values() + .map(|consumer| consumer.registration.clone()) + .collect() + } + + pub async fn shutdown(&self) { + let consumers = self.consumers.lock().await.drain().collect::>(); + for (_, consumer) in consumers { + consumer.handle.abort(); + } + } + + async fn register(&self, registration: RegisteredSubscriber) -> Result<(), String> { + if registration.spec.queue.trim().is_empty() { + return Err("queue is required for durable:subscriber trigger".to_string()); + } + + let mut consumers = self.consumers.lock().await; + if consumers.contains_key(®istration.trigger_id) { + return Err(format!( + "durable:subscriber trigger '{}' is already registered", + registration.trigger_id + )); + } + + let handle = spawn_consumer_loop( + self.store.clone(), + self.invoker.clone(), + registration.clone(), + ); + consumers.insert( + registration.trigger_id.clone(), + Consumer { + registration, + handle, + }, + ); + Ok(()) + } +} + +#[async_trait] +impl TriggerHandler for QueueTriggerHandler { + async fn register_trigger(&self, config: TriggerConfig) -> Result<(), Error> { + let spec: SubscriberSpec = serde_json::from_value(config.config.clone()) + .map_err(|e| Error::Handler(format!("invalid durable:subscriber config: {e}")))?; + self.register(RegisteredSubscriber { + trigger_id: config.id, + function_id: config.function_id, + spec, + }) + .await + .map_err(Error::Handler) + } + + async fn unregister_trigger(&self, config: TriggerConfig) -> Result<(), Error> { + if let Some(consumer) = self.consumers.lock().await.remove(&config.id) { + consumer.handle.abort(); + } + Ok(()) + } +} + +fn spawn_consumer_loop( + store: Arc, + invoker: Arc, + registration: RegisteredSubscriber, +) -> JoinHandle<()> { + tokio::spawn(async move { + let queue = registration.spec.queue.clone(); + let max_retries = effective_max_retries(®istration.spec); + let backoff_ms = effective_backoff_ms(®istration.spec); + + loop { + let Some(job) = store.dequeue(&queue).await else { + tokio::time::sleep(Duration::from_millis(IDLE_POLL_MS)).await; + continue; + }; + + if let Some(condition_id) = registration.spec.condition_function_id.as_deref() { + match invoker.call(condition_id, job.payload.clone()).await { + Ok(Some(Value::Bool(false))) => { + store.ack(&queue, &job.id).await; + continue; + } + Ok(_) => {} + Err(_) => { + store.nack(&queue, job, max_retries, backoff_ms).await; + continue; + } + } + } + + match invoker + .call(®istration.function_id, job.payload.clone()) + .await + { + Ok(_) => store.ack(&queue, &job.id).await, + Err(_) => store.nack(&queue, job, max_retries, backoff_ms).await, + } + } + }) +} + +fn effective_max_retries(spec: &SubscriberSpec) -> u32 { + spec.max_retries + .or_else(|| spec.queue_config.as_ref().and_then(|c| c.max_retries)) + .unwrap_or(DEFAULT_MAX_RETRIES) +} + +fn effective_backoff_ms(spec: &SubscriberSpec) -> u64 { + spec.backoff_ms + .or_else(|| spec.queue_config.as_ref().and_then(|c| c.backoff_delay_ms)) + .unwrap_or(DEFAULT_BACKOFF_MS) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::{InMemoryStore, QueueStore}; + use serde_json::json; + use std::future::Future; + use std::sync::atomic::{AtomicBool, Ordering}; + use tokio::time::{sleep, Duration, Instant}; + + #[derive(Default)] + struct FakeInvoker { + calls: Mutex>, + fail_backend: AtomicBool, + condition_value: Mutex>, + } + + #[async_trait] + impl Invoker for FakeInvoker { + async fn call(&self, function_id: &str, payload: Value) -> Result, String> { + self.calls + .lock() + .await + .push((function_id.to_string(), payload)); + if function_id == "condition" { + return Ok(self.condition_value.lock().await.clone()); + } + if self.fail_backend.load(Ordering::SeqCst) { + Err("backend failed".to_string()) + } else { + Ok(Some(json!({"ok": true}))) + } + } + } + + fn trigger_config(id: &str, function_id: &str, config: Value) -> TriggerConfig { + TriggerConfig { + id: id.to_string(), + function_id: function_id.to_string(), + config, + metadata: None, + } + } + + async fn wait_until(mut pred: F) + where + F: FnMut() -> Fut, + Fut: Future, + { + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline { + if pred().await { + return; + } + sleep(Duration::from_millis(20)).await; + } + assert!(pred().await, "condition did not become true before timeout"); + } + + #[tokio::test] + async fn register_spawns_loop_and_acks_on_success() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + let handler = QueueTriggerHandler::new(store.clone(), invoker.clone()); + store + .enqueue("demo", json!({"hello": "world"})) + .await + .unwrap(); + + handler + .register_trigger(trigger_config( + "t1", + "backend", + json!({"queue": "demo", "max_retries": 2, "backoff_ms": 1}), + )) + .await + .unwrap(); + + wait_until(|| { + let invoker = invoker.clone(); + async move { invoker.calls.lock().await.len() == 1 } + }) + .await; + assert_eq!(store.topic_stats("demo").await.delivered, 1); + assert!(store.dequeue("demo").await.is_none()); + handler.shutdown().await; + } + + #[tokio::test] + async fn failed_invocation_nacks_until_dlq() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + invoker.fail_backend.store(true, Ordering::SeqCst); + let handler = QueueTriggerHandler::new(store.clone(), invoker.clone()); + store + .enqueue("demo", json!({"hello": "world"})) + .await + .unwrap(); + + handler + .register_trigger(trigger_config( + "t1", + "backend", + json!({"queue": "demo", "max_retries": 1, "backoff_ms": 1}), + )) + .await + .unwrap(); + + wait_until(|| { + let invoker = invoker.clone(); + async move { !invoker.calls.lock().await.is_empty() } + }) + .await; + wait_until(|| { + let store = store.clone(); + async move { !store.dlq_messages("demo", 10).await.is_empty() } + }) + .await; + assert_eq!(store.topic_stats("demo").await.failed, 1); + handler.shutdown().await; + } + + #[tokio::test] + async fn register_rejects_empty_queue() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + let handler = QueueTriggerHandler::new(store, invoker); + let err = handler + .register_trigger(trigger_config("t1", "backend", json!({"queue": ""}))) + .await + .unwrap_err(); + assert!(err.to_string().contains("queue")); + } + + #[tokio::test] + async fn duplicate_trigger_id_is_rejected() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + let handler = QueueTriggerHandler::new(store, invoker); + let cfg = trigger_config("t1", "backend", json!({"queue": "demo"})); + handler.register_trigger(cfg.clone()).await.unwrap(); + let err = handler.register_trigger(cfg).await.unwrap_err(); + assert!(err.to_string().contains("already registered")); + handler.shutdown().await; + } + + #[tokio::test] + async fn unregister_by_id_stops_consumption() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + let handler = QueueTriggerHandler::new(store.clone(), invoker.clone()); + handler + .register_trigger(trigger_config("t1", "backend", json!({"queue": "demo"}))) + .await + .unwrap(); + handler + .unregister_trigger(trigger_config("t1", "", Value::Null)) + .await + .unwrap(); + store + .enqueue("demo", json!({"after": "stop"})) + .await + .unwrap(); + sleep(Duration::from_millis(120)).await; + assert!(invoker.calls.lock().await.is_empty()); + } + + #[tokio::test] + async fn explicit_false_condition_skips_and_acks() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + *invoker.condition_value.lock().await = Some(Value::Bool(false)); + let handler = QueueTriggerHandler::new(store.clone(), invoker.clone()); + store + .enqueue("demo", json!({"hello": "world"})) + .await + .unwrap(); + + handler + .register_trigger(trigger_config( + "t1", + "backend", + json!({"queue": "demo", "condition_function_id": "condition"}), + )) + .await + .unwrap(); + + wait_until(|| { + let invoker = invoker.clone(); + async move { invoker.calls.lock().await.len() == 1 } + }) + .await; + assert_eq!(invoker.calls.lock().await[0].0, "condition"); + assert_eq!(store.topic_stats("demo").await.delivered, 1); + assert!(store.dequeue("demo").await.is_none()); + handler.shutdown().await; + } + + #[test] + fn supports_builtin_topic_and_queue_config_aliases() { + let spec: SubscriberSpec = serde_json::from_value(json!({ + "topic": "demo", + "queue_config": { + "maxRetries": 9, + "backoffDelayMs": 25 + } + })) + .unwrap(); + assert_eq!(spec.queue, "demo"); + assert_eq!(effective_max_retries(&spec), 9); + assert_eq!(effective_backoff_ms(&spec), 25); + } +} From 670229198b22cfb7c3e1e49095902a92c930f199 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 08:50:56 -0300 Subject: [PATCH 05/28] feat(queue): queue/DLQ service functions with exact builtin id parity --- queue/src/functions.rs | 465 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 465 insertions(+) diff --git a/queue/src/functions.rs b/queue/src/functions.rs index 2d38d2066..a0c450b04 100644 --- a/queue/src/functions.rs +++ b/queue/src/functions.rs @@ -1 +1,466 @@ //! Queue and DLQ service function registration. + +use std::sync::Arc; + +use iii_sdk::errors::Error; +use iii_sdk::{IIIClient, RegisterFunction}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::store::{Job, QueueStore}; + +pub const PUBLISH_FN_ID: &str = "iii::durable::publish"; +pub const REDRIVE_FN_ID: &str = "iii::queue::redrive"; +pub const REDRIVE_MESSAGE_FN_ID: &str = "iii::queue::redrive_message"; +pub const DISCARD_MESSAGE_FN_ID: &str = "iii::queue::discard_message"; +pub const LIST_TOPICS_FN_ID: &str = "engine::queue::list_topics"; +pub const TOPIC_STATS_FN_ID: &str = "engine::queue::topic_stats"; +pub const DLQ_TOPICS_FN_ID: &str = "engine::queue::dlq_topics"; +pub const DLQ_MESSAGES_FN_ID: &str = "engine::queue::dlq_messages"; + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct PublishInput { + /// Topic to publish to. `queue` is accepted for the migration worker API. + #[serde(alias = "queue")] + pub topic: String, + pub data: Value, +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct RedriveInput { + #[serde(alias = "topic")] + pub queue: String, +} + +#[derive(Debug, Clone, Serialize, JsonSchema, PartialEq, Eq)] +pub struct RedriveResult { + pub queue: String, + pub redriven: u64, +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct RedriveSingleInput { + #[serde(alias = "topic")] + pub queue: String, + pub message_id: String, +} + +#[derive(Debug, Clone, Serialize, JsonSchema, PartialEq, Eq)] +pub struct RedriveSingleResult { + pub queue: String, + pub message_id: String, + pub redriven: u64, +} + +#[derive(Debug, Clone, Serialize, JsonSchema, PartialEq, Eq)] +pub struct TopicInfo { + pub name: String, + pub broker_type: String, + pub subscriber_count: u64, +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct TopicStatsInput { + #[serde(alias = "queue")] + pub topic: String, +} + +#[derive(Debug, Clone, Serialize, JsonSchema, PartialEq, Eq)] +pub struct TopicStatsOutput { + pub depth: u64, + pub consumer_count: u64, + pub dlq_depth: u64, + pub config: Option, +} + +#[derive(Debug, Clone, Serialize, JsonSchema, PartialEq, Eq)] +pub struct DlqTopicInfo { + pub topic: String, + pub broker_type: String, + pub message_count: u64, +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct DlqMessagesInput { + #[serde(alias = "queue")] + pub topic: String, + #[serde(default)] + pub offset: u64, + #[serde(default = "default_dlq_limit")] + pub limit: u64, +} + +fn default_dlq_limit() -> u64 { + 50 +} + +#[derive(Debug, Clone, Serialize, JsonSchema, PartialEq)] +pub struct DlqMessage { + pub id: String, + pub payload: Value, + pub error: String, + pub failed_at: u64, + pub retries: u32, + pub size_bytes: u64, +} + +pub fn register_all(iii: &Arc, store: Arc) { + let publish_store = store.clone(); + iii.register_function( + PUBLISH_FN_ID, + RegisterFunction::new_async(move |input: PublishInput| { + let store = publish_store.clone(); + async move { publish(store, input).await } + }) + .description("Enqueue a message"), + ); + + let redrive_store = store.clone(); + iii.register_function( + REDRIVE_FN_ID, + RegisterFunction::new_async(move |input: RedriveInput| { + let store = redrive_store.clone(); + async move { redrive(store, input).await } + }) + .description("Redrive all DLQ messages back to the main queue"), + ); + + let redrive_message_store = store.clone(); + iii.register_function( + REDRIVE_MESSAGE_FN_ID, + RegisterFunction::new_async(move |input: RedriveSingleInput| { + let store = redrive_message_store.clone(); + async move { redrive_message(store, input).await } + }) + .description("Redrive a single DLQ message by ID back to the main queue"), + ); + + let discard_store = store.clone(); + iii.register_function( + DISCARD_MESSAGE_FN_ID, + RegisterFunction::new_async(move |input: RedriveSingleInput| { + let store = discard_store.clone(); + async move { discard_message(store, input).await } + }) + .description("Discard (purge) a single DLQ message by ID"), + ); + + let list_store = store.clone(); + iii.register_function( + LIST_TOPICS_FN_ID, + RegisterFunction::new_async(move |_input: Value| { + let store = list_store.clone(); + async move { list_topics(store).await } + }) + .description("List all queue topics"), + ); + + let stats_store = store.clone(); + iii.register_function( + TOPIC_STATS_FN_ID, + RegisterFunction::new_async(move |input: TopicStatsInput| { + let store = stats_store.clone(); + async move { topic_stats(store, input).await } + }) + .description("Get stats for a queue topic"), + ); + + let dlq_topics_store = store.clone(); + iii.register_function( + DLQ_TOPICS_FN_ID, + RegisterFunction::new_async(move |_input: Value| { + let store = dlq_topics_store.clone(); + async move { dlq_topics(store).await } + }) + .description("List DLQ topics with counts"), + ); + + let dlq_messages_store = store; + iii.register_function( + DLQ_MESSAGES_FN_ID, + RegisterFunction::new_async(move |input: DlqMessagesInput| { + let store = dlq_messages_store.clone(); + async move { dlq_messages(store, input).await } + }) + .description("Browse DLQ messages"), + ); +} + +pub async fn publish( + store: Arc, + input: PublishInput, +) -> Result, Error> { + if input.topic.is_empty() { + return Err(Error::Handler("Topic is not set".to_string())); + } + store + .enqueue(&input.topic, input.data) + .await + .map_err(|e| Error::Handler(e.to_string()))?; + Ok(None) +} + +pub async fn redrive( + store: Arc, + input: RedriveInput, +) -> Result { + if input.queue.is_empty() { + return Err(Error::Handler("Queue name is required".to_string())); + } + let redriven = store.redrive_dlq(&input.queue).await; + Ok(RedriveResult { + queue: input.queue, + redriven, + }) +} + +pub async fn redrive_message( + store: Arc, + input: RedriveSingleInput, +) -> Result { + validate_single_input(&input)?; + let found = store + .redrive_dlq_message(&input.queue, &input.message_id) + .await; + Ok(RedriveSingleResult { + queue: input.queue, + message_id: input.message_id, + redriven: u64::from(found), + }) +} + +pub async fn discard_message( + store: Arc, + input: RedriveSingleInput, +) -> Result { + validate_single_input(&input)?; + let found = store + .discard_dlq_message(&input.queue, &input.message_id) + .await; + Ok(RedriveSingleResult { + queue: input.queue, + message_id: input.message_id, + redriven: u64::from(found), + }) +} + +pub async fn list_topics(store: Arc) -> Result, Error> { + Ok(store + .list_topics() + .await + .into_iter() + .map(|name| TopicInfo { + name, + broker_type: "builtin".to_string(), + subscriber_count: 0, + }) + .collect()) +} + +pub async fn topic_stats( + store: Arc, + input: TopicStatsInput, +) -> Result { + if input.topic.is_empty() { + return Err(Error::Handler("topic is required".to_string())); + } + let stats = store.topic_stats(&input.topic).await; + Ok(TopicStatsOutput { + depth: stats.depth, + consumer_count: 0, + dlq_depth: stats.dlq_depth, + config: None, + }) +} + +pub async fn dlq_topics(store: Arc) -> Result, Error> { + Ok(store + .dlq_topics() + .await + .into_iter() + .map(|(topic, message_count)| DlqTopicInfo { + topic, + broker_type: "builtin".to_string(), + message_count, + }) + .collect()) +} + +pub async fn dlq_messages( + store: Arc, + input: DlqMessagesInput, +) -> Result, Error> { + if input.topic.is_empty() { + return Err(Error::Handler("topic is required".to_string())); + } + let messages = store + .dlq_messages(&input.topic, input.offset.saturating_add(input.limit)) + .await + .into_iter() + .skip(input.offset as usize) + .take(input.limit as usize) + .map(dlq_message_from_job) + .collect(); + Ok(messages) +} + +fn validate_single_input(input: &RedriveSingleInput) -> Result<(), Error> { + if input.queue.is_empty() { + return Err(Error::Handler("Queue name is required".to_string())); + } + if input.message_id.is_empty() { + return Err(Error::Handler("Message ID is required".to_string())); + } + Ok(()) +} + +fn dlq_message_from_job(job: Job) -> DlqMessage { + let size_bytes = serde_json::to_vec(&job.payload).map_or(0, |bytes| bytes.len() as u64); + DlqMessage { + id: job.id, + payload: job.payload, + error: "function call failed".to_string(), + failed_at: job.enqueued_at_ms, + retries: job.attempts, + size_bytes, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::{InMemoryStore, QueueStore}; + use serde_json::json; + + fn store() -> Arc { + Arc::new(InMemoryStore::new()) + } + + async fn move_one_to_dlq(store: Arc, queue: &str) -> String { + store.enqueue(queue, json!({"dead": true})).await.unwrap(); + let job = store.dequeue(queue).await.unwrap(); + let id = job.id.clone(); + store.nack(queue, job, 1, 1).await; + id + } + + #[tokio::test] + async fn publish_enqueues_message() { + let store = store(); + publish( + store.clone(), + PublishInput { + topic: "demo".to_string(), + data: json!({"hello": "world"}), + }, + ) + .await + .unwrap(); + assert_eq!( + store.dequeue("demo").await.unwrap().payload, + json!({"hello": "world"}) + ); + } + + #[tokio::test] + async fn redrive_moves_dlq_back() { + let store = store(); + move_one_to_dlq(store.clone(), "demo").await; + let result = redrive( + store.clone(), + RedriveInput { + queue: "demo".to_string(), + }, + ) + .await + .unwrap(); + assert_eq!(result.redriven, 1); + assert!(store.dequeue("demo").await.is_some()); + } + + #[tokio::test] + async fn redrive_message_moves_one_dlq_message() { + let store = store(); + let id = move_one_to_dlq(store.clone(), "demo").await; + let result = redrive_message( + store.clone(), + RedriveSingleInput { + queue: "demo".to_string(), + message_id: id.clone(), + }, + ) + .await + .unwrap(); + assert_eq!(result.redriven, 1); + assert_eq!(store.dequeue("demo").await.unwrap().id, id); + } + + #[tokio::test] + async fn discard_message_purges_one_dlq_message() { + let store = store(); + let id = move_one_to_dlq(store.clone(), "demo").await; + let result = discard_message( + store.clone(), + RedriveSingleInput { + queue: "demo".to_string(), + message_id: id, + }, + ) + .await + .unwrap(); + assert_eq!(result.redriven, 1); + assert!(store.dlq_messages("demo", 10).await.is_empty()); + } + + #[tokio::test] + async fn list_and_stats_return_store_state() { + let store = store(); + store.enqueue("demo", json!("ready")).await.unwrap(); + move_one_to_dlq(store.clone(), "demo").await; + assert_eq!( + list_topics(store.clone()).await.unwrap()[0], + TopicInfo { + name: "demo".to_string(), + broker_type: "builtin".to_string(), + subscriber_count: 0, + } + ); + let stats = topic_stats( + store.clone(), + TopicStatsInput { + topic: "demo".to_string(), + }, + ) + .await + .unwrap(); + assert_eq!(stats.depth, 1); + assert_eq!(stats.dlq_depth, 1); + } + + #[tokio::test] + async fn dlq_browse_returns_topics_and_messages() { + let store = store(); + let id = move_one_to_dlq(store.clone(), "demo").await; + assert_eq!( + dlq_topics(store.clone()).await.unwrap(), + vec![DlqTopicInfo { + topic: "demo".to_string(), + broker_type: "builtin".to_string(), + message_count: 1, + }] + ); + let messages = dlq_messages( + store, + DlqMessagesInput { + topic: "demo".to_string(), + offset: 0, + limit: 50, + }, + ) + .await + .unwrap(); + assert_eq!(messages[0].id, id); + assert_eq!(messages[0].retries, 1); + } +} From ec4bbed1c2b300bc141400d7766ca17ca0175550 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 08:52:04 -0300 Subject: [PATCH 06/28] feat(queue): boot with builtin iii-queue guard and trigger type registration --- queue/src/boot.rs | 202 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) diff --git a/queue/src/boot.rs b/queue/src/boot.rs index 615bc2dee..ab75d5079 100644 --- a/queue/src/boot.rs +++ b/queue/src/boot.rs @@ -1 +1,203 @@ //! Worker boot and shutdown wiring. + +use std::sync::Arc; + +use iii_sdk::protocol::TriggerRequest; +use iii_sdk::{IIIClient, RegisterTriggerType}; +use serde::Deserialize; +use tokio::sync::{Mutex, RwLock}; + +use crate::config::QueueConfig; +use crate::store::{FileStore, InMemoryStore, QueueStore}; +use crate::trigger::{IiiInvoker, QueueTriggerHandler, SubscriberSpec}; +use crate::TRIGGER_TYPE; + +const LIST_WORKERS_FUNCTION_ID: &str = "engine::workers::list"; +pub const BUILTIN_III_QUEUE_WORKER_ID: &str = "iii-queue"; + +pub type ConfigCell = Arc>>; +pub type ApplyLock = Arc>; + +pub struct BootHandle { + pub store: Arc, + pub trigger_handler: QueueTriggerHandler, + pub config: ConfigCell, + pub apply_lock: ApplyLock, +} + +impl BootHandle { + pub async fn shutdown(self) { + self.trigger_handler.shutdown().await; + } +} + +pub async fn start(iii: Arc, config: QueueConfig) -> anyhow::Result { + guard_against_builtin_iii_queue(&iii).await?; + + let store = build_store(&config).await?; + let config = Arc::new(RwLock::new(Arc::new(config.normalized()))); + let apply_lock = Arc::new(Mutex::new(())); + + crate::functions::register_all(&iii, store.clone()); + + let trigger_handler = + QueueTriggerHandler::new(store.clone(), Arc::new(IiiInvoker::new(iii.clone()))); + let _ = iii.register_trigger_type( + RegisterTriggerType::new( + TRIGGER_TYPE, + "Durable queue subscriber", + trigger_handler.clone(), + ) + .trigger_request_format::(), + ); + + Ok(BootHandle { + store, + trigger_handler, + config, + apply_lock, + }) +} + +pub async fn build_store(config: &QueueConfig) -> anyhow::Result> { + let adapter_name = config.effective_adapter_name(); + if adapter_name != "builtin" && adapter_name != "file_based" && adapter_name != "in_memory" { + anyhow::bail!( + "queue adapter '{adapter_name}' is not implemented by the standalone queue worker yet" + ); + } + + let adapter_config = config + .adapter + .as_ref() + .and_then(|adapter| adapter.config.clone()) + .unwrap_or(serde_json::Value::Null); + let builtin = serde_json::from_value::(adapter_config)?; + + let store_method = builtin.store_method.as_deref().unwrap_or(adapter_name); + match store_method { + "file_based" => { + let path = builtin + .file_path + .unwrap_or_else(|| "queue_store_data".to_string()); + let save_interval_ms = builtin.save_interval_ms.unwrap_or(5000); + Ok(Arc::new(FileStore::open(path, save_interval_ms).await?)) + } + "builtin" | "in_memory" => Ok(Arc::new(InMemoryStore::new())), + other => anyhow::bail!("unknown builtin queue store_method '{other}'"), + } +} + +async fn guard_against_builtin_iii_queue(iii: &Arc) -> anyhow::Result<()> { + let workers_list = iii + .trigger(TriggerRequest { + function_id: LIST_WORKERS_FUNCTION_ID.to_string(), + payload: serde_json::json!({}), + action: None, + timeout_ms: Some(5000), + }) + .await + .map_err(|e| anyhow::anyhow!("failed to query {LIST_WORKERS_FUNCTION_ID}: {e}"))?; + + if builtin_iii_queue_active(&workers_list) { + anyhow::bail!( + "cannot start the queue worker: the built-in iii-queue worker is active and owns the \ + 'durable:subscriber' trigger type. Remove iii-queue from the engine config, then \ + start this worker." + ); + } + + Ok(()) +} + +fn builtin_iii_queue_active(workers_list: &serde_json::Value) -> bool { + workers_list + .get("workers") + .and_then(|workers| workers.as_array()) + .is_some_and(|workers| { + workers.iter().any(|worker| { + worker.get("id").and_then(|v| v.as_str()) == Some(BUILTIN_III_QUEUE_WORKER_ID) + || worker.get("name").and_then(|v| v.as_str()) + == Some(BUILTIN_III_QUEUE_WORKER_ID) + }) + }) +} + +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct BuiltinAdapterConfig { + #[serde(default)] + store_method: Option, + #[serde(default)] + file_path: Option, + #[serde(default)] + save_interval_ms: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::AdapterEntry; + use serde_json::json; + use uuid::Uuid; + + #[test] + fn detects_builtin_iii_queue_by_id() { + let workers_list = json!({ + "workers": [ + { "id": "iii-queue", "name": "queue builtin", "status": "connected" }, + { "id": "shell", "name": "shell", "status": "connected" }, + ] + }); + assert!(builtin_iii_queue_active(&workers_list)); + } + + #[test] + fn detects_builtin_iii_queue_by_name() { + let workers_list = json!({ + "workers": [ + { "id": "worker-123", "name": "iii-queue", "status": "connected" }, + ] + }); + assert!(builtin_iii_queue_active(&workers_list)); + } + + #[test] + fn no_match_when_absent() { + let workers_list = json!({ + "workers": [ + { "id": "shell", "name": "shell", "status": "connected" }, + ] + }); + assert!(!builtin_iii_queue_active(&workers_list)); + } + + #[test] + fn no_match_when_workers_empty() { + assert!(!builtin_iii_queue_active(&json!({ "workers": [] }))); + } + + #[test] + fn no_match_when_workers_missing() { + assert!(!builtin_iii_queue_active(&json!({}))); + } + + #[tokio::test] + async fn build_store_accepts_file_based_config() { + let dir = std::env::temp_dir().join(format!("queue_boot_{}", Uuid::new_v4())); + let config = QueueConfig { + adapter: Some(AdapterEntry { + name: "builtin".to_string(), + config: Some(json!({ + "store_method": "file_based", + "file_path": dir.to_string_lossy(), + "save_interval_ms": 5 + })), + }), + }; + let store = build_store(&config).await.unwrap(); + store.enqueue("demo", json!({"ok": true})).await.unwrap(); + assert!(dir.join("queue_store.json").exists()); + let _ = std::fs::remove_dir_all(dir); + } +} From 730fafb81e9fe3186bd7df5d731ce4cb5ba00cdf Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 08:54:29 -0300 Subject: [PATCH 07/28] feat(queue): configuration integration with transport hot-swap --- queue/src/boot.rs | 6 +- queue/src/configuration.rs | 246 +++++++++++++++++++++++++++++++++++++ queue/src/store.rs | 79 +++++++++++- queue/src/trigger.rs | 7 +- 4 files changed, 332 insertions(+), 6 deletions(-) diff --git a/queue/src/boot.rs b/queue/src/boot.rs index ab75d5079..8b8af93af 100644 --- a/queue/src/boot.rs +++ b/queue/src/boot.rs @@ -8,7 +8,7 @@ use serde::Deserialize; use tokio::sync::{Mutex, RwLock}; use crate::config::QueueConfig; -use crate::store::{FileStore, InMemoryStore, QueueStore}; +use crate::store::{FileStore, InMemoryStore, QueueStore, SwappableStore}; use crate::trigger::{IiiInvoker, QueueTriggerHandler, SubscriberSpec}; use crate::TRIGGER_TYPE; @@ -19,7 +19,7 @@ pub type ConfigCell = Arc>>; pub type ApplyLock = Arc>; pub struct BootHandle { - pub store: Arc, + pub store: Arc, pub trigger_handler: QueueTriggerHandler, pub config: ConfigCell, pub apply_lock: ApplyLock, @@ -34,7 +34,7 @@ impl BootHandle { pub async fn start(iii: Arc, config: QueueConfig) -> anyhow::Result { guard_against_builtin_iii_queue(&iii).await?; - let store = build_store(&config).await?; + let store = Arc::new(SwappableStore::new(build_store(&config).await?)); let config = Arc::new(RwLock::new(Arc::new(config.normalized()))); let apply_lock = Arc::new(Mutex::new(())); diff --git a/queue/src/configuration.rs b/queue/src/configuration.rs index 8650a73b7..484cc710e 100644 --- a/queue/src/configuration.rs +++ b/queue/src/configuration.rs @@ -1 +1,247 @@ //! Configuration worker integration. + +use std::sync::Arc; +use std::time::Duration; + +use iii_sdk::errors::Error; +use iii_sdk::protocol::{RegisterTriggerInput, TriggerRequest}; +use iii_sdk::{IIIClient, RegisterFunction}; +use serde_json::{json, Value}; + +use crate::boot::{ApplyLock, ConfigCell}; +use crate::config::QueueConfig; +use crate::store::SwappableStore; +use crate::trigger::QueueTriggerHandler; + +pub const CONFIG_ID: &str = "queue"; +pub const CONFIG_FN_ID: &str = "queue::on-config-change"; +const CONFIG_RETRIES: u32 = 3; +const CONFIG_RETRY_BACKOFF_MS: u64 = 250; +const CONFIG_BUS_TIMEOUT_MS: u64 = 10_000; + +pub fn new_cell(config: QueueConfig) -> ConfigCell { + Arc::new(tokio::sync::RwLock::new(Arc::new(config.normalized()))) +} + +pub async fn register_config(iii: &IIIClient, seed: Option<&QueueConfig>) -> Result<(), String> { + let mut payload = json!({ + "id": CONFIG_ID, + "name": "Queue", + "description": "Durable queue worker settings: in-process/file-backed transport persistence.", + "schema": QueueConfig::json_schema(), + }); + if should_seed_initial_value(iii).await? { + let seed = seed.cloned().unwrap_or_default().normalized(); + payload["initial_value"] = seed.to_json(); + } + trigger_with_retry( + iii, + "configuration::register", + payload, + CONFIG_BUS_TIMEOUT_MS, + ) + .await?; + Ok(()) +} + +pub async fn fetch_config(iii: &IIIClient) -> Result { + match try_get_config_value(iii).await? { + Some(value) if !value.is_null() => QueueConfig::from_json(&value).map(|c| c.normalized()), + _ => { + tracing::info!("no `{CONFIG_ID}` configuration value stored; using built-in default"); + Ok(QueueConfig::default()) + } + } +} + +pub fn register_config_trigger( + iii: &Arc, + store: Arc, + trigger_handler: QueueTriggerHandler, + config: ConfigCell, + apply_lock: ApplyLock, +) -> Result<(), Error> { + let engine = iii.clone(); + iii.register_function( + CONFIG_FN_ID, + RegisterFunction::new_async(move |_payload: ConfigChangeRequest| { + let engine = engine.clone(); + let store = store.clone(); + let trigger_handler = trigger_handler.clone(); + let config = config.clone(); + let apply_lock = apply_lock.clone(); + async move { + on_config_change(&engine, store, trigger_handler, config, apply_lock).await; + Ok::<_, Error>(ConfigChangeAck { ok: true }) + } + }) + .description("Internal: reload queue configuration from the authoritative store."), + ); + + iii.register_trigger(RegisterTriggerInput { + trigger_type: "configuration".to_string(), + function_id: CONFIG_FN_ID.to_string(), + config: json!({ + "configuration_id": CONFIG_ID, + "event_types": ["configuration:updated"], + }), + metadata: None, + })?; + Ok(()) +} + +async fn on_config_change( + iii: &IIIClient, + store: Arc, + trigger_handler: QueueTriggerHandler, + config: ConfigCell, + apply_lock: ApplyLock, +) { + let _guard = apply_lock.lock().await; + + let next = match fetch_config(iii).await { + Ok(config) => config.normalized(), + Err(err) => { + tracing::error!(error = %err, "queue config-change: fetch failed; keeping previous config"); + return; + } + }; + + let old = config.read().await.clone(); + if swap_needed(&old, &next) { + let registrations = trigger_handler.registrations().await; + let new_store = match crate::boot::build_store(&next).await { + Ok(store) => store, + Err(err) => { + tracing::error!(error = %err, "queue config-change: failed to build replacement store; keeping previous config"); + return; + } + }; + trigger_handler.shutdown().await; + store.replace(new_store).await; + *config.write().await = Arc::new(next); + + for registration in registrations { + if let Err(err) = trigger_handler.register_subscriber(registration).await { + tracing::error!(error = %err, "queue config-change: failed to restart subscriber"); + } + } + tracing::info!("queue transport hot-swapped after configuration change"); + return; + } + + *config.write().await = Arc::new(next); + tracing::info!("queue configuration reloaded without transport swap"); +} + +pub fn swap_needed(old: &QueueConfig, new: &QueueConfig) -> bool { + effective_adapter(old) != effective_adapter(new) +} + +fn effective_adapter(config: &QueueConfig) -> (String, Value) { + ( + config.effective_adapter_name().to_string(), + config + .adapter + .as_ref() + .and_then(|adapter| adapter.config.clone()) + .unwrap_or(Value::Null), + ) +} + +async fn should_seed_initial_value(iii: &IIIClient) -> Result { + match try_get_config_value(iii).await? { + Some(value) if !value.is_null() => Ok(false), + _ => Ok(true), + } +} + +async fn try_get_config_value(iii: &IIIClient) -> Result, String> { + match trigger_with_retry( + iii, + "configuration::get", + json!({ "id": CONFIG_ID }), + CONFIG_BUS_TIMEOUT_MS, + ) + .await + { + Ok(resp) => Ok(resp.get("value").cloned()), + Err(e) if e.to_ascii_uppercase().contains("NOT_FOUND") => Ok(None), + Err(e) => Err(e), + } +} + +async fn trigger_with_retry( + iii: &IIIClient, + function_id: &str, + payload: Value, + timeout_ms: u64, +) -> Result { + let mut last_err = String::new(); + for attempt in 1..=CONFIG_RETRIES { + match iii + .trigger(TriggerRequest { + function_id: function_id.to_string(), + payload: payload.clone(), + action: None, + timeout_ms: Some(timeout_ms), + }) + .await + { + Ok(value) => return Ok(value), + Err(err) => { + last_err = err.to_string(); + if attempt < CONFIG_RETRIES { + tokio::time::sleep(Duration::from_millis( + CONFIG_RETRY_BACKOFF_MS * u64::from(attempt), + )) + .await; + } + } + } + } + Err(format!( + "{function_id} failed after {CONFIG_RETRIES} attempts: {last_err}" + )) +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] +pub struct ConfigChangeAck { + pub ok: bool, +} + +#[derive(Debug, Default, Clone, serde::Deserialize, schemars::JsonSchema)] +pub struct ConfigChangeRequest {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::AdapterEntry; + + #[test] + fn swap_needed_ignores_implicit_builtin_default() { + let old = QueueConfig::default(); + let new = QueueConfig { + adapter: Some(AdapterEntry { + name: "builtin".to_string(), + config: None, + }), + }; + assert!(!swap_needed(&old, &new)); + } + + #[test] + fn swap_needed_when_adapter_config_changes() { + let old = QueueConfig::default(); + let new = QueueConfig { + adapter: Some(AdapterEntry { + name: "builtin".to_string(), + config: Some(json!({ + "store_method": "file_based", + "file_path": "./data/queue" + })), + }), + }; + assert!(swap_needed(&old, &new)); + } +} diff --git a/queue/src/store.rs b/queue/src/store.rs index 4b956f5a6..2e6096f05 100644 --- a/queue/src/store.rs +++ b/queue/src/store.rs @@ -15,7 +15,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::Value; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, RwLock}; use uuid::Uuid; const STORE_FILE_NAME: &str = "queue_store.json"; @@ -53,6 +53,83 @@ pub trait QueueStore: Send + Sync + 'static { async fn discard_dlq_message(&self, topic: &str, job_id: &str) -> bool; } +#[derive(Clone)] +pub struct SwappableStore { + inner: Arc>>, +} + +impl SwappableStore { + pub fn new(store: Arc) -> Self { + Self { + inner: Arc::new(RwLock::new(store)), + } + } + + pub async fn current(&self) -> Arc { + self.inner.read().await.clone() + } + + pub async fn replace(&self, store: Arc) { + *self.inner.write().await = store; + } +} + +#[async_trait] +impl QueueStore for SwappableStore { + async fn enqueue(&self, topic: &str, payload: Value) -> anyhow::Result { + self.current().await.enqueue(topic, payload).await + } + + async fn dequeue(&self, topic: &str) -> Option { + self.current().await.dequeue(topic).await + } + + async fn ack(&self, topic: &str, job_id: &str) { + self.current().await.ack(topic, job_id).await; + } + + async fn nack(&self, topic: &str, job: Job, max_retries: u32, backoff_ms: u64) { + self.current() + .await + .nack(topic, job, max_retries, backoff_ms) + .await; + } + + async fn list_topics(&self) -> Vec { + self.current().await.list_topics().await + } + + async fn topic_stats(&self, topic: &str) -> TopicStats { + self.current().await.topic_stats(topic).await + } + + async fn dlq_topics(&self) -> Vec<(String, u64)> { + self.current().await.dlq_topics().await + } + + async fn dlq_messages(&self, topic: &str, limit: u64) -> Vec { + self.current().await.dlq_messages(topic, limit).await + } + + async fn redrive_dlq(&self, topic: &str) -> u64 { + self.current().await.redrive_dlq(topic).await + } + + async fn redrive_dlq_message(&self, topic: &str, job_id: &str) -> bool { + self.current() + .await + .redrive_dlq_message(topic, job_id) + .await + } + + async fn discard_dlq_message(&self, topic: &str, job_id: &str) -> bool { + self.current() + .await + .discard_dlq_message(topic, job_id) + .await + } +} + #[derive(Debug, Clone, Default, Serialize, Deserialize)] struct StoreData { queues: HashMap>, diff --git a/queue/src/trigger.rs b/queue/src/trigger.rs index 65c54299d..319e61524 100644 --- a/queue/src/trigger.rs +++ b/queue/src/trigger.rs @@ -128,7 +128,10 @@ impl QueueTriggerHandler { } } - async fn register(&self, registration: RegisteredSubscriber) -> Result<(), String> { + pub async fn register_subscriber( + &self, + registration: RegisteredSubscriber, + ) -> Result<(), String> { if registration.spec.queue.trim().is_empty() { return Err("queue is required for durable:subscriber trigger".to_string()); } @@ -162,7 +165,7 @@ impl TriggerHandler for QueueTriggerHandler { async fn register_trigger(&self, config: TriggerConfig) -> Result<(), Error> { let spec: SubscriberSpec = serde_json::from_value(config.config.clone()) .map_err(|e| Error::Handler(format!("invalid durable:subscriber config: {e}")))?; - self.register(RegisteredSubscriber { + self.register_subscriber(RegisteredSubscriber { trigger_id: config.id, function_id: config.function_id, spec, From 0650964d80b550e17d3d9c1172e1e646d0c7fe2a Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 08:55:52 -0300 Subject: [PATCH 08/28] feat(queue): binary entry with seed config and manifest output --- queue/src/main.rs | 106 +++++++++++++++++++++++++++++++++++++++++- queue/src/manifest.rs | 57 +++++++++++++++++++++++ 2 files changed, 161 insertions(+), 2 deletions(-) diff --git a/queue/src/main.rs b/queue/src/main.rs index 284177ae8..a2e693bf7 100644 --- a/queue/src/main.rs +++ b/queue/src/main.rs @@ -1,3 +1,105 @@ -fn main() { - println!("queue worker - wiring lands in a later task on this branch"); +//! `queue` binary entry. + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use clap::Parser; +use iii_queue::config::QueueConfig; +use iii_queue::configuration; +use iii_sdk::runtime::WorkerMetadata; +use iii_sdk::{register_worker, InitOptions}; + +#[derive(Parser, Debug)] +#[command(name = "queue", about = "Durable queue worker for iii.")] +struct Cli { + /// Optional seed config.yaml. This seeds the configuration entry only when + /// nothing is stored yet; thereafter the configuration worker is + /// authoritative. + #[arg(long)] + config: Option, + #[arg(long, default_value = "ws://127.0.0.1:49134")] + url: String, + #[arg(long)] + manifest: bool, +} + +fn worker_metadata() -> WorkerMetadata { + WorkerMetadata { + runtime: "rust".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + name: "queue".to_string(), + os: std::env::consts::OS.to_string(), + pid: Some(std::process::id()), + telemetry: None, + ..WorkerMetadata::default() + } +} + +#[tokio::main] +async fn main() -> Result<()> { + let filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); + tracing_subscriber::fmt().with_env_filter(filter).init(); + + let cli = Cli::parse(); + if cli.manifest { + println!( + "{}", + serde_json::to_string_pretty(&iii_queue::manifest::build_manifest()).unwrap() + ); + return Ok(()); + } + + let seed: Option = match cli.config.as_deref() { + Some(path) => match std::fs::read_to_string(path) + .map_err(|e| e.to_string()) + .and_then(|s| serde_yaml::from_str::(&s).map_err(|e| e.to_string())) + { + Ok(config) => { + tracing::info!(path = %path, "loaded seed config"); + Some(config.normalized()) + } + Err(err) => { + tracing::warn!(path = %path, error = %err, "failed to load --config seed; continuing without seed"); + None + } + }, + None => None, + }; + + let iii = Arc::new(register_worker( + &cli.url, + InitOptions { + metadata: Some(worker_metadata()), + ..InitOptions::default() + }, + )); + + configuration::register_config(&iii, seed.as_ref()) + .await + .map_err(anyhow::Error::msg) + .context("registering queue configuration schema")?; + let config = configuration::fetch_config(&iii) + .await + .map_err(anyhow::Error::msg) + .context("loading queue configuration")?; + + let boot = iii_queue::boot::start(iii.clone(), config).await?; + tracing::info!("queue worker ready"); + + configuration::register_config_trigger( + &iii, + boot.store.clone(), + boot.trigger_handler.clone(), + boot.config.clone(), + boot.apply_lock.clone(), + ) + .map_err(anyhow::Error::msg) + .context("binding queue configuration trigger")?; + + tokio::signal::ctrl_c().await?; + tracing::info!("queue worker shutting down"); + boot.shutdown().await; + iii.shutdown_async().await; + Ok(()) } diff --git a/queue/src/manifest.rs b/queue/src/manifest.rs index 8655b32bd..58a9291a1 100644 --- a/queue/src/manifest.rs +++ b/queue/src/manifest.rs @@ -1 +1,58 @@ //! Worker manifest generation. + +use serde::Serialize; + +use crate::config::QueueConfig; +use crate::functions::{ + DISCARD_MESSAGE_FN_ID, DLQ_MESSAGES_FN_ID, DLQ_TOPICS_FN_ID, LIST_TOPICS_FN_ID, PUBLISH_FN_ID, + REDRIVE_FN_ID, REDRIVE_MESSAGE_FN_ID, TOPIC_STATS_FN_ID, +}; +use crate::TRIGGER_TYPE; + +#[derive(Debug, Serialize)] +pub struct ModuleManifest { + pub name: String, + pub version: String, + pub description: String, + pub default_config: serde_json::Value, + pub config_schema: serde_json::Value, + pub trigger_types: Vec, + pub functions: Vec, + pub supported_targets: Vec, +} + +pub fn build_manifest() -> ModuleManifest { + ModuleManifest { + name: "queue".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + description: "Durable queue worker for iii.".to_string(), + default_config: QueueConfig::default().to_json(), + config_schema: QueueConfig::json_schema(), + trigger_types: vec![TRIGGER_TYPE.to_string()], + functions: vec![ + PUBLISH_FN_ID.to_string(), + REDRIVE_FN_ID.to_string(), + REDRIVE_MESSAGE_FN_ID.to_string(), + DISCARD_MESSAGE_FN_ID.to_string(), + LIST_TOPICS_FN_ID.to_string(), + TOPIC_STATS_FN_ID.to_string(), + DLQ_TOPICS_FN_ID.to_string(), + DLQ_MESSAGES_FN_ID.to_string(), + ], + supported_targets: vec![option_env!("TARGET").unwrap_or("unknown").to_string()], + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manifest_has_queue_identity_and_interfaces() { + let value = serde_json::to_value(build_manifest()).unwrap(); + assert_eq!(value["name"], "queue"); + assert_eq!(value["trigger_types"][0], crate::TRIGGER_TYPE); + assert!(value["functions"].as_array().unwrap().len() >= 8); + assert!(value["config_schema"].is_object()); + } +} From 87439100c9dfaa4edd728fb3c38ceb3b767dfff5 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 08:58:53 -0300 Subject: [PATCH 09/28] test(queue): e2e delivery, DLQ redrive, and restart survival --- queue/tests/common/engine.rs | 105 ++++++++++++++++++ queue/tests/common/mod.rs | 1 + queue/tests/e2e_durability.rs | 199 ++++++++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+) create mode 100644 queue/tests/common/engine.rs create mode 100644 queue/tests/common/mod.rs create mode 100644 queue/tests/e2e_durability.rs diff --git a/queue/tests/common/engine.rs b/queue/tests/common/engine.rs new file mode 100644 index 000000000..45097d389 --- /dev/null +++ b/queue/tests/common/engine.rs @@ -0,0 +1,105 @@ +#![allow(dead_code)] +//! Connect-or-skip wrapper around the iii SDK. +//! +//! These are e2e tests: they only make sense with a running engine. By +//! default, when the engine is unreachable, `get_or_init`/`connect_fresh` +//! return `None` and the caller skips the test — this keeps CI (which runs +//! with no engine) and casual local runs green. Set `III_E2E_REQUIRE` to any +//! value to fail loudly instead (for intentional e2e runs where a missing +//! engine should be treated as a bug). +//! +//! One engine connection per test binary process via `OnceCell` — the +//! WebSocket handshake dwarfs per-test overhead. +//! +//! Point the tests at an engine with `III_ENGINE_WS_URL` (default +//! `ws://127.0.0.1:49134`). + +use std::sync::Arc; +use std::time::Duration; + +use iii_sdk::errors::Error; +use iii_sdk::protocol::TriggerRequest; +use iii_sdk::{register_worker, IIIClient, InitOptions}; +use serde_json::json; +use tokio::sync::OnceCell; + +const DEFAULT_WS_URL: &str = "ws://127.0.0.1:49134"; + +static ENGINE: OnceCell>> = OnceCell::const_new(); + +pub fn ws_url() -> String { + std::env::var("III_ENGINE_WS_URL").unwrap_or_else(|_| DEFAULT_WS_URL.to_string()) +} + +/// Turn a failed connection attempt into either a panic (when +/// `III_E2E_REQUIRE` is set — an intentional e2e run that must fail loudly +/// on a missing engine) or a skip (the default — logs and returns `None` so +/// the caller can bail out of the test without failing it). +fn require_or_skip(connected: Option>) -> Option> { + if connected.is_some() { + return connected; + } + + if std::env::var("III_E2E_REQUIRE").is_ok() { + panic!( + "e2e requires a running iii engine at {} — start `iii` or set \ + III_ENGINE_WS_URL to point at one", + ws_url() + ); + } + + eprintln!( + "[skip] no iii engine reachable at {} — skipping e2e test (set III_E2E_REQUIRE=1 to \ + fail instead)", + ws_url() + ); + None +} + +async fn try_connect_raw() -> Option> { + let url = ws_url(); + let iii = Arc::new(register_worker(&url, InitOptions::default())); + + for _ in 0..20 { + tokio::time::sleep(Duration::from_millis(250)).await; + let probe = iii + .trigger(TriggerRequest { + function_id: "engine::workers::list".to_string(), + payload: json!({}), + action: None, + timeout_ms: Some(800), + }) + .await; + match probe { + Ok(_) => return Some(iii), + Err(Error::NotConnected) => continue, + Err(_) => continue, + } + } + + iii.shutdown_async().await; + None +} + +/// Open a fresh, dedicated engine connection (NOT the shared `OnceCell` one). +/// +/// Needed when a test registers a function id that another test in the same +/// binary also registers (e.g. the `http::on-config-change` reload function): +/// registering the same id twice on one client panics, so such a test needs its +/// own client. Returns `None` (the caller should skip the test) if no engine +/// is reachable, unless `III_E2E_REQUIRE` is set, in which case it panics. +/// The caller should `shutdown_async()` the returned client when done. +pub async fn connect_fresh() -> Option> { + require_or_skip(try_connect_raw().await) +} + +/// Get-or-init the shared engine handle for this test binary. +/// +/// Returns `None` (the caller should skip the test) if no engine is +/// reachable, unless `III_E2E_REQUIRE` is set, in which case it panics. +pub async fn get_or_init() -> Option> { + ENGINE + .get_or_init(|| async { require_or_skip(try_connect_raw().await) }) + .await + .clone() +} diff --git a/queue/tests/common/mod.rs b/queue/tests/common/mod.rs new file mode 100644 index 000000000..702e611f1 --- /dev/null +++ b/queue/tests/common/mod.rs @@ -0,0 +1 @@ +pub mod engine; diff --git a/queue/tests/e2e_durability.rs b/queue/tests/e2e_durability.rs new file mode 100644 index 000000000..6334ee3d0 --- /dev/null +++ b/queue/tests/e2e_durability.rs @@ -0,0 +1,199 @@ +mod common; + +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use iii_queue::config::{AdapterEntry, QueueConfig}; +use iii_queue::functions::{DLQ_MESSAGES_FN_ID, PUBLISH_FN_ID, REDRIVE_FN_ID}; +use iii_queue::TRIGGER_TYPE; +use iii_sdk::errors::Error; +use iii_sdk::protocol::{RegisterTriggerInput, TriggerRequest}; +use iii_sdk::{IIIClient, RegisterFunction}; +use serde_json::{json, Value}; +use serial_test::serial; +use tokio::time::Instant; +use uuid::Uuid; + +use common::engine; + +fn temp_store_dir() -> std::path::PathBuf { + std::env::temp_dir().join(format!("queue_e2e_{}", Uuid::new_v4())) +} + +fn file_config(dir: &std::path::Path) -> QueueConfig { + QueueConfig { + adapter: Some(AdapterEntry { + name: "builtin".to_string(), + config: Some(json!({ + "store_method": "file_based", + "file_path": dir.to_string_lossy(), + "save_interval_ms": 5 + })), + }), + } +} + +fn register_counting_function( + iii: &Arc, + function_id: &str, + fires: Arc, + fail: Arc, +) { + iii.register_function( + function_id, + RegisterFunction::new_async(move |_payload: Value| { + let fires = fires.clone(); + let fail = fail.clone(); + async move { + fires.fetch_add(1, Ordering::SeqCst); + if fail.load(Ordering::SeqCst) { + Err(Error::Handler("e2e forced failure".to_string())) + } else { + Ok::(json!({"ok": true})) + } + } + }), + ); +} + +fn register_subscriber(iii: &Arc, function_id: &str, queue: &str) { + iii.register_trigger(RegisterTriggerInput { + trigger_type: TRIGGER_TYPE.to_string(), + function_id: function_id.to_string(), + config: json!({ + "queue": queue, + "max_retries": 1, + "backoff_ms": 5 + }), + metadata: None, + }) + .expect("register durable subscriber trigger"); +} + +async fn trigger(iii: &IIIClient, function_id: &str, payload: Value) -> Value { + iii.trigger(TriggerRequest { + function_id: function_id.to_string(), + payload, + action: None, + timeout_ms: Some(5_000), + }) + .await + .expect("trigger should succeed") +} + +async fn wait_for_fires(fires: &AtomicUsize, expected: usize) { + let deadline = Instant::now() + Duration::from_secs(3); + while Instant::now() < deadline { + if fires.load(Ordering::SeqCst) >= expected { + return; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + assert!( + fires.load(Ordering::SeqCst) >= expected, + "expected at least {expected} fires" + ); +} + +async fn wait_for_dlq(iii: &IIIClient, queue: &str) -> Vec { + let deadline = Instant::now() + Duration::from_secs(3); + while Instant::now() < deadline { + let value = trigger( + iii, + DLQ_MESSAGES_FN_ID, + json!({ + "queue": queue, + "limit": 50 + }), + ) + .await; + let messages = serde_json::from_value::>(value).unwrap_or_default(); + if !messages.is_empty() { + return messages; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + panic!("message did not reach DLQ"); +} + +#[tokio::test] +#[serial] +async fn delivery_dlq_and_redrive_connect_or_skip() { + let Some(iii) = engine::connect_fresh().await else { + return; + }; + + let dir = temp_store_dir(); + let boot = iii_queue::boot::start(iii.clone(), file_config(&dir)) + .await + .expect("queue worker should boot"); + + let fires = Arc::new(AtomicUsize::new(0)); + let fail = Arc::new(AtomicBool::new(false)); + let function_id = format!("queue.e2e.{}", Uuid::new_v4()); + register_counting_function(&iii, &function_id, fires.clone(), fail.clone()); + register_subscriber(&iii, &function_id, "demo"); + + trigger( + &iii, + PUBLISH_FN_ID, + json!({"queue": "demo", "data": {"hello": "world"}}), + ) + .await; + wait_for_fires(&fires, 1).await; + + fail.store(true, Ordering::SeqCst); + trigger( + &iii, + PUBLISH_FN_ID, + json!({"queue": "demo", "data": {"should": "dlq"}}), + ) + .await; + let messages = wait_for_dlq(&iii, "demo").await; + assert_eq!(messages.len(), 1); + + fail.store(false, Ordering::SeqCst); + trigger(&iii, REDRIVE_FN_ID, json!({"queue": "demo"})).await; + wait_for_fires(&fires, 3).await; + + boot.shutdown().await; + iii.shutdown_async().await; + let _ = std::fs::remove_dir_all(dir); +} + +#[tokio::test] +#[serial] +async fn file_based_pending_message_survives_worker_restart_connect_or_skip() { + let Some(iii) = engine::connect_fresh().await else { + return; + }; + + let dir = temp_store_dir(); + let boot = iii_queue::boot::start(iii.clone(), file_config(&dir)) + .await + .expect("queue worker should boot"); + + trigger( + &iii, + PUBLISH_FN_ID, + json!({"queue": "restart", "data": {"survives": true}}), + ) + .await; + boot.shutdown().await; + + let boot = iii_queue::boot::start(iii.clone(), file_config(&dir)) + .await + .expect("queue worker should reboot"); + + let fires = Arc::new(AtomicUsize::new(0)); + let fail = Arc::new(AtomicBool::new(false)); + let function_id = format!("queue.restart.{}", Uuid::new_v4()); + register_counting_function(&iii, &function_id, fires.clone(), fail); + register_subscriber(&iii, &function_id, "restart"); + wait_for_fires(&fires, 1).await; + + boot.shutdown().await; + iii.shutdown_async().await; + let _ = std::fs::remove_dir_all(dir); +} From e3d0d50889715d6896adc610f3ffa9660084a0c3 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 09:00:51 -0300 Subject: [PATCH 10/28] feat(queue): release tag, README, parity checklist --- .github/workflows/release.yml | 1 + queue/README.md | 94 ++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77f64f9d5..35b0113c1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,6 +28,7 @@ on: - 'provider-openai/v*' - 'provider-xai/v*' - 'pubsub/v*' + - 'queue/v*' - 'rbac-proxy/v*' - 'session-manager/v*' - 'slack/v*' diff --git a/queue/README.md b/queue/README.md index 5d840799d..9f898be26 100644 --- a/queue/README.md +++ b/queue/README.md @@ -1,3 +1,93 @@ -# queue worker +# queue -Durable function queues for iii. The complete registry migration documentation lands later in this branch. +Durable function queues for iii. This worker registers the +`durable:subscriber` trigger type and the queue/DLQ service functions that +replace the built-in `iii-queue` worker. + +## Install + +```bash +iii worker add queue +``` + +`iii worker add` fetches the binary, writes a config block into +`~/.iii/config.yaml`, and the engine starts the worker on the next +`iii start`. + +## Trigger Type + +Bind a function to `durable:subscriber` to consume a topic/queue: + +| Field | Required | Default | Description | +|---|---|---|---| +| `queue` | yes | - | Topic/queue name to consume. `topic` is accepted as a compatibility alias. | +| `max_retries` | no | `3` | Maximum failed deliveries before the message moves to DLQ. | +| `backoff_ms` | no | `1000` | Base retry delay in milliseconds. Retries use exponential backoff. | +| `condition_function_id` | no | - | Function invoked first. Only explicit `false` skips the handler. | + +The worker also accepts the built-in subscriber `queue_config` shape for +compatibility, including `maxRetries` and `backoffDelayMs`. + +## Functions + +| Function id | Input | Output | +|---|---|---| +| `iii::durable::publish` | `{ "queue" \| "topic", "data" }` | `null` | +| `iii::queue::redrive` | `{ "queue" \| "topic" }` | `{ "queue", "redriven" }` | +| `iii::queue::redrive_message` | `{ "queue" \| "topic", "message_id" }` | `{ "queue", "message_id", "redriven" }` | +| `iii::queue::discard_message` | `{ "queue" \| "topic", "message_id" }` | `{ "queue", "message_id", "redriven" }` | +| `engine::queue::list_topics` | `{}` | topic list | +| `engine::queue::topic_stats` | `{ "topic" \| "queue" }` | `{ "depth", "consumer_count", "dlq_depth", "config" }` | +| `engine::queue::dlq_topics` | `{}` | DLQ topic list | +| `engine::queue::dlq_messages` | `{ "topic" \| "queue", "offset", "limit" }` | DLQ messages | + +## Configuration + +Configuration is owned by the `configuration` worker under id `queue`. +Seed it once with `--config .yaml`; runtime edits come from the +configuration worker after that. + +```yaml +adapter: + name: builtin + config: + store_method: file_based + file_path: ./data/queue + save_interval_ms: 5000 +``` + +| Field | Default | Description | +|---|---|---| +| `adapter.name` | `builtin` | In-process queue transport. `redis` and `rabbitmq` are follow-up transports. | +| `adapter.config.store_method` | `in_memory` | `in_memory` or `file_based`. | +| `adapter.config.file_path` | `queue_store_data` | Directory used by `file_based`. | +| `adapter.config.save_interval_ms` | `5000` | Accepted for parity; this worker persists on mutation. | + +Changing the adapter config hot-swaps the transport and restarts every +consumer. Pending in-memory jobs are lost on swap/restart, matching the +built-in adapter's in-process durability profile. File-backed jobs survive. + +## Requires Removing The Built-In `iii-queue` Worker + +The built-in `iii-queue` worker also owns `durable:subscriber`. Two owners of +the same trigger type on one engine collide, so this worker requires +`iii-queue` to be absent from the engine config. + +On boot, this worker queries `engine::workers::list` and refuses to start if +`iii-queue` is active. + +## Parity Vs Builtin + +| Behavior | Builtin | This worker | +|---|---|---| +| Trigger type | `durable:subscriber` | same | +| Function ids | 8 public ids listed above | same, verbatim | +| Retry | `max_retries` + exponential `backoff_ms * 2^(attempts - 1)` | same | +| DLQ | after retries exhausted; redrive/redrive_message/discard | same | +| Restart survival | file-backed store | same (`file_based`) | +| In-memory mode | jobs lost on restart/swap | same | +| Transports | builtin/memory/redis/rabbitmq | builtin memory/file; redis+rabbitmq follow-up | +| Enqueue failure when worker offline | n/a, in-process | invocation fails explicitly once the engine remote-enqueue cut lands | + +The full engine `TriggerAction::Enqueue` path depends on the separate +`QueueEnqueuer` engine cut tracked in the migration master plan. From d9d7f77759f7463e3becaa89c49f4f17f3ec8668 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 10:17:38 -0300 Subject: [PATCH 11/28] docs(queue): track latency benchmark gate --- queue/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/queue/README.md b/queue/README.md index 9f898be26..f91729fa0 100644 --- a/queue/README.md +++ b/queue/README.md @@ -88,6 +88,7 @@ On boot, this worker queries `engine::workers::list` and refuses to start if | In-memory mode | jobs lost on restart/swap | same | | Transports | builtin/memory/redis/rabbitmq | builtin memory/file; redis+rabbitmq follow-up | | Enqueue failure when worker offline | n/a, in-process | invocation fails explicitly once the engine remote-enqueue cut lands | +| Latency benchmark | in-process baseline required | tracked in `queue/benches/enqueue_latency.md`; final budget is a pre-deprecation gate | The full engine `TriggerAction::Enqueue` path depends on the separate `QueueEnqueuer` engine cut tracked in the migration master plan. From bfc96d6204687bfa558d93b979ce07fe0e4f7425 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Fri, 3 Jul 2026 16:19:53 -0300 Subject: [PATCH 12/28] feat(queue): add published skill and create-tag option --- .github/workflows/create-tag.yml | 1 + queue/skills/SKILL.md | 105 +++++++++++++++++++++++++++++++ queue/tests/e2e_durability.rs | 4 ++ 3 files changed, 110 insertions(+) create mode 100644 queue/skills/SKILL.md diff --git a/.github/workflows/create-tag.yml b/.github/workflows/create-tag.yml index fa8641309..73f638d77 100644 --- a/.github/workflows/create-tag.yml +++ b/.github/workflows/create-tag.yml @@ -34,6 +34,7 @@ on: - provider-openai - provider-xai - pubsub + - queue - rbac-proxy - session-manager - slack diff --git a/queue/skills/SKILL.md b/queue/skills/SKILL.md new file mode 100644 index 000000000..41151e283 --- /dev/null +++ b/queue/skills/SKILL.md @@ -0,0 +1,105 @@ +--- +name: queue +tags: queue, durable, retries, dlq, pubsub +description: >- + Durable topic queues for iii: subscribe functions to topics, publish work + asynchronously, retry failed deliveries with backoff, and inspect/redrive + dead-lettered messages. +--- + +# queue + +The `queue` worker provides durable topic delivery for iii. Register a consumer +with a `durable:subscriber` trigger, publish with `iii::durable::publish`, and +the worker persists messages, retries failed deliveries, moves exhausted jobs to +DLQ, and exposes redrive/discard inspection functions. + +Install it with `iii worker add queue`. The built-in `iii-queue` worker must be +removed from the engine config before this worker starts because both would own +the same `durable:subscriber` trigger type. + +## When to Use + +- A topic event must be processed reliably by a subscribed function. +- A failed consumer should retry automatically before the message lands in DLQ. +- You need to inspect, redrive, or discard dead-lettered messages. +- Local or single-instance durable queues are enough, using in-memory storage + for development or file-backed storage for restart survival. + +## Boundaries + +- Not fire-and-forget broadcast. Use `pubsub` when missed events are acceptable. +- Redis and RabbitMQ transports are follow-up work; this worker currently ships + the builtin in-process transport with `in_memory` and `file_based` storage. +- Engine `TriggerAction.Enqueue` / named function queues require the separate + `QueueEnqueuer` engine cut. Until that lands, use `iii::durable::publish` and + `durable:subscriber` triggers for this standalone worker. +- File-backed mode survives worker restarts. In-memory mode loses pending jobs + on restart or transport hot-swap. +- This worker carries work items, not key/value or stream state. Use `state` or + `stream` for those. + +## Functions + +- `iii::durable::publish`: publish a JSON payload to a queue/topic. Input accepts + `{ "queue": "...", "data": ... }` and also accepts `topic` as an alias. +- `iii::queue::redrive`: move all DLQ messages for a queue back to the main + queue. Input accepts `queue` or `topic`. +- `iii::queue::redrive_message`: move one DLQ message back by `message_id`. +- `iii::queue::discard_message`: purge one DLQ message by `message_id`. +- `engine::queue::list_topics`: list known queue topics. +- `engine::queue::topic_stats`: return `depth`, `consumer_count`, and + `dlq_depth` for one topic. +- `engine::queue::dlq_topics`: list DLQ topics with message counts. +- `engine::queue::dlq_messages`: browse DLQ messages with `offset` and `limit`. + +## Reactive Triggers + +Bind a `durable:subscriber` trigger when a function should consume every +message published to a queue/topic with retries and DLQ handling. + +```typescript +iii.registerTrigger({ + type: 'durable:subscriber', + function_id: 'orders::process', + config: { + queue: 'orders.created', + max_retries: 3, + backoff_ms: 1000, + }, +}) +``` + +Trigger config: + +- `queue` (required): topic/queue name to consume. `topic` is accepted as a + compatibility alias. +- `max_retries` (default `3`): failed attempts before moving to DLQ. +- `backoff_ms` (default `1000`): base retry delay. Retries use the same + exponential curve as the builtin queue: `backoff_ms * 2^(attempts - 1)`. +- `condition_function_id` (optional): called before the handler; only explicit + `false` skips the handler. + +## Configuration + +Use the `configuration` worker entry named `queue`. The default is in-memory: + +```yaml +adapter: + name: builtin +``` + +For restart survival, use file-backed storage: + +```yaml +adapter: + name: builtin + config: + store_method: file_based + file_path: ./data/queue + save_interval_ms: 5000 +``` + +Changing the adapter config hot-swaps the transport and restarts every +consumer. File-backed jobs survive by construction; in-memory jobs are lost, +matching the builtin adapter's in-process durability profile. diff --git a/queue/tests/e2e_durability.rs b/queue/tests/e2e_durability.rs index 6334ee3d0..4ce7682ec 100644 --- a/queue/tests/e2e_durability.rs +++ b/queue/tests/e2e_durability.rs @@ -181,7 +181,11 @@ async fn file_based_pending_message_survives_worker_restart_connect_or_skip() { ) .await; boot.shutdown().await; + iii.shutdown_async().await; + let Some(iii) = engine::connect_fresh().await else { + return; + }; let boot = iii_queue::boot::start(iii.clone(), file_config(&dir)) .await .expect("queue worker should reboot"); From 884b28088955f4dd776f148cbcb35a2673b9b031 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Mon, 6 Jul 2026 19:51:36 -0300 Subject: [PATCH 13/28] =?UTF-8?q?fix(queue):=20CI=20fixes=20=E2=80=94=20cl?= =?UTF-8?q?ippy=20useless=5Fconversion,=20Null=20adapter-config=20boot,=20?= =?UTF-8?q?unique=20e2e=20queue=20names?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - store: drop redundant .into_iter() (clippy 1.96 useless_conversion) - boot: treat absent/Null adapter.config as builtin defaults instead of failing deserialization; regression tests for the no-config boot path (this was the interface-boot-smoke CI failure) - e2e: unique queue names per run — the engine re-delivers persisted durable:subscriber triggers on boot, so fixed names race against leftover consumers from earlier local runs --- queue/src/boot.rs | 29 ++++++++++++++++++++++++++--- queue/src/store.rs | 2 +- queue/tests/e2e_durability.rs | 19 ++++++++++++------- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/queue/src/boot.rs b/queue/src/boot.rs index 8b8af93af..0303887ad 100644 --- a/queue/src/boot.rs +++ b/queue/src/boot.rs @@ -67,12 +67,16 @@ pub async fn build_store(config: &QueueConfig) -> anyhow::Result(adapter_config)?; + { + Some(value) if !value.is_null() => serde_json::from_value::(value)?, + _ => BuiltinAdapterConfig::default(), + }; let store_method = builtin.store_method.as_deref().unwrap_or(adapter_name); match store_method { @@ -182,6 +186,25 @@ mod tests { assert!(!builtin_iii_queue_active(&json!({}))); } + #[tokio::test] + async fn build_store_defaults_to_in_memory_without_adapter() { + let config = QueueConfig::default(); + let store = build_store(&config).await.unwrap(); + store.enqueue("demo", json!({"ok": true})).await.unwrap(); + } + + #[tokio::test] + async fn build_store_accepts_adapter_without_config() { + let config = QueueConfig { + adapter: Some(AdapterEntry { + name: "builtin".to_string(), + config: None, + }), + }; + let store = build_store(&config).await.unwrap(); + store.enqueue("demo", json!({"ok": true})).await.unwrap(); + } + #[tokio::test] async fn build_store_accepts_file_based_config() { let dir = std::env::temp_dir().join(format!("queue_boot_{}", Uuid::new_v4())); diff --git a/queue/src/store.rs b/queue/src/store.rs index 2e6096f05..8eb6baa4d 100644 --- a/queue/src/store.rs +++ b/queue/src/store.rs @@ -415,7 +415,7 @@ async fn redrive_dlq(shared: &SharedStore, topic: &str) -> u64 { data.queues .entry(topic.to_string()) .or_default() - .extend(jobs.into_iter()); + .extend(jobs); let depth = data.queues.get(topic).map_or(0, |q| q.len() as u64); let stats = data.stats.entry(topic.to_string()).or_default(); stats.depth = depth; diff --git a/queue/tests/e2e_durability.rs b/queue/tests/e2e_durability.rs index 4ce7682ec..32db877e4 100644 --- a/queue/tests/e2e_durability.rs +++ b/queue/tests/e2e_durability.rs @@ -132,13 +132,17 @@ async fn delivery_dlq_and_redrive_connect_or_skip() { let fires = Arc::new(AtomicUsize::new(0)); let fail = Arc::new(AtomicBool::new(false)); let function_id = format!("queue.e2e.{}", Uuid::new_v4()); + // Unique queue name: the engine re-delivers previously registered + // durable:subscriber triggers on boot, so a fixed name races against + // leftover consumers from earlier local runs. + let queue_name = format!("e2e-demo-{}", Uuid::new_v4()); register_counting_function(&iii, &function_id, fires.clone(), fail.clone()); - register_subscriber(&iii, &function_id, "demo"); + register_subscriber(&iii, &function_id, &queue_name); trigger( &iii, PUBLISH_FN_ID, - json!({"queue": "demo", "data": {"hello": "world"}}), + json!({"queue": queue_name, "data": {"hello": "world"}}), ) .await; wait_for_fires(&fires, 1).await; @@ -147,14 +151,14 @@ async fn delivery_dlq_and_redrive_connect_or_skip() { trigger( &iii, PUBLISH_FN_ID, - json!({"queue": "demo", "data": {"should": "dlq"}}), + json!({"queue": queue_name, "data": {"should": "dlq"}}), ) .await; - let messages = wait_for_dlq(&iii, "demo").await; + let messages = wait_for_dlq(&iii, &queue_name).await; assert_eq!(messages.len(), 1); fail.store(false, Ordering::SeqCst); - trigger(&iii, REDRIVE_FN_ID, json!({"queue": "demo"})).await; + trigger(&iii, REDRIVE_FN_ID, json!({"queue": queue_name})).await; wait_for_fires(&fires, 3).await; boot.shutdown().await; @@ -174,10 +178,11 @@ async fn file_based_pending_message_survives_worker_restart_connect_or_skip() { .await .expect("queue worker should boot"); + let queue_name = format!("e2e-restart-{}", Uuid::new_v4()); trigger( &iii, PUBLISH_FN_ID, - json!({"queue": "restart", "data": {"survives": true}}), + json!({"queue": queue_name, "data": {"survives": true}}), ) .await; boot.shutdown().await; @@ -194,7 +199,7 @@ async fn file_based_pending_message_survives_worker_restart_connect_or_skip() { let fail = Arc::new(AtomicBool::new(false)); let function_id = format!("queue.restart.{}", Uuid::new_v4()); register_counting_function(&iii, &function_id, fires.clone(), fail); - register_subscriber(&iii, &function_id, "restart"); + register_subscriber(&iii, &function_id, &queue_name); wait_for_fires(&fires, 1).await; boot.shutdown().await; From 21bd2d1c09a93e580db3fadfca587e80d0b3c84a Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Mon, 6 Jul 2026 21:20:38 -0300 Subject: [PATCH 14/28] feat(queue): port QueueAdapter trait from the engine builtin --- queue/src/adapter.rs | 203 +++++++++++++++++++++++++++++++++++++++++++ queue/src/lib.rs | 1 + 2 files changed, 204 insertions(+) create mode 100644 queue/src/adapter.rs diff --git a/queue/src/adapter.rs b/queue/src/adapter.rs new file mode 100644 index 000000000..be1368e2d --- /dev/null +++ b/queue/src/adapter.rs @@ -0,0 +1,203 @@ +//! `QueueAdapter` trait — the transport abstraction every queue backend +//! (builtin in-process store, RabbitMQ, SQS, ...) implements. +//! +//! Ported from the engine builtin `iii-queue` worker +//! (`engine/src/workers/queue/mod.rs:40-160`). Seams applied for the +//! standalone worker: the engine trait is invoked from `Arc` +//! call sites; here adapters are driven by the worker's own trigger loop +//! and invoke functions through `Arc` instead. +//! Telemetry/tracing spans present on the engine's trait methods are +//! dropped — the worker emits its own `tracing` events at call sites. + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tokio::sync::mpsc; + +use crate::store::TopicStats; +// Until Task 2 lands this is the existing 2-field struct in trigger.rs; +// Task 2 replaces it with the full port at crate::subscriber_config. +use crate::trigger::SubscriberQueueConfig; + +/// Summary of a known queue topic, returned by [`QueueAdapter::list_topics`]. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct TopicInfo { + pub name: String, + pub depth: u64, +} + +/// A message handed from a function-queue adapter to the consumer loop. +/// +/// Ported from the engine builtin's `QueueMessage` +/// (`engine/src/workers/queue/message.rs`). Only used by the +/// function-queue transport path (`consume_function_queue` and friends), +/// which is call-site-less until the engine `QueueEnqueuer` cut lands. +#[derive(Debug, Clone)] +pub struct QueueMessage { + /// Adapter-specific tag for ack/nack (e.g. RabbitMQ delivery tag). + pub delivery_id: u64, + /// The function to invoke. + pub function_id: String, + /// The payload to pass to the function. + pub data: Value, + /// Current attempt number (derived from transport-native retry count). + pub attempt: u32, + /// Publisher-assigned message ID for tracking and cancellation. + pub message_id: Option, + /// W3C traceparent header for distributed tracing. + pub traceparent: Option, + /// W3C baggage header for context propagation. + pub baggage: Option, +} + +/// Per-function-queue transport settings, passed to +/// [`QueueAdapter::setup_function_queue`]. +/// +/// Minimal port of the engine builtin's `FunctionQueueConfig` +/// (`engine/src/workers/queue/config.rs`) — only the fields the +/// function-queue adapter methods need to compile against. Call-site-less +/// until the engine `QueueEnqueuer` cut lands; a later task may replace +/// this with the full config-module port. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FunctionQueueConfig { + /// Maximum delivery attempts before a message is sent to the + /// dead-letter queue. + pub max_retries: u32, + /// Number of messages processed concurrently. + pub concurrency: u32, + /// Base delay in milliseconds for the exponential retry backoff. + pub backoff_ms: u64, +} + +/// Transport abstraction implemented by every queue backend adapter +/// (builtin in-process store, RabbitMQ, SQS, ...). +/// +/// Ported from the engine builtin's `QueueAdapter` +/// (`engine/src/workers/queue/mod.rs:52-159`). +#[allow(clippy::too_many_arguments)] +#[async_trait] +pub trait QueueAdapter: Send + Sync + 'static { + /// Enqueue a message onto `topic`. + async fn enqueue( + &self, + topic: &str, + data: Value, + traceparent: Option, + baggage: Option, + ); + + /// Register a subscriber (`id`) on `topic` that invokes `function_id` + /// for each delivered message. + async fn subscribe( + &self, + topic: &str, + id: &str, + function_id: &str, + condition_function_id: Option, + queue_config: Option, + ); + + /// Remove a previously registered subscriber. + async fn unsubscribe(&self, topic: &str, id: &str); + + /// Redrive every dead-lettered message on `topic` back onto the live + /// queue. Returns the number of messages redriven. + async fn redrive_dlq(&self, topic: &str) -> anyhow::Result; + + /// Redrive a single dead-lettered message by ID. Returns `true` if the + /// message was found and redriven. + async fn redrive_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result; + + /// Discard a single dead-lettered message by ID. Returns `true` if the + /// message was found and discarded. + async fn discard_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result; + + /// Count of messages currently in `topic`'s dead-letter queue. + async fn dlq_count(&self, topic: &str) -> anyhow::Result; + + /// Peek at up to `count` dead-lettered messages on `topic`, + /// non-destructively. Default implementation delegates to + /// [`Self::dlq_peek`] starting at offset 0. + async fn dlq_messages(&self, topic: &str, count: usize) -> anyhow::Result> { + self.dlq_peek(topic, 0, count as u64).await + } + + /// List all known queue topics. + async fn list_topics(&self) -> anyhow::Result>; + + /// Get stats for a specific topic. + async fn topic_stats(&self, topic: &str) -> anyhow::Result; + + /// Peek at dead-lettered messages on `topic` non-destructively, paginated + /// by `offset`/`limit`. + async fn dlq_peek(&self, topic: &str, offset: u64, limit: u64) -> anyhow::Result>; + + /// Gracefully tear down this adapter (stop consumers, close + /// connections). The worker calls this deterministically on shutdown; + /// the engine builtin instead aborts consumer tasks in place. + async fn shutdown(&self); + + // -- Function queue transport methods -- + // + // Call-site-less until the engine `QueueEnqueuer` cut lands (see + // `engine/src/workers/queue/mod.rs:78-158`). Ported here so adapters + // implementing this trait have the same surface as the engine builtin; + // default bodies match the engine's no-op/unimplemented defaults. + + /// Publish a message directly onto a function's dedicated queue. + #[allow(clippy::too_many_arguments)] + async fn publish_to_function_queue( + &self, + _queue_name: &str, + _function_id: &str, + _data: Value, + _message_id: &str, + _max_retries: u32, + _backoff_ms: u64, + _traceparent: Option, + _baggage: Option, + // AMQP message priority (`None` = default). Honored by adapters whose + // queues are declared as priority queues; others ignore it. + _priority: Option, + ) { + unimplemented!("publish_to_function_queue not implemented for this adapter") + } + + /// Set up transport topology for a function queue (exchanges, queues, + /// bindings). Called once per queue during initialization. + async fn setup_function_queue( + &self, + _queue_name: &str, + _config: &FunctionQueueConfig, + ) -> anyhow::Result<()> { + Ok(()) // no-op by default (schemaless stores like builtin) + } + + /// Start consuming from a function queue. Returns a receiver that yields + /// `QueueMessage`s. + async fn consume_function_queue( + &self, + _queue_name: &str, + _prefetch: u32, + ) -> anyhow::Result> { + unimplemented!("consume_function_queue not implemented for this adapter") + } + + /// Acknowledge successful processing of a message. + async fn ack_function_queue(&self, _queue_name: &str, _delivery_id: u64) -> anyhow::Result<()> { + Ok(()) // no-op by default + } + + /// Negative-acknowledge a message. When `attempt < max_retries`, the + /// adapter should route to retry. When `attempt >= max_retries`, the + /// adapter should route to the DLQ. + async fn nack_function_queue( + &self, + _queue_name: &str, + _delivery_id: u64, + _attempt: u32, + _max_retries: u32, + ) -> anyhow::Result<()> { + Ok(()) // no-op by default + } +} diff --git a/queue/src/lib.rs b/queue/src/lib.rs index 380c95789..2c2dbde07 100644 --- a/queue/src/lib.rs +++ b/queue/src/lib.rs @@ -3,6 +3,7 @@ /// before this worker can boot (see [`boot::start`]'s guard). pub const TRIGGER_TYPE: &str = "durable:subscriber"; +pub mod adapter; pub mod boot; pub mod config; pub mod configuration; From 5a86659254206dcdbce0f3264bd5411def3e0bdc Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Mon, 6 Jul 2026 21:25:54 -0300 Subject: [PATCH 15/28] fix(queue): restore engine default bodies on QueueAdapter optional methods --- queue/src/adapter.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/queue/src/adapter.rs b/queue/src/adapter.rs index be1368e2d..dccfccab2 100644 --- a/queue/src/adapter.rs +++ b/queue/src/adapter.rs @@ -123,14 +123,25 @@ pub trait QueueAdapter: Send + Sync + 'static { } /// List all known queue topics. - async fn list_topics(&self) -> anyhow::Result>; + async fn list_topics(&self) -> anyhow::Result> { + Ok(vec![]) + } /// Get stats for a specific topic. - async fn topic_stats(&self, topic: &str) -> anyhow::Result; + async fn topic_stats(&self, _topic: &str) -> anyhow::Result { + Ok(TopicStats::default()) + } /// Peek at dead-lettered messages on `topic` non-destructively, paginated /// by `offset`/`limit`. - async fn dlq_peek(&self, topic: &str, offset: u64, limit: u64) -> anyhow::Result>; + async fn dlq_peek( + &self, + _topic: &str, + _offset: u64, + _limit: u64, + ) -> anyhow::Result> { + Ok(vec![]) + } /// Gracefully tear down this adapter (stop consumers, close /// connections). The worker calls this deterministically on shutdown; From 56cb8a0ceb89c72b8631d8ad033fec420a245014 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Mon, 6 Jul 2026 21:30:16 -0300 Subject: [PATCH 16/28] feat(queue): full SubscriberQueueConfig parity (fifo, concurrency, priority, backoff fields) --- queue/src/adapter.rs | 4 +- queue/src/lib.rs | 1 + queue/src/subscriber_config.rs | 265 +++++++++++++++++++++++++++++++++ queue/src/trigger.rs | 13 +- 4 files changed, 268 insertions(+), 15 deletions(-) create mode 100644 queue/src/subscriber_config.rs diff --git a/queue/src/adapter.rs b/queue/src/adapter.rs index dccfccab2..4bf151cb7 100644 --- a/queue/src/adapter.rs +++ b/queue/src/adapter.rs @@ -15,9 +15,7 @@ use serde_json::Value; use tokio::sync::mpsc; use crate::store::TopicStats; -// Until Task 2 lands this is the existing 2-field struct in trigger.rs; -// Task 2 replaces it with the full port at crate::subscriber_config. -use crate::trigger::SubscriberQueueConfig; +use crate::subscriber_config::SubscriberQueueConfig; /// Summary of a known queue topic, returned by [`QueueAdapter::list_topics`]. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/queue/src/lib.rs b/queue/src/lib.rs index 2c2dbde07..b1ebee7c0 100644 --- a/queue/src/lib.rs +++ b/queue/src/lib.rs @@ -10,4 +10,5 @@ pub mod configuration; pub mod functions; pub mod manifest; pub mod store; +pub mod subscriber_config; pub mod trigger; diff --git a/queue/src/subscriber_config.rs b/queue/src/subscriber_config.rs new file mode 100644 index 000000000..fb62b792c --- /dev/null +++ b/queue/src/subscriber_config.rs @@ -0,0 +1,265 @@ +//! Full `SubscriberQueueConfig` — per-subscriber queue tuning knobs +//! (fifo/standard mode, retries, concurrency, visibility timeout, delay, +//! backoff, RabbitMQ priority levels). +//! +//! Ported from the engine builtin `iii-queue` worker +//! (`engine/src/workers/queue/subscriber_config.rs`). Two independent +//! deserialization paths must both keep working: +//! +//! - `Deserialize` (serde derive below) — used when the worker parses +//! `SubscriberSpec.queue_config` out of a `TriggerConfig`'s JSON config. +//! - [`SubscriberQueueConfig::from_value`] — a manual, permissive extractor +//! ported verbatim from the engine, matching its exact json keys +//! (including `maxPriority`, camelCase) and its "ignore unknown/malformed +//! fields, return `None` if nothing matched" semantics. +//! +//! The engine struct carries no serde derives; this port adds +//! `Serialize + Deserialize + JsonSchema + PartialEq + Default` additively +//! so the worker can embed it in `SubscriberSpec`'s schema. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +macro_rules! extract_field { + ($config:expr, $json_key:expr, $field:expr, $has_any:expr, $extract:ident, $convert:expr) => { + if let Some(value) = $config.get($json_key).and_then(|v| v.$extract()) { + $field = Some($convert(value)); + $has_any = true; + } + }; +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq)] +pub struct SubscriberQueueConfig { + #[serde(default, rename = "type", skip_serializing_if = "Option::is_none")] + pub queue_mode: Option, + #[serde(default, alias = "maxRetries", skip_serializing_if = "Option::is_none")] + pub max_retries: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub concurrency: Option, + #[serde( + default, + alias = "visibilityTimeout", + skip_serializing_if = "Option::is_none" + )] + pub visibility_timeout: Option, + #[serde( + default, + alias = "delaySeconds", + skip_serializing_if = "Option::is_none" + )] + pub delay_seconds: Option, + #[serde( + default, + alias = "backoffType", + skip_serializing_if = "Option::is_none" + )] + pub backoff_type: Option, + #[serde( + default, + alias = "backoffDelayMs", + skip_serializing_if = "Option::is_none" + )] + pub backoff_delay_ms: Option, + /// Declares this subscriber's queue as a RabbitMQ priority queue with this + /// many levels (`x-max-priority`, 1–255). RabbitMQ-only; the priority value + /// of each message comes from the adapter-level `priority_field`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_priority: Option, +} + +impl SubscriberQueueConfig { + pub fn from_value(config: Option<&Value>) -> Option { + let config = config?; + + if !config.is_object() { + return None; + } + + let mut subscriber_config = Self::default(); + let mut has_any_value = false; + + extract_field!( + config, + "type", + subscriber_config.queue_mode, + has_any_value, + as_str, + |v: &str| v.to_string() + ); + extract_field!( + config, + "maxRetries", + subscriber_config.max_retries, + has_any_value, + as_u64, + |v| v as u32 + ); + extract_field!( + config, + "concurrency", + subscriber_config.concurrency, + has_any_value, + as_u64, + |v| v as u32 + ); + extract_field!( + config, + "visibilityTimeout", + subscriber_config.visibility_timeout, + has_any_value, + as_u64, + |v| v + ); + extract_field!( + config, + "delaySeconds", + subscriber_config.delay_seconds, + has_any_value, + as_u64, + |v| v + ); + extract_field!( + config, + "backoffType", + subscriber_config.backoff_type, + has_any_value, + as_str, + |v: &str| v.to_string() + ); + extract_field!( + config, + "backoffDelayMs", + subscriber_config.backoff_delay_ms, + has_any_value, + as_u64, + |v| v + ); + extract_field!( + config, + "maxPriority", + subscriber_config.max_priority, + has_any_value, + as_u64, + |v| v as u8 + ); + + if has_any_value { + Some(subscriber_config) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_from_value_with_all_fields() { + let config = json!({ + "type": "fifo", + "maxRetries": 5, + "concurrency": 20, + "visibilityTimeout": 3000, + "delaySeconds": 10, + "backoffType": "exponential", + "backoffDelayMs": 2000 + }); + + let result = SubscriberQueueConfig::from_value(Some(&config)); + assert!(result.is_some()); + + let subscriber_config = result.unwrap(); + assert_eq!(subscriber_config.queue_mode, Some("fifo".to_string())); + assert_eq!(subscriber_config.max_retries, Some(5)); + assert_eq!(subscriber_config.concurrency, Some(20)); + assert_eq!(subscriber_config.visibility_timeout, Some(3000)); + assert_eq!(subscriber_config.delay_seconds, Some(10)); + assert_eq!( + subscriber_config.backoff_type, + Some("exponential".to_string()) + ); + assert_eq!(subscriber_config.backoff_delay_ms, Some(2000)); + } + + #[test] + fn test_from_value_parses_max_priority() { + let config = json!({ "maxPriority": 10 }); + let result = SubscriberQueueConfig::from_value(Some(&config)).expect("should parse"); + assert_eq!(result.max_priority, Some(10)); + + // Absent maxPriority leaves it unset (not a priority queue). + let without = json!({ "type": "standard" }); + let parsed = SubscriberQueueConfig::from_value(Some(&without)).expect("should parse"); + assert_eq!(parsed.max_priority, None); + } + + #[test] + fn test_from_value_with_partial_fields() { + let config = json!({ + "type": "standard", + "maxRetries": 3 + }); + + let result = SubscriberQueueConfig::from_value(Some(&config)); + assert!(result.is_some()); + + let subscriber_config = result.unwrap(); + assert_eq!(subscriber_config.queue_mode, Some("standard".to_string())); + assert_eq!(subscriber_config.max_retries, Some(3)); + assert_eq!(subscriber_config.concurrency, None); + } + + #[test] + fn test_from_value_with_empty_object() { + let config = json!({}); + let result = SubscriberQueueConfig::from_value(Some(&config)); + assert!(result.is_none()); + } + + #[test] + fn test_from_value_with_none() { + let result = SubscriberQueueConfig::from_value(None); + assert!(result.is_none()); + } + + #[test] + fn test_from_value_with_non_object() { + let config = json!("not an object"); + let result = SubscriberQueueConfig::from_value(Some(&config)); + assert!(result.is_none()); + } + + #[test] + fn test_deserialize_accepts_engine_json_keys() { + let config: SubscriberQueueConfig = serde_json::from_value(json!({ + "type": "fifo", + "maxRetries": 5, + "concurrency": 20, + "visibilityTimeout": 3000, + "delaySeconds": 10, + "backoffType": "exponential", + "backoffDelayMs": 2000, + "max_priority": 7 + })) + .unwrap(); + + assert_eq!( + config, + SubscriberQueueConfig { + queue_mode: Some("fifo".to_string()), + max_retries: Some(5), + concurrency: Some(20), + visibility_timeout: Some(3000), + delay_seconds: Some(10), + backoff_type: Some("exponential".to_string()), + backoff_delay_ms: Some(2000), + max_priority: Some(7), + } + ); + } +} diff --git a/queue/src/trigger.rs b/queue/src/trigger.rs index 319e61524..3fd880aed 100644 --- a/queue/src/trigger.rs +++ b/queue/src/trigger.rs @@ -16,6 +16,7 @@ use tokio::sync::Mutex; use tokio::task::JoinHandle; use crate::store::QueueStore; +use crate::subscriber_config::SubscriberQueueConfig; pub const DEFAULT_MAX_RETRIES: u32 = 3; pub const DEFAULT_BACKOFF_MS: u64 = 1000; @@ -72,18 +73,6 @@ pub struct SubscriberSpec { pub queue_config: Option, } -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] -pub struct SubscriberQueueConfig { - #[serde(default, alias = "maxRetries", skip_serializing_if = "Option::is_none")] - pub max_retries: Option, - #[serde( - default, - alias = "backoffDelayMs", - skip_serializing_if = "Option::is_none" - )] - pub backoff_delay_ms: Option, -} - #[derive(Debug, Clone)] pub struct RegisteredSubscriber { pub trigger_id: String, From 35d09731d5e689026a0bf28c9bafe963981721e7 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Mon, 6 Jul 2026 21:36:31 -0300 Subject: [PATCH 17/28] fix(queue): accept camelCase maxPriority on the serde path --- queue/src/subscriber_config.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/queue/src/subscriber_config.rs b/queue/src/subscriber_config.rs index fb62b792c..dd13a5a1c 100644 --- a/queue/src/subscriber_config.rs +++ b/queue/src/subscriber_config.rs @@ -65,7 +65,11 @@ pub struct SubscriberQueueConfig { /// Declares this subscriber's queue as a RabbitMQ priority queue with this /// many levels (`x-max-priority`, 1–255). RabbitMQ-only; the priority value /// of each message comes from the adapter-level `priority_field`. - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde( + default, + alias = "maxPriority", + skip_serializing_if = "Option::is_none" + )] pub max_priority: Option, } @@ -262,4 +266,11 @@ mod tests { } ); } + + #[test] + fn test_deserialize_accepts_camel_case_max_priority() { + let config: SubscriberQueueConfig = + serde_json::from_value(json!({"maxPriority": 10})).unwrap(); + assert_eq!(config.max_priority, Some(10)); + } } From 28be86355fd86323223716d7da5fae4524accfed Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Mon, 6 Jul 2026 21:46:55 -0300 Subject: [PATCH 18/28] feat(queue): builtin adapter with per-subscriber concurrency and fifo parity --- queue/src/adapters/builtin.rs | 681 ++++++++++++++++++++++++++++++++++ queue/src/adapters/mod.rs | 3 + queue/src/lib.rs | 1 + 3 files changed, 685 insertions(+) create mode 100644 queue/src/adapters/builtin.rs create mode 100644 queue/src/adapters/mod.rs diff --git a/queue/src/adapters/builtin.rs b/queue/src/adapters/builtin.rs new file mode 100644 index 000000000..46c585874 --- /dev/null +++ b/queue/src/adapters/builtin.rs @@ -0,0 +1,681 @@ +//! Builtin transport adapter — subscribe model with per-subscriber +//! concurrency, fifo mode, and condition-function short-circuit, built on +//! top of the worker's own [`QueueStore`]. +//! +//! Ported from the engine builtin `iii-queue` worker: +//! - `engine/src/workers/queue/adapters/builtin/adapter.rs` (the +//! `QueueAdapter` impl, `FunctionHandler`, `SubscriptionConfig` mapping). +//! - `engine/src/builtins/queue.rs` (`QueueConfig` defaults, the +//! `Worker`/`FifoWorker`/`GroupedFifoWorker` polling loops). +//! +//! Deliberate deviations from the engine port: +//! - No per-function internal-queue fan-out (`topic::function_id`). The +//! engine redirects each subscribed function to its own copy of every +//! message; this port hands `enqueue` straight to the shared +//! [`QueueStore`] topic, matching this task's brief ("enqueue delegates +//! to store"). Two subscribers on the same topic therefore compete for +//! messages rather than each receiving a copy. +//! - No `GroupedFifoWorker` (fifo + concurrency > 1, partitioned by +//! `job.group_id`). This store's `Job` carries no group id, so fifo mode +//! always processes strictly one job at a time regardless of the +//! configured concurrency. +//! - Retries reuse [`QueueStore::nack`] (which already implements the +//! engine's exponential backoff and DLQ-at-exhaustion) instead of the +//! engine's `FifoWorker`, which retries a failed job in-place, blocking +//! its single worker slot until it succeeds or is exhausted. Using +//! `nack` for both modes means a retried fifo job goes back into the +//! shared queue with a delay and may be interleaved with newer arrivals, +//! rather than blocking the whole subscription on one job. Fifo +//! *ordering* for jobs that succeed on their first attempt is unaffected +//! (the poller is single-threaded and never dequeues the next job before +//! the current one is acked or nacked). +//! - Duplicate `(topic, id)` subscriptions warn and no-op instead of +//! silently replacing the old subscription's map entry and leaking its +//! polling task (the actual, un-guarded, engine builtin behavior). +//! Redis and RabbitMQ adapters in the engine already guard this way; +//! this port applies the same guard to the builtin adapter to preserve +//! the "one subscription per (topic, id)" invariant. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use serde_json::Value; +use tokio::sync::{Mutex, Semaphore}; +use tokio::task::JoinHandle; + +use crate::adapter::{QueueAdapter, TopicInfo}; +use crate::store::{Job, QueueStore, TopicStats}; +use crate::subscriber_config::SubscriberQueueConfig; +use crate::trigger::Invoker; + +/// `engine/src/builtins/queue.rs` `QueueConfig::default()` (max_attempts). +const DEFAULT_MAX_RETRIES: u32 = 3; +/// `engine/src/builtins/queue.rs` `QueueConfig::default()` (backoff_ms). +const DEFAULT_BACKOFF_MS: u64 = 1000; +/// `engine/src/builtins/queue.rs` `QueueConfig::default()` (concurrency). +const DEFAULT_CONCURRENCY: u32 = 10; +/// `engine/src/builtins/queue.rs` `QueueConfig::default()` (poll_interval_ms). +const DEFAULT_POLL_INTERVAL_MS: u64 = 100; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Mode { + Concurrent, + Fifo, +} + +/// Per-subscription settings resolved from [`SubscriberQueueConfig`], +/// shared (read-only) across a subscription's polling task(s). +struct PollerConfig { + topic: String, + function_id: String, + condition_function_id: Option, + max_retries: u32, + backoff_ms: u64, + poll_interval_ms: u64, +} + +/// Builtin transport adapter: subscriptions are backed by pure in-process +/// polling tasks over the worker's [`QueueStore`]. +pub struct BuiltinAdapter { + store: Arc, + invoker: Arc, + subscriptions: Mutex>>, + poll_interval_ms: u64, +} + +impl BuiltinAdapter { + pub fn new(store: Arc, invoker: Arc) -> Self { + Self::with_poll_interval_ms(store, invoker, DEFAULT_POLL_INTERVAL_MS) + } + + /// Test-only knob: a short poll interval keeps polling-loop-driven + /// tests fast without changing observable subscribe/fifo/concurrency + /// behavior. + fn with_poll_interval_ms( + store: Arc, + invoker: Arc, + poll_interval_ms: u64, + ) -> Self { + Self { + store, + invoker, + subscriptions: Mutex::new(HashMap::new()), + poll_interval_ms, + } + } +} + +#[async_trait] +impl QueueAdapter for BuiltinAdapter { + async fn enqueue( + &self, + topic: &str, + data: Value, + _traceparent: Option, + _baggage: Option, + ) { + // This store's `Job` carries no trace context yet, so traceparent/ + // baggage are accepted (for trait parity) but dropped. + let _ = self.store.enqueue(topic, data).await; + } + + async fn subscribe( + &self, + topic: &str, + id: &str, + function_id: &str, + condition_function_id: Option, + queue_config: Option, + ) { + let key = (topic.to_string(), id.to_string()); + let mut subs = self.subscriptions.lock().await; + if subs.contains_key(&key) { + tracing::warn!(topic = %topic, id = %id, "Already subscribed to topic"); + return; + } + + let concurrency = queue_config + .as_ref() + .and_then(|c| c.concurrency) + .unwrap_or(DEFAULT_CONCURRENCY) + .max(1); + let max_retries = queue_config + .as_ref() + .and_then(|c| c.max_retries) + .unwrap_or(DEFAULT_MAX_RETRIES); + let backoff_ms = queue_config + .as_ref() + .and_then(|c| c.backoff_delay_ms) + .unwrap_or(DEFAULT_BACKOFF_MS); + let mode = match queue_config.as_ref().and_then(|c| c.queue_mode.as_deref()) { + Some("fifo") => Mode::Fifo, + _ => Mode::Concurrent, + }; + + let cfg = PollerConfig { + topic: topic.to_string(), + function_id: function_id.to_string(), + condition_function_id, + max_retries, + backoff_ms, + poll_interval_ms: self.poll_interval_ms, + }; + + let task = spawn_poller( + Arc::clone(&self.store), + Arc::clone(&self.invoker), + cfg, + mode, + concurrency, + ); + subs.insert(key, task); + + tracing::debug!(topic = %topic, id = %id, function_id = %function_id, ?mode, "Subscribed to queue via BuiltinAdapter"); + } + + async fn unsubscribe(&self, topic: &str, id: &str) { + let key = (topic.to_string(), id.to_string()); + let mut subs = self.subscriptions.lock().await; + if let Some(task) = subs.remove(&key) { + task.abort(); + tracing::debug!(topic = %topic, id = %id, "Unsubscribed from queue"); + } else { + tracing::warn!(topic = %topic, id = %id, "No subscription found to unsubscribe"); + } + } + + async fn redrive_dlq(&self, topic: &str) -> anyhow::Result { + Ok(self.store.redrive_dlq(topic).await) + } + + async fn redrive_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { + Ok(self.store.redrive_dlq_message(topic, message_id).await) + } + + async fn discard_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { + Ok(self.store.discard_dlq_message(topic, message_id).await) + } + + async fn dlq_count(&self, topic: &str) -> anyhow::Result { + Ok(self.store.topic_stats(topic).await.dlq_depth) + } + + async fn dlq_peek(&self, topic: &str, offset: u64, limit: u64) -> anyhow::Result> { + let jobs = self + .store + .dlq_messages(topic, offset.saturating_add(limit)) + .await; + Ok(jobs + .into_iter() + .skip(offset as usize) + .map(|job| serde_json::to_value(job).unwrap_or(Value::Null)) + .collect()) + } + + async fn list_topics(&self) -> anyhow::Result> { + let mut infos = Vec::new(); + for name in self.store.list_topics().await { + let depth = self.store.topic_stats(&name).await.depth; + infos.push(TopicInfo { name, depth }); + } + Ok(infos) + } + + async fn topic_stats(&self, topic: &str) -> anyhow::Result { + Ok(self.store.topic_stats(topic).await) + } + + async fn shutdown(&self) { + let tasks = self.subscriptions.lock().await.drain().collect::>(); + for (_, task) in tasks { + task.abort(); + } + } +} + +fn spawn_poller( + store: Arc, + invoker: Arc, + cfg: PollerConfig, + mode: Mode, + concurrency: u32, +) -> JoinHandle<()> { + tokio::spawn(async move { + match mode { + Mode::Fifo => run_fifo(store, invoker, cfg).await, + Mode::Concurrent => run_concurrent(store, invoker, cfg, concurrency).await, + } + }) +} + +/// Strictly one in-flight invocation, processed in dequeue order. Mirrors +/// the engine's `FifoWorker` (`concurrency <= 1` case) without the inline +/// retry-in-place behavior — see the module doc's "Deliberate deviations". +async fn run_fifo(store: Arc, invoker: Arc, cfg: PollerConfig) { + loop { + match store.dequeue(&cfg.topic).await { + Some(job) => process_job(&store, &invoker, &cfg, job).await, + None => tokio::time::sleep(Duration::from_millis(cfg.poll_interval_ms)).await, + } + } +} + +/// Up to `concurrency` in-flight invocations. Mirrors the engine's +/// `Worker`: a semaphore permit is held for the lifetime of each handler +/// invocation, gating how many jobs are processed concurrently. +async fn run_concurrent( + store: Arc, + invoker: Arc, + cfg: PollerConfig, + concurrency: u32, +) { + let semaphore = Arc::new(Semaphore::new(concurrency as usize)); + let cfg = Arc::new(cfg); + + loop { + let permit = match Arc::clone(&semaphore).acquire_owned().await { + Ok(permit) => permit, + Err(_) => return, // semaphore closed; subscription is being torn down + }; + + match store.dequeue(&cfg.topic).await { + Some(job) => { + let store = Arc::clone(&store); + let invoker = Arc::clone(&invoker); + let cfg = Arc::clone(&cfg); + tokio::spawn(async move { + process_job(&store, &invoker, &cfg, job).await; + drop(permit); + }); + } + None => { + drop(permit); + tokio::time::sleep(Duration::from_millis(cfg.poll_interval_ms)).await; + } + } + } +} + +async fn process_job( + store: &Arc, + invoker: &Arc, + cfg: &PollerConfig, + job: Job, +) { + let result = invoke( + invoker.as_ref(), + &cfg.function_id, + cfg.condition_function_id.as_deref(), + job.payload.clone(), + ) + .await; + + match result { + Ok(()) => store.ack(&cfg.topic, &job.id).await, + Err(_err) => { + store + .nack(&cfg.topic, job, cfg.max_retries, cfg.backoff_ms) + .await + } + } +} + +/// Port of the engine's `FunctionHandler::handle` condition-skip semantics +/// (`queue/src/trigger.rs:201-213`): only an explicit `Ok(Some(false))` +/// from the condition function skips (and acks, since it returns `Ok(())` +/// here); any other `Ok` continues to the target function; `Err` fails the +/// job (nacks). +async fn invoke( + invoker: &dyn Invoker, + function_id: &str, + condition_function_id: Option<&str>, + payload: Value, +) -> Result<(), String> { + if let Some(condition_id) = condition_function_id { + match invoker.call(condition_id, payload.clone()).await { + Ok(Some(Value::Bool(false))) => return Ok(()), + Ok(_) => {} + Err(err) => return Err(err), + } + } + invoker.call(function_id, payload).await.map(|_| ()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::InMemoryStore; + use serde_json::json; + use std::future::Future; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use tokio::time::{sleep, Instant}; + + async fn wait_until(mut pred: F) + where + F: FnMut() -> Fut, + Fut: Future, + { + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline { + if pred().await { + return; + } + sleep(Duration::from_millis(10)).await; + } + assert!(pred().await, "condition did not become true before timeout"); + } + + /// Cheap deterministic jitter so ordering tests exercise varied handler + /// latency without adding a `rand` dependency. + fn jitter_ms(seed: u64) -> u64 { + let mut x = seed ^ 0x9E37_79B9_7F4A_7C15; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x % 15) + 1 + } + + #[derive(Default)] + struct FakeInvoker { + calls: Mutex>, + fail_backend: AtomicBool, + condition_value: Mutex>, + } + + impl FakeInvoker { + async fn calls(&self) -> Vec<(String, Value)> { + self.calls.lock().await.clone() + } + } + + #[async_trait] + impl Invoker for FakeInvoker { + async fn call(&self, function_id: &str, payload: Value) -> Result, String> { + self.calls + .lock() + .await + .push((function_id.to_string(), payload)); + if function_id == "condition" { + return Ok(self.condition_value.lock().await.clone()); + } + if self.fail_backend.load(Ordering::SeqCst) { + Err("backend failed".to_string()) + } else { + Ok(Some(json!({"ok": true}))) + } + } + } + + /// Tracks the maximum number of concurrent in-flight `call`s observed. + #[derive(Default)] + struct ConcurrencyGateInvoker { + in_flight: AtomicUsize, + max_seen: AtomicUsize, + hold_ms: u64, + } + + impl ConcurrencyGateInvoker { + fn new(hold_ms: u64) -> Self { + Self { + in_flight: AtomicUsize::new(0), + max_seen: AtomicUsize::new(0), + hold_ms, + } + } + + fn max_seen(&self) -> usize { + self.max_seen.load(Ordering::SeqCst) + } + } + + #[async_trait] + impl Invoker for ConcurrencyGateInvoker { + async fn call(&self, _function_id: &str, _payload: Value) -> Result, String> { + let now = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1; + self.max_seen.fetch_max(now, Ordering::SeqCst); + sleep(Duration::from_millis(self.hold_ms)).await; + self.in_flight.fetch_sub(1, Ordering::SeqCst); + Ok(Some(json!({"ok": true}))) + } + } + + /// Records the order (and payload) of every `call`, sleeping a jittered + /// delay first so a broken fifo poller (e.g. one that spawns handlers + /// concurrently) would visibly scramble the order. + #[derive(Default)] + struct OrderRecordingInvoker { + order: Mutex>, + } + + impl OrderRecordingInvoker { + async fn order(&self) -> Vec { + self.order.lock().await.clone() + } + } + + #[async_trait] + impl Invoker for OrderRecordingInvoker { + async fn call(&self, _function_id: &str, payload: Value) -> Result, String> { + let n = payload.as_i64().unwrap_or_default(); + sleep(Duration::from_millis(jitter_ms(n as u64))).await; + self.order.lock().await.push(n); + Ok(Some(json!({"ok": true}))) + } + } + + fn config(overrides: SubscriberQueueConfig) -> Option { + Some(overrides) + } + + // (a) subscribe+enqueue delivers. + #[tokio::test] + async fn subscribe_then_enqueue_delivers() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 5); + + adapter + .subscribe("demo", "sub1", "backend", None, None) + .await; + adapter + .enqueue("demo", json!({"hello": "world"}), None, None) + .await; + + wait_until(|| { + let invoker = invoker.clone(); + async move { invoker.calls().await.len() == 1 } + }) + .await; + assert_eq!(store.topic_stats("demo").await.delivered, 1); + adapter.shutdown().await; + } + + // (b) concurrency=3 -> 3 in-flight invocations observed, never more. + #[tokio::test] + async fn concurrency_limits_in_flight_invocations() { + let store: Arc = Arc::new(InMemoryStore::new()); + for i in 0..12 { + store.enqueue("demo", json!(i)).await.unwrap(); + } + let invoker = Arc::new(ConcurrencyGateInvoker::new(60)); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 5); + + adapter + .subscribe( + "demo", + "sub1", + "backend", + None, + config(SubscriberQueueConfig { + concurrency: Some(3), + ..Default::default() + }), + ) + .await; + + wait_until(|| { + let invoker = invoker.clone(); + async move { invoker.max_seen() >= 3 } + }) + .await; + // Give any (incorrect) over-eager spawning a chance to show up. + sleep(Duration::from_millis(100)).await; + assert_eq!(invoker.max_seen(), 3); + adapter.shutdown().await; + } + + // (c) fifo -> strict ordering under 20 enqueues with random handler latency. + #[tokio::test] + async fn fifo_processes_strictly_in_order() { + let store: Arc = Arc::new(InMemoryStore::new()); + for i in 0..20 { + store.enqueue("demo", json!(i)).await.unwrap(); + } + let invoker = Arc::new(OrderRecordingInvoker::default()); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 3); + + adapter + .subscribe( + "demo", + "sub1", + "backend", + None, + config(SubscriberQueueConfig { + queue_mode: Some("fifo".to_string()), + // Deliberately set >1: fifo mode must ignore it (no + // grouped-fifo support) and still process one at a time. + concurrency: Some(5), + ..Default::default() + }), + ) + .await; + + wait_until(|| { + let invoker = invoker.clone(); + async move { invoker.order().await.len() == 20 } + }) + .await; + let order = invoker.order().await; + let expected: Vec = (0..20).collect(); + assert_eq!(order, expected); + adapter.shutdown().await; + } + + // (d) failing handler -> DLQ after max_retries. + #[tokio::test] + async fn failing_handler_moves_to_dlq_after_max_retries() { + let store: Arc = Arc::new(InMemoryStore::new()); + store.enqueue("demo", json!("job")).await.unwrap(); + let invoker = Arc::new(FakeInvoker::default()); + invoker.fail_backend.store(true, Ordering::SeqCst); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 3); + + adapter + .subscribe( + "demo", + "sub1", + "backend", + None, + config(SubscriberQueueConfig { + max_retries: Some(2), + backoff_delay_ms: Some(1), + ..Default::default() + }), + ) + .await; + + wait_until(|| { + let store = store.clone(); + async move { !store.dlq_messages("demo", 10).await.is_empty() } + }) + .await; + let stats = store.topic_stats("demo").await; + assert_eq!(stats.failed, 1); + assert_eq!(store.dlq_messages("demo", 10).await[0].attempts, 2); + adapter.shutdown().await; + } + + // (e) unsubscribe stops delivery. + #[tokio::test] + async fn unsubscribe_stops_delivery() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 3); + + adapter + .subscribe("demo", "sub1", "backend", None, None) + .await; + adapter.unsubscribe("demo", "sub1").await; + store.enqueue("demo", json!("after")).await.unwrap(); + sleep(Duration::from_millis(120)).await; + assert!(invoker.calls().await.is_empty()); + } + + // (f) condition=false skips + acks without invoking the target function. + #[tokio::test] + async fn condition_false_skips_and_acks_without_invoking_target() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + *invoker.condition_value.lock().await = Some(Value::Bool(false)); + store + .enqueue("demo", json!({"hello": "world"})) + .await + .unwrap(); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 3); + + adapter + .subscribe( + "demo", + "sub1", + "backend", + Some("condition".to_string()), + None, + ) + .await; + + wait_until(|| { + let invoker = invoker.clone(); + async move { invoker.calls().await.len() == 1 } + }) + .await; + assert_eq!(invoker.calls().await[0].0, "condition"); + assert_eq!(store.topic_stats("demo").await.delivered, 1); + adapter.shutdown().await; + } + + // Duplicate (topic, id) subscribe is a warn+no-op, not a silent replace. + #[tokio::test] + async fn duplicate_subscription_is_noop() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 3); + + adapter + .subscribe("demo", "sub1", "backend", None, None) + .await; + adapter + .subscribe("demo", "sub1", "other-backend", None, None) + .await; + assert_eq!(adapter.subscriptions.lock().await.len(), 1); + adapter.shutdown().await; + } + + // dlq_peek returns store DLQ jobs as JSON. + #[tokio::test] + async fn dlq_peek_returns_store_jobs_as_json() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker, 3); + + store.enqueue("demo", json!("job")).await.unwrap(); + let job = store.dequeue("demo").await.unwrap(); + store.nack("demo", job, 1, 1).await; + + let peeked = adapter.dlq_peek("demo", 0, 10).await.unwrap(); + assert_eq!(peeked.len(), 1); + assert_eq!(peeked[0]["payload"], json!("job")); + assert_eq!(peeked[0]["attempts"], json!(1)); + } +} diff --git a/queue/src/adapters/mod.rs b/queue/src/adapters/mod.rs new file mode 100644 index 000000000..951f768a7 --- /dev/null +++ b/queue/src/adapters/mod.rs @@ -0,0 +1,3 @@ +//! Transport adapters implementing [`crate::adapter::QueueAdapter`]. + +pub mod builtin; diff --git a/queue/src/lib.rs b/queue/src/lib.rs index b1ebee7c0..8b0ee3020 100644 --- a/queue/src/lib.rs +++ b/queue/src/lib.rs @@ -4,6 +4,7 @@ pub const TRIGGER_TYPE: &str = "durable:subscriber"; pub mod adapter; +pub mod adapters; pub mod boot; pub mod config; pub mod configuration; From c45cfde888e8bce7744157b6dc17449fbadccbd3 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Mon, 6 Jul 2026 22:03:23 -0300 Subject: [PATCH 19/28] fix(queue): builtin adapter fan-out per subscriber and engine concurrency semantics --- queue/src/adapters/builtin.rs | 445 +++++++++++++++++++++++++++++----- 1 file changed, 384 insertions(+), 61 deletions(-) diff --git a/queue/src/adapters/builtin.rs b/queue/src/adapters/builtin.rs index 46c585874..c5b3f4fb1 100644 --- a/queue/src/adapters/builtin.rs +++ b/queue/src/adapters/builtin.rs @@ -8,13 +8,31 @@ //! - `engine/src/builtins/queue.rs` (`QueueConfig` defaults, the //! `Worker`/`FifoWorker`/`GroupedFifoWorker` polling loops). //! +//! Fan-out model (matches the engine exactly — see +//! `engine/src/workers/queue/adapters/builtin/adapter.rs:196-337`): +//! - Each `(topic, id)` subscription gets its own internal queue named +//! `format!("{topic}::{function_id}")` (`adapter.rs:206,229`); `topic_functions` +//! tracks which function ids are currently subscribed to a topic. +//! - `enqueue` pushes a **separate copy** of the message onto every +//! subscribed function's internal queue when the topic has subscribers +//! (`adapter.rs:203-215`, broadcast — N subscribers each receive every +//! message, they do not compete for one shared queue); with no +//! subscribers it enqueues straight onto the bare topic name +//! (`adapter.rs:216-218`), where it sits until a subscriber later +//! attaches (same buffering behavior as the engine). +//! - `redrive_dlq`, `redrive_dlq_message`, `discard_dlq_message`, +//! `dlq_count`/`topic_stats`, and `dlq_peek` all resolve a bare topic +//! name the same way: if it has subscribers, operate on/aggregate over +//! every subscriber's internal queue (`adapter.rs:310-384,586-637`); +//! otherwise operate on the bare topic directly. This lets service +//! functions and operators keep passing the bare topic name for these +//! ops regardless of how many subscribers exist. +//! - `list_topics` mirrors `adapter.rs:558-584`: topics are enumerated +//! from the subscription registry (`topic_functions`), not by scanning +//! every key the store happens to hold — a topic that only ever had a +//! bare, subscriber-less `enqueue` does not appear (same as the engine). +//! //! Deliberate deviations from the engine port: -//! - No per-function internal-queue fan-out (`topic::function_id`). The -//! engine redirects each subscribed function to its own copy of every -//! message; this port hands `enqueue` straight to the shared -//! [`QueueStore`] topic, matching this task's brief ("enqueue delegates -//! to store"). Two subscribers on the same topic therefore compete for -//! messages rather than each receiving a copy. //! - No `GroupedFifoWorker` (fifo + concurrency > 1, partitioned by //! `job.group_id`). This store's `Job` carries no group id, so fifo mode //! always processes strictly one job at a time regardless of the @@ -36,7 +54,7 @@ //! this port applies the same guard to the builtin adapter to preserve //! the "one subscription per (topic, id)" invariant. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; @@ -68,7 +86,10 @@ enum Mode { /// Per-subscription settings resolved from [`SubscriberQueueConfig`], /// shared (read-only) across a subscription's polling task(s). struct PollerConfig { - topic: String, + /// The internal queue this subscription's poller reads from — + /// `format!("{topic}::{function_id}")`, not the bare topic name (see + /// the module doc's fan-out model). + queue_name: String, function_id: String, condition_function_id: Option, max_retries: u32, @@ -76,15 +97,39 @@ struct PollerConfig { poll_interval_ms: u64, } +/// A tracked `(topic, id)` subscription: its polling task plus the +/// function id it targets, so `unsubscribe` can decide whether any other +/// subscription still needs that function's entry in `topic_functions` +/// (mirrors the engine's `trigger_function_map`, `adapter.rs:50,273-308` — +/// folded into `subscriptions` here since the two maps always share the +/// same `(topic, id)` key set). +struct Subscription { + task: JoinHandle<()>, + function_id: String, +} + /// Builtin transport adapter: subscriptions are backed by pure in-process /// polling tasks over the worker's [`QueueStore`]. pub struct BuiltinAdapter { store: Arc, invoker: Arc, - subscriptions: Mutex>>, + subscriptions: Mutex>, + /// topic -> set of function ids currently subscribed to it. Resolves a + /// bare topic name to its subscribers' internal queue names + /// (`format!("{topic}::{function_id}")`) for fan-out enqueue and all + /// DLQ/stat ops — mirrors the engine's `topic_functions` + /// (`adapter.rs:49`). + topic_functions: Mutex>>, poll_interval_ms: u64, } +/// `format!("{topic}::{function_id}")` — the internal queue name a +/// subscription's messages, acks/nacks, and DLQ entries live under. +/// Mirrors the engine's internal-queue naming (`adapter.rs:206,229`). +fn internal_queue_name(topic: &str, function_id: &str) -> String { + format!("{topic}::{function_id}") +} + impl BuiltinAdapter { pub fn new(store: Arc, invoker: Arc) -> Self { Self::with_poll_interval_ms(store, invoker, DEFAULT_POLL_INTERVAL_MS) @@ -102,6 +147,7 @@ impl BuiltinAdapter { store, invoker, subscriptions: Mutex::new(HashMap::new()), + topic_functions: Mutex::new(HashMap::new()), poll_interval_ms, } } @@ -118,7 +164,23 @@ impl QueueAdapter for BuiltinAdapter { ) { // This store's `Job` carries no trace context yet, so traceparent/ // baggage are accepted (for trait parity) but dropped. - let _ = self.store.enqueue(topic, data).await; + // + // Fan-out mirrors `adapter.rs:196-219`: a topic with subscribers + // gets a separate copy pushed onto EACH subscriber's own internal + // queue (broadcast); a topic with none enqueues straight onto the + // bare topic name. + let function_ids = self.topic_functions.lock().await.get(topic).cloned(); + match function_ids { + Some(function_ids) if !function_ids.is_empty() => { + for function_id in &function_ids { + let queue_name = internal_queue_name(topic, function_id); + let _ = self.store.enqueue(&queue_name, data.clone()).await; + } + } + _ => { + let _ = self.store.enqueue(topic, data).await; + } + } } async fn subscribe( @@ -136,11 +198,12 @@ impl QueueAdapter for BuiltinAdapter { return; } + // No clamp: the engine has none either — `concurrency: 0` means + // consumption is paused (the semaphore never yields a permit). let concurrency = queue_config .as_ref() .and_then(|c| c.concurrency) - .unwrap_or(DEFAULT_CONCURRENCY) - .max(1); + .unwrap_or(DEFAULT_CONCURRENCY); let max_retries = queue_config .as_ref() .and_then(|c| c.max_retries) @@ -154,8 +217,17 @@ impl QueueAdapter for BuiltinAdapter { _ => Mode::Concurrent, }; + let queue_name = internal_queue_name(topic, function_id); + + self.topic_functions + .lock() + .await + .entry(topic.to_string()) + .or_default() + .insert(function_id.to_string()); + let cfg = PollerConfig { - topic: topic.to_string(), + queue_name, function_id: function_id.to_string(), condition_function_id, max_retries, @@ -170,67 +242,190 @@ impl QueueAdapter for BuiltinAdapter { mode, concurrency, ); - subs.insert(key, task); + subs.insert( + key, + Subscription { + task, + function_id: function_id.to_string(), + }, + ); tracing::debug!(topic = %topic, id = %id, function_id = %function_id, ?mode, "Subscribed to queue via BuiltinAdapter"); } async fn unsubscribe(&self, topic: &str, id: &str) { let key = (topic.to_string(), id.to_string()); - let mut subs = self.subscriptions.lock().await; - if let Some(task) = subs.remove(&key) { - task.abort(); - tracing::debug!(topic = %topic, id = %id, "Unsubscribed from queue"); - } else { + let removed = self.subscriptions.lock().await.remove(&key); + + let Some(sub) = removed else { tracing::warn!(topic = %topic, id = %id, "No subscription found to unsubscribe"); + return; + }; + sub.task.abort(); + tracing::debug!(topic = %topic, id = %id, "Unsubscribed from queue"); + + // Only drop `function_id` from `topic_functions[topic]` once no + // other subscription on this topic still targets it — mirrors the + // engine's ref-counted cleanup (`adapter.rs:291-307`). + let still_targeted = self + .subscriptions + .lock() + .await + .iter() + .any(|((t, _), s)| t == topic && s.function_id == sub.function_id); + + if !still_targeted { + let mut tf = self.topic_functions.lock().await; + if let Some(function_ids) = tf.get_mut(topic) { + function_ids.remove(&sub.function_id); + if function_ids.is_empty() { + tf.remove(topic); + } + } } } async fn redrive_dlq(&self, topic: &str) -> anyhow::Result { - Ok(self.store.redrive_dlq(topic).await) + let function_ids = self.topic_functions.lock().await.get(topic).cloned(); + match function_ids { + Some(function_ids) if !function_ids.is_empty() => { + let mut total = 0u64; + for function_id in &function_ids { + let queue_name = internal_queue_name(topic, function_id); + total += self.store.redrive_dlq(&queue_name).await; + } + Ok(total) + } + _ => Ok(self.store.redrive_dlq(topic).await), + } } async fn redrive_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { - Ok(self.store.redrive_dlq_message(topic, message_id).await) + let function_ids = self.topic_functions.lock().await.get(topic).cloned(); + match function_ids { + Some(function_ids) if !function_ids.is_empty() => { + for function_id in &function_ids { + let queue_name = internal_queue_name(topic, function_id); + if self + .store + .redrive_dlq_message(&queue_name, message_id) + .await + { + return Ok(true); + } + } + Ok(false) + } + _ => Ok(self.store.redrive_dlq_message(topic, message_id).await), + } } async fn discard_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { - Ok(self.store.discard_dlq_message(topic, message_id).await) + let function_ids = self.topic_functions.lock().await.get(topic).cloned(); + match function_ids { + Some(function_ids) if !function_ids.is_empty() => { + for function_id in &function_ids { + let queue_name = internal_queue_name(topic, function_id); + if self + .store + .discard_dlq_message(&queue_name, message_id) + .await + { + return Ok(true); + } + } + Ok(false) + } + _ => Ok(self.store.discard_dlq_message(topic, message_id).await), + } } async fn dlq_count(&self, topic: &str) -> anyhow::Result { - Ok(self.store.topic_stats(topic).await.dlq_depth) + let function_ids = self.topic_functions.lock().await.get(topic).cloned(); + match function_ids { + Some(function_ids) if !function_ids.is_empty() => { + let mut total = 0u64; + for function_id in &function_ids { + let queue_name = internal_queue_name(topic, function_id); + total += self.store.topic_stats(&queue_name).await.dlq_depth; + } + Ok(total) + } + _ => Ok(self.store.topic_stats(topic).await.dlq_depth), + } } async fn dlq_peek(&self, topic: &str, offset: u64, limit: u64) -> anyhow::Result> { - let jobs = self - .store - .dlq_messages(topic, offset.saturating_add(limit)) - .await; + let function_ids = self.topic_functions.lock().await.get(topic).cloned(); + let jobs: Vec = match function_ids { + Some(function_ids) if !function_ids.is_empty() => { + let mut all = Vec::new(); + for function_id in &function_ids { + let queue_name = internal_queue_name(topic, function_id); + all.extend( + self.store + .dlq_messages(&queue_name, offset.saturating_add(limit)) + .await, + ); + } + all + } + _ => { + self.store + .dlq_messages(topic, offset.saturating_add(limit)) + .await + } + }; Ok(jobs .into_iter() .skip(offset as usize) + .take(limit as usize) .map(|job| serde_json::to_value(job).unwrap_or(Value::Null)) .collect()) } async fn list_topics(&self) -> anyhow::Result> { - let mut infos = Vec::new(); - for name in self.store.list_topics().await { - let depth = self.store.topic_stats(&name).await.depth; - infos.push(TopicInfo { name, depth }); + // Enumerated from the subscription registry, not by scanning every + // key the store happens to hold — mirrors `adapter.rs:558-584`. + let topic_functions = self.topic_functions.lock().await.clone(); + let mut infos = Vec::with_capacity(topic_functions.len()); + for (topic, function_ids) in &topic_functions { + let mut depth = 0u64; + for function_id in function_ids { + let queue_name = internal_queue_name(topic, function_id); + depth += self.store.topic_stats(&queue_name).await.depth; + } + infos.push(TopicInfo { + name: topic.clone(), + depth, + }); } Ok(infos) } async fn topic_stats(&self, topic: &str) -> anyhow::Result { - Ok(self.store.topic_stats(topic).await) + let function_ids = self.topic_functions.lock().await.get(topic).cloned(); + match function_ids { + Some(function_ids) if !function_ids.is_empty() => { + let mut aggregate = TopicStats::default(); + for function_id in &function_ids { + let queue_name = internal_queue_name(topic, function_id); + let stats = self.store.topic_stats(&queue_name).await; + aggregate.depth += stats.depth; + aggregate.dlq_depth += stats.dlq_depth; + aggregate.delivered += stats.delivered; + aggregate.failed += stats.failed; + } + Ok(aggregate) + } + _ => Ok(self.store.topic_stats(topic).await), + } } async fn shutdown(&self) { - let tasks = self.subscriptions.lock().await.drain().collect::>(); - for (_, task) in tasks { - task.abort(); + let subs = self.subscriptions.lock().await.drain().collect::>(); + for (_, sub) in subs { + sub.task.abort(); } } } @@ -255,7 +450,7 @@ fn spawn_poller( /// retry-in-place behavior — see the module doc's "Deliberate deviations". async fn run_fifo(store: Arc, invoker: Arc, cfg: PollerConfig) { loop { - match store.dequeue(&cfg.topic).await { + match store.dequeue(&cfg.queue_name).await { Some(job) => process_job(&store, &invoker, &cfg, job).await, None => tokio::time::sleep(Duration::from_millis(cfg.poll_interval_ms)).await, } @@ -280,7 +475,7 @@ async fn run_concurrent( Err(_) => return, // semaphore closed; subscription is being torn down }; - match store.dequeue(&cfg.topic).await { + match store.dequeue(&cfg.queue_name).await { Some(job) => { let store = Arc::clone(&store); let invoker = Arc::clone(&invoker); @@ -313,10 +508,10 @@ async fn process_job( .await; match result { - Ok(()) => store.ack(&cfg.topic, &job.id).await, + Ok(()) => store.ack(&cfg.queue_name, &job.id).await, Err(_err) => { store - .nack(&cfg.topic, job, cfg.max_retries, cfg.backoff_ms) + .nack(&cfg.queue_name, job, cfg.max_retries, cfg.backoff_ms) .await } } @@ -488,7 +683,90 @@ mod tests { async move { invoker.calls().await.len() == 1 } }) .await; - assert_eq!(store.topic_stats("demo").await.delivered, 1); + assert_eq!(adapter.topic_stats("demo").await.unwrap().delivered, 1); + adapter.shutdown().await; + } + + // (g) enqueue fans out a separate copy to EVERY subscriber on the same + // topic (broadcast) instead of the subscribers competing for one + // shared queue — mirrors `adapter.rs:203-215`. + #[tokio::test] + async fn enqueue_fans_out_to_every_subscriber() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 5); + + adapter.subscribe("demo", "sub-a", "fn-a", None, None).await; + adapter.subscribe("demo", "sub-b", "fn-b", None, None).await; + + for i in 0..3 { + adapter.enqueue("demo", json!(i), None, None).await; + } + + wait_until(|| { + let invoker = invoker.clone(); + async move { invoker.calls().await.len() == 6 } + }) + .await; + + let calls = invoker.calls().await; + let fn_a_calls = calls.iter().filter(|(f, _)| f == "fn-a").count(); + let fn_b_calls = calls.iter().filter(|(f, _)| f == "fn-b").count(); + assert_eq!(fn_a_calls, 3, "fn-a should receive every enqueued message"); + assert_eq!(fn_b_calls, 3, "fn-b should receive every enqueued message"); + adapter.shutdown().await; + } + + // enqueue onto a topic with no subscribers buffers directly on the + // bare topic name (no internal-queue namespacing) — mirrors + // `adapter.rs:216-218`. + #[tokio::test] + async fn enqueue_without_subscribers_buffers_on_bare_topic() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker, 5); + + adapter + .enqueue("demo", json!("no subscribers yet"), None, None) + .await; + + assert_eq!(store.topic_stats("demo").await.depth, 1); + let job = store + .dequeue("demo") + .await + .expect("job should be on the bare topic"); + assert_eq!(job.payload, json!("no subscribers yet")); + } + + // concurrency: 0 pauses consumption entirely — no clamp, matching the + // engine (a semaphore with zero permits never yields one). + #[tokio::test] + async fn concurrency_zero_pauses_consumption() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 5); + + adapter + .subscribe( + "demo", + "sub1", + "backend", + None, + config(SubscriberQueueConfig { + concurrency: Some(0), + ..Default::default() + }), + ) + .await; + adapter + .enqueue("demo", json!({"hello": "world"}), None, None) + .await; + + sleep(Duration::from_millis(200)).await; + assert!( + invoker.calls().await.is_empty(), + "concurrency: 0 must not deliver any message" + ); adapter.shutdown().await; } @@ -496,9 +774,6 @@ mod tests { #[tokio::test] async fn concurrency_limits_in_flight_invocations() { let store: Arc = Arc::new(InMemoryStore::new()); - for i in 0..12 { - store.enqueue("demo", json!(i)).await.unwrap(); - } let invoker = Arc::new(ConcurrencyGateInvoker::new(60)); let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 5); @@ -514,6 +789,9 @@ mod tests { }), ) .await; + for i in 0..12 { + adapter.enqueue("demo", json!(i), None, None).await; + } wait_until(|| { let invoker = invoker.clone(); @@ -530,9 +808,6 @@ mod tests { #[tokio::test] async fn fifo_processes_strictly_in_order() { let store: Arc = Arc::new(InMemoryStore::new()); - for i in 0..20 { - store.enqueue("demo", json!(i)).await.unwrap(); - } let invoker = Arc::new(OrderRecordingInvoker::default()); let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 3); @@ -551,6 +826,9 @@ mod tests { }), ) .await; + for i in 0..20 { + adapter.enqueue("demo", json!(i), None, None).await; + } wait_until(|| { let invoker = invoker.clone(); @@ -563,11 +841,13 @@ mod tests { adapter.shutdown().await; } - // (d) failing handler -> DLQ after max_retries. + // (d) failing handler -> DLQ after max_retries. Also exercises finding + // 1's requirement that DLQ ops keep working against the bare topic + // name a service function passes, even though the job actually lives + // under the internal `demo::backend` queue. #[tokio::test] async fn failing_handler_moves_to_dlq_after_max_retries() { let store: Arc = Arc::new(InMemoryStore::new()); - store.enqueue("demo", json!("job")).await.unwrap(); let invoker = Arc::new(FakeInvoker::default()); invoker.fail_backend.store(true, Ordering::SeqCst); let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 3); @@ -585,15 +865,59 @@ mod tests { }), ) .await; + adapter.enqueue("demo", json!("job"), None, None).await; + + wait_until(|| async { adapter.dlq_count("demo").await.unwrap_or(0) > 0 }).await; + + assert_eq!(adapter.dlq_count("demo").await.unwrap(), 1); + let peeked = adapter.dlq_peek("demo", 0, 10).await.unwrap(); + assert_eq!(peeked.len(), 1); + assert_eq!(peeked[0]["attempts"], json!(2)); + adapter.shutdown().await; + } + + // DLQ ops (redrive_dlq, discard_dlq_message, dlq_count, dlq_peek) + // resolve a bare topic across MULTIPLE subscribers by aggregating over + // each one's internal queue — mirrors `adapter.rs:310-384`. + #[tokio::test] + async fn dlq_ops_resolve_bare_topic_across_multiple_subscribers() { + let store: Arc = Arc::new(InMemoryStore::new()); + let invoker = Arc::new(FakeInvoker::default()); + invoker.fail_backend.store(true, Ordering::SeqCst); + let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 3); + + let retry_cfg = || { + config(SubscriberQueueConfig { + max_retries: Some(1), + backoff_delay_ms: Some(1), + ..Default::default() + }) + }; + adapter + .subscribe("demo", "sub-a", "fn-a", None, retry_cfg()) + .await; + adapter + .subscribe("demo", "sub-b", "fn-b", None, retry_cfg()) + .await; + + adapter.enqueue("demo", json!("job"), None, None).await; + + // Both subscribers get their own copy and both fail straight to + // their own DLQ (max_retries: 1) -> dlq_count aggregates both. + wait_until(|| async { adapter.dlq_count("demo").await.unwrap_or(0) >= 2 }).await; + assert_eq!(adapter.dlq_count("demo").await.unwrap(), 2); + + let peeked = adapter.dlq_peek("demo", 0, 10).await.unwrap(); + assert_eq!(peeked.len(), 2); + + let job_id = peeked[0]["id"].as_str().unwrap().to_string(); + assert!(adapter.discard_dlq_message("demo", &job_id).await.unwrap()); + assert_eq!(adapter.dlq_count("demo").await.unwrap(), 1); + + let redriven = adapter.redrive_dlq("demo").await.unwrap(); + assert_eq!(redriven, 1); + assert_eq!(adapter.dlq_count("demo").await.unwrap(), 0); - wait_until(|| { - let store = store.clone(); - async move { !store.dlq_messages("demo", 10).await.is_empty() } - }) - .await; - let stats = store.topic_stats("demo").await; - assert_eq!(stats.failed, 1); - assert_eq!(store.dlq_messages("demo", 10).await[0].attempts, 2); adapter.shutdown().await; } @@ -619,10 +943,6 @@ mod tests { let store: Arc = Arc::new(InMemoryStore::new()); let invoker = Arc::new(FakeInvoker::default()); *invoker.condition_value.lock().await = Some(Value::Bool(false)); - store - .enqueue("demo", json!({"hello": "world"})) - .await - .unwrap(); let adapter = BuiltinAdapter::with_poll_interval_ms(store.clone(), invoker.clone(), 3); adapter @@ -634,6 +954,9 @@ mod tests { None, ) .await; + adapter + .enqueue("demo", json!({"hello": "world"}), None, None) + .await; wait_until(|| { let invoker = invoker.clone(); @@ -641,7 +964,7 @@ mod tests { }) .await; assert_eq!(invoker.calls().await[0].0, "condition"); - assert_eq!(store.topic_stats("demo").await.delivered, 1); + assert_eq!(adapter.topic_stats("demo").await.unwrap().delivered, 1); adapter.shutdown().await; } From 5ec5f205f4181ea9af287f75c7bbcd245c26d799 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Tue, 7 Jul 2026 05:27:17 -0300 Subject: [PATCH 20/28] refactor(queue): route trigger and service functions through QueueAdapter --- queue/src/adapter.rs | 253 ++++++++++++++++++- queue/src/boot.rs | 19 +- queue/src/configuration.rs | 23 +- queue/src/functions.rs | 492 +++++++++++++++++++++++++++++-------- queue/src/main.rs | 2 +- queue/src/trigger.rs | 439 +++++++++++++++++---------------- 6 files changed, 889 insertions(+), 339 deletions(-) diff --git a/queue/src/adapter.rs b/queue/src/adapter.rs index 4bf151cb7..6c6564c62 100644 --- a/queue/src/adapter.rs +++ b/queue/src/adapter.rs @@ -9,10 +9,12 @@ //! Telemetry/tracing spans present on the engine's trait methods are //! dropped — the worker emits its own `tracing` events at call sites. +use std::sync::Arc; + use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::Value; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, RwLock}; use crate::store::TopicStats; use crate::subscriber_config::SubscriberQueueConfig; @@ -210,3 +212,252 @@ pub trait QueueAdapter: Send + Sync + 'static { Ok(()) // no-op by default } } + +/// Hot-swappable [`QueueAdapter`]: holds the currently active adapter behind +/// an `RwLock` and forwards every trait method to it. Same pattern as +/// [`crate::store::SwappableStore`] — a configuration change replaces the +/// inner adapter (e.g. builtin -> file-backed builtin) without the trigger +/// handler or service functions needing to know a swap happened. +#[derive(Clone)] +pub struct SwappableAdapter { + inner: Arc>>, +} + +impl SwappableAdapter { + pub fn new(adapter: Arc) -> Self { + Self { + inner: Arc::new(RwLock::new(adapter)), + } + } + + pub async fn current(&self) -> Arc { + self.inner.read().await.clone() + } + + pub async fn replace(&self, adapter: Arc) { + *self.inner.write().await = adapter; + } +} + +#[async_trait] +impl QueueAdapter for SwappableAdapter { + async fn enqueue( + &self, + topic: &str, + data: Value, + traceparent: Option, + baggage: Option, + ) { + self.current() + .await + .enqueue(topic, data, traceparent, baggage) + .await; + } + + async fn subscribe( + &self, + topic: &str, + id: &str, + function_id: &str, + condition_function_id: Option, + queue_config: Option, + ) { + self.current() + .await + .subscribe(topic, id, function_id, condition_function_id, queue_config) + .await; + } + + async fn unsubscribe(&self, topic: &str, id: &str) { + self.current().await.unsubscribe(topic, id).await; + } + + async fn redrive_dlq(&self, topic: &str) -> anyhow::Result { + self.current().await.redrive_dlq(topic).await + } + + async fn redrive_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { + self.current() + .await + .redrive_dlq_message(topic, message_id) + .await + } + + async fn discard_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { + self.current() + .await + .discard_dlq_message(topic, message_id) + .await + } + + async fn dlq_count(&self, topic: &str) -> anyhow::Result { + self.current().await.dlq_count(topic).await + } + + async fn dlq_messages(&self, topic: &str, count: usize) -> anyhow::Result> { + self.current().await.dlq_messages(topic, count).await + } + + async fn list_topics(&self) -> anyhow::Result> { + self.current().await.list_topics().await + } + + async fn topic_stats(&self, topic: &str) -> anyhow::Result { + self.current().await.topic_stats(topic).await + } + + async fn dlq_peek(&self, topic: &str, offset: u64, limit: u64) -> anyhow::Result> { + self.current().await.dlq_peek(topic, offset, limit).await + } + + async fn shutdown(&self) { + self.current().await.shutdown().await; + } + + async fn publish_to_function_queue( + &self, + queue_name: &str, + function_id: &str, + data: Value, + message_id: &str, + max_retries: u32, + backoff_ms: u64, + traceparent: Option, + baggage: Option, + priority: Option, + ) { + self.current() + .await + .publish_to_function_queue( + queue_name, + function_id, + data, + message_id, + max_retries, + backoff_ms, + traceparent, + baggage, + priority, + ) + .await; + } + + async fn setup_function_queue( + &self, + queue_name: &str, + config: &FunctionQueueConfig, + ) -> anyhow::Result<()> { + self.current() + .await + .setup_function_queue(queue_name, config) + .await + } + + async fn consume_function_queue( + &self, + queue_name: &str, + prefetch: u32, + ) -> anyhow::Result> { + self.current() + .await + .consume_function_queue(queue_name, prefetch) + .await + } + + async fn ack_function_queue(&self, queue_name: &str, delivery_id: u64) -> anyhow::Result<()> { + self.current() + .await + .ack_function_queue(queue_name, delivery_id) + .await + } + + async fn nack_function_queue( + &self, + queue_name: &str, + delivery_id: u64, + attempt: u32, + max_retries: u32, + ) -> anyhow::Result<()> { + self.current() + .await + .nack_function_queue(queue_name, delivery_id, attempt, max_retries) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + + #[derive(Default)] + struct CountingAdapter { + enqueue_calls: AtomicU64, + } + + #[async_trait] + impl QueueAdapter for CountingAdapter { + async fn enqueue( + &self, + _topic: &str, + _data: Value, + _traceparent: Option, + _baggage: Option, + ) { + self.enqueue_calls.fetch_add(1, Ordering::SeqCst); + } + + async fn subscribe( + &self, + _topic: &str, + _id: &str, + _function_id: &str, + _condition_function_id: Option, + _queue_config: Option, + ) { + } + + async fn unsubscribe(&self, _topic: &str, _id: &str) {} + + async fn redrive_dlq(&self, _topic: &str) -> anyhow::Result { + Ok(0) + } + + async fn redrive_dlq_message( + &self, + _topic: &str, + _message_id: &str, + ) -> anyhow::Result { + Ok(false) + } + + async fn discard_dlq_message( + &self, + _topic: &str, + _message_id: &str, + ) -> anyhow::Result { + Ok(false) + } + + async fn dlq_count(&self, _topic: &str) -> anyhow::Result { + Ok(0) + } + + async fn shutdown(&self) {} + } + + #[tokio::test] + async fn current_reflects_replace() { + let first = Arc::new(CountingAdapter::default()); + let swappable = SwappableAdapter::new(first.clone()); + swappable.enqueue("demo", Value::Null, None, None).await; + assert_eq!(first.enqueue_calls.load(Ordering::SeqCst), 1); + + let second = Arc::new(CountingAdapter::default()); + swappable.replace(second.clone()).await; + swappable.enqueue("demo", Value::Null, None, None).await; + + assert_eq!(first.enqueue_calls.load(Ordering::SeqCst), 1); + assert_eq!(second.enqueue_calls.load(Ordering::SeqCst), 1); + } +} diff --git a/queue/src/boot.rs b/queue/src/boot.rs index 0303887ad..1292d6790 100644 --- a/queue/src/boot.rs +++ b/queue/src/boot.rs @@ -7,8 +7,10 @@ use iii_sdk::{IIIClient, RegisterTriggerType}; use serde::Deserialize; use tokio::sync::{Mutex, RwLock}; +use crate::adapter::SwappableAdapter; +use crate::adapters::builtin::BuiltinAdapter; use crate::config::QueueConfig; -use crate::store::{FileStore, InMemoryStore, QueueStore, SwappableStore}; +use crate::store::{FileStore, InMemoryStore, QueueStore}; use crate::trigger::{IiiInvoker, QueueTriggerHandler, SubscriberSpec}; use crate::TRIGGER_TYPE; @@ -19,7 +21,7 @@ pub type ConfigCell = Arc>>; pub type ApplyLock = Arc>; pub struct BootHandle { - pub store: Arc, + pub adapter: Arc, pub trigger_handler: QueueTriggerHandler, pub config: ConfigCell, pub apply_lock: ApplyLock, @@ -34,14 +36,17 @@ impl BootHandle { pub async fn start(iii: Arc, config: QueueConfig) -> anyhow::Result { guard_against_builtin_iii_queue(&iii).await?; - let store = Arc::new(SwappableStore::new(build_store(&config).await?)); + let store = build_store(&config).await?; + let invoker = Arc::new(IiiInvoker::new(iii.clone())); + let adapter = Arc::new(SwappableAdapter::new(Arc::new(BuiltinAdapter::new( + store, invoker, + )))); let config = Arc::new(RwLock::new(Arc::new(config.normalized()))); let apply_lock = Arc::new(Mutex::new(())); - crate::functions::register_all(&iii, store.clone()); + crate::functions::register_all(&iii, adapter.clone()); - let trigger_handler = - QueueTriggerHandler::new(store.clone(), Arc::new(IiiInvoker::new(iii.clone()))); + let trigger_handler = QueueTriggerHandler::new(adapter.clone()); let _ = iii.register_trigger_type( RegisterTriggerType::new( TRIGGER_TYPE, @@ -52,7 +57,7 @@ pub async fn start(iii: Arc, config: QueueConfig) -> anyhow::Result Result { pub fn register_config_trigger( iii: &Arc, - store: Arc, + adapter: Arc, trigger_handler: QueueTriggerHandler, config: ConfigCell, apply_lock: ApplyLock, @@ -66,12 +67,12 @@ pub fn register_config_trigger( CONFIG_FN_ID, RegisterFunction::new_async(move |_payload: ConfigChangeRequest| { let engine = engine.clone(); - let store = store.clone(); + let adapter = adapter.clone(); let trigger_handler = trigger_handler.clone(); let config = config.clone(); let apply_lock = apply_lock.clone(); async move { - on_config_change(&engine, store, trigger_handler, config, apply_lock).await; + on_config_change(engine, adapter, trigger_handler, config, apply_lock).await; Ok::<_, Error>(ConfigChangeAck { ok: true }) } }) @@ -91,15 +92,15 @@ pub fn register_config_trigger( } async fn on_config_change( - iii: &IIIClient, - store: Arc, + iii: Arc, + adapter: Arc, trigger_handler: QueueTriggerHandler, config: ConfigCell, apply_lock: ApplyLock, ) { let _guard = apply_lock.lock().await; - let next = match fetch_config(iii).await { + let next = match fetch_config(&iii).await { Ok(config) => config.normalized(), Err(err) => { tracing::error!(error = %err, "queue config-change: fetch failed; keeping previous config"); @@ -117,8 +118,12 @@ async fn on_config_change( return; } }; + let new_adapter = Arc::new(BuiltinAdapter::new( + new_store, + Arc::new(IiiInvoker::new(iii.clone())), + )); trigger_handler.shutdown().await; - store.replace(new_store).await; + adapter.replace(new_adapter).await; *config.write().await = Arc::new(next); for registration in registrations { diff --git a/queue/src/functions.rs b/queue/src/functions.rs index a0c450b04..1f5776766 100644 --- a/queue/src/functions.rs +++ b/queue/src/functions.rs @@ -8,7 +8,8 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::store::{Job, QueueStore}; +use crate::adapter::{QueueAdapter, SwappableAdapter}; +use crate::store::Job; pub const PUBLISH_FN_ID: &str = "iii::durable::publish"; pub const REDRIVE_FN_ID: &str = "iii::queue::redrive"; @@ -105,110 +106,110 @@ pub struct DlqMessage { pub size_bytes: u64, } -pub fn register_all(iii: &Arc, store: Arc) { - let publish_store = store.clone(); +pub fn register_all(iii: &Arc, adapter: Arc) { + let publish_adapter = adapter.clone(); iii.register_function( PUBLISH_FN_ID, RegisterFunction::new_async(move |input: PublishInput| { - let store = publish_store.clone(); - async move { publish(store, input).await } + let adapter = publish_adapter.clone(); + async move { publish(adapter, input).await } }) .description("Enqueue a message"), ); - let redrive_store = store.clone(); + let redrive_adapter = adapter.clone(); iii.register_function( REDRIVE_FN_ID, RegisterFunction::new_async(move |input: RedriveInput| { - let store = redrive_store.clone(); - async move { redrive(store, input).await } + let adapter = redrive_adapter.clone(); + async move { redrive(adapter, input).await } }) .description("Redrive all DLQ messages back to the main queue"), ); - let redrive_message_store = store.clone(); + let redrive_message_adapter = adapter.clone(); iii.register_function( REDRIVE_MESSAGE_FN_ID, RegisterFunction::new_async(move |input: RedriveSingleInput| { - let store = redrive_message_store.clone(); - async move { redrive_message(store, input).await } + let adapter = redrive_message_adapter.clone(); + async move { redrive_message(adapter, input).await } }) .description("Redrive a single DLQ message by ID back to the main queue"), ); - let discard_store = store.clone(); + let discard_adapter = adapter.clone(); iii.register_function( DISCARD_MESSAGE_FN_ID, RegisterFunction::new_async(move |input: RedriveSingleInput| { - let store = discard_store.clone(); - async move { discard_message(store, input).await } + let adapter = discard_adapter.clone(); + async move { discard_message(adapter, input).await } }) .description("Discard (purge) a single DLQ message by ID"), ); - let list_store = store.clone(); + let list_adapter = adapter.clone(); iii.register_function( LIST_TOPICS_FN_ID, RegisterFunction::new_async(move |_input: Value| { - let store = list_store.clone(); - async move { list_topics(store).await } + let adapter = list_adapter.clone(); + async move { list_topics(adapter).await } }) .description("List all queue topics"), ); - let stats_store = store.clone(); + let stats_adapter = adapter.clone(); iii.register_function( TOPIC_STATS_FN_ID, RegisterFunction::new_async(move |input: TopicStatsInput| { - let store = stats_store.clone(); - async move { topic_stats(store, input).await } + let adapter = stats_adapter.clone(); + async move { topic_stats(adapter, input).await } }) .description("Get stats for a queue topic"), ); - let dlq_topics_store = store.clone(); + let dlq_topics_adapter = adapter.clone(); iii.register_function( DLQ_TOPICS_FN_ID, RegisterFunction::new_async(move |_input: Value| { - let store = dlq_topics_store.clone(); - async move { dlq_topics(store).await } + let adapter = dlq_topics_adapter.clone(); + async move { dlq_topics(adapter).await } }) .description("List DLQ topics with counts"), ); - let dlq_messages_store = store; + let dlq_messages_adapter = adapter; iii.register_function( DLQ_MESSAGES_FN_ID, RegisterFunction::new_async(move |input: DlqMessagesInput| { - let store = dlq_messages_store.clone(); - async move { dlq_messages(store, input).await } + let adapter = dlq_messages_adapter.clone(); + async move { dlq_messages(adapter, input).await } }) .description("Browse DLQ messages"), ); } pub async fn publish( - store: Arc, + adapter: Arc, input: PublishInput, ) -> Result, Error> { if input.topic.is_empty() { return Err(Error::Handler("Topic is not set".to_string())); } - store - .enqueue(&input.topic, input.data) - .await - .map_err(|e| Error::Handler(e.to_string()))?; + adapter.enqueue(&input.topic, input.data, None, None).await; Ok(None) } pub async fn redrive( - store: Arc, + adapter: Arc, input: RedriveInput, ) -> Result { if input.queue.is_empty() { return Err(Error::Handler("Queue name is required".to_string())); } - let redriven = store.redrive_dlq(&input.queue).await; + let redriven = adapter + .redrive_dlq(&input.queue) + .await + .map_err(|e| Error::Handler(e.to_string()))?; Ok(RedriveResult { queue: input.queue, redriven, @@ -216,13 +217,14 @@ pub async fn redrive( } pub async fn redrive_message( - store: Arc, + adapter: Arc, input: RedriveSingleInput, ) -> Result { validate_single_input(&input)?; - let found = store + let found = adapter .redrive_dlq_message(&input.queue, &input.message_id) - .await; + .await + .map_err(|e| Error::Handler(e.to_string()))?; Ok(RedriveSingleResult { queue: input.queue, message_id: input.message_id, @@ -231,13 +233,14 @@ pub async fn redrive_message( } pub async fn discard_message( - store: Arc, + adapter: Arc, input: RedriveSingleInput, ) -> Result { validate_single_input(&input)?; - let found = store + let found = adapter .discard_dlq_message(&input.queue, &input.message_id) - .await; + .await + .map_err(|e| Error::Handler(e.to_string()))?; Ok(RedriveSingleResult { queue: input.queue, message_id: input.message_id, @@ -245,13 +248,15 @@ pub async fn discard_message( }) } -pub async fn list_topics(store: Arc) -> Result, Error> { - Ok(store +pub async fn list_topics(adapter: Arc) -> Result, Error> { + let topics = adapter .list_topics() .await + .map_err(|e| Error::Handler(e.to_string()))?; + Ok(topics .into_iter() - .map(|name| TopicInfo { - name, + .map(|topic| TopicInfo { + name: topic.name, broker_type: "builtin".to_string(), subscriber_count: 0, }) @@ -259,13 +264,16 @@ pub async fn list_topics(store: Arc) -> Result, E } pub async fn topic_stats( - store: Arc, + adapter: Arc, input: TopicStatsInput, ) -> Result { if input.topic.is_empty() { return Err(Error::Handler("topic is required".to_string())); } - let stats = store.topic_stats(&input.topic).await; + let stats = adapter + .topic_stats(&input.topic) + .await + .map_err(|e| Error::Handler(e.to_string()))?; Ok(TopicStatsOutput { depth: stats.depth, consumer_count: 0, @@ -274,32 +282,49 @@ pub async fn topic_stats( }) } -pub async fn dlq_topics(store: Arc) -> Result, Error> { - Ok(store - .dlq_topics() +/// Ported from the engine builtin's `console_dlq_topics` +/// (`engine/src/workers/queue/queue.rs:524-560`): iterate every known topic +/// and pair it with its DLQ depth, keeping only topics that actually have +/// dead-lettered messages. A per-topic `dlq_count` failure is treated as +/// zero (matching the engine's `unwrap_or(0)`) rather than failing the +/// whole call. +pub async fn dlq_topics(adapter: Arc) -> Result, Error> { + let topics = adapter + .list_topics() .await - .into_iter() - .map(|(topic, message_count)| DlqTopicInfo { - topic, - broker_type: "builtin".to_string(), - message_count, - }) - .collect()) + .map_err(|e| Error::Handler(e.to_string()))?; + + let mut dlq_topics = Vec::new(); + for topic in topics { + let dlq_count = adapter.dlq_count(&topic.name).await.unwrap_or(0); + if dlq_count > 0 { + dlq_topics.push(DlqTopicInfo { + topic: topic.name, + broker_type: "builtin".to_string(), + message_count: dlq_count, + }); + } + } + Ok(dlq_topics) } pub async fn dlq_messages( - store: Arc, + adapter: Arc, input: DlqMessagesInput, ) -> Result, Error> { if input.topic.is_empty() { return Err(Error::Handler("topic is required".to_string())); } - let messages = store - .dlq_messages(&input.topic, input.offset.saturating_add(input.limit)) + let count = input.offset.saturating_add(input.limit) as usize; + let values = adapter + .dlq_messages(&input.topic, count) .await + .map_err(|e| Error::Handler(e.to_string()))?; + let messages = values .into_iter() .skip(input.offset as usize) .take(input.limit as usize) + .filter_map(|value| serde_json::from_value::(value).ok()) .map(dlq_message_from_job) .collect(); Ok(messages) @@ -330,26 +355,176 @@ fn dlq_message_from_job(job: Job) -> DlqMessage { #[cfg(test)] mod tests { use super::*; - use crate::store::{InMemoryStore, QueueStore}; + use crate::adapter::TopicInfo as AdapterTopicInfo; + use crate::store::TopicStats; + use crate::subscriber_config::SubscriberQueueConfig; use serde_json::json; + use std::sync::Mutex as StdMutex; + + #[derive(Debug, Clone, PartialEq)] + enum Call { + Enqueue { topic: String, data: Value }, + RedriveDlq { topic: String }, + RedriveDlqMessage { topic: String, message_id: String }, + DiscardDlqMessage { topic: String, message_id: String }, + DlqCount { topic: String }, + ListTopics, + TopicStats { topic: String }, + DlqMessages { topic: String, count: usize }, + } - fn store() -> Arc { - Arc::new(InMemoryStore::new()) + /// Records every call and lets a test script canned return values (an + /// `Err` here is how a real adapter surfaces a transport-specific + /// failure, e.g. redis's "does not support DLQ" -- functions must + /// propagate it 1:1). + #[derive(Default)] + struct MockAdapter { + calls: StdMutex>, + redrive_dlq_result: StdMutex>>, + redrive_dlq_message_result: StdMutex>>, + discard_dlq_message_result: StdMutex>>, + dlq_count_result: StdMutex>>, + list_topics_result: StdMutex>>>, + topic_stats_result: StdMutex>>, + dlq_messages_result: StdMutex>>>, } - async fn move_one_to_dlq(store: Arc, queue: &str) -> String { - store.enqueue(queue, json!({"dead": true})).await.unwrap(); - let job = store.dequeue(queue).await.unwrap(); - let id = job.id.clone(); - store.nack(queue, job, 1, 1).await; - id + impl MockAdapter { + fn calls(&self) -> Vec { + self.calls.lock().unwrap().clone() + } + } + + #[async_trait::async_trait] + impl QueueAdapter for MockAdapter { + async fn enqueue( + &self, + topic: &str, + data: Value, + _traceparent: Option, + _baggage: Option, + ) { + self.calls.lock().unwrap().push(Call::Enqueue { + topic: topic.to_string(), + data, + }); + } + + async fn subscribe( + &self, + _topic: &str, + _id: &str, + _function_id: &str, + _condition_function_id: Option, + _queue_config: Option, + ) { + } + + async fn unsubscribe(&self, _topic: &str, _id: &str) {} + + async fn redrive_dlq(&self, topic: &str) -> anyhow::Result { + self.calls.lock().unwrap().push(Call::RedriveDlq { + topic: topic.to_string(), + }); + self.redrive_dlq_result + .lock() + .unwrap() + .take() + .unwrap_or(Ok(0)) + } + + async fn redrive_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { + self.calls.lock().unwrap().push(Call::RedriveDlqMessage { + topic: topic.to_string(), + message_id: message_id.to_string(), + }); + self.redrive_dlq_message_result + .lock() + .unwrap() + .take() + .unwrap_or(Ok(false)) + } + + async fn discard_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { + self.calls.lock().unwrap().push(Call::DiscardDlqMessage { + topic: topic.to_string(), + message_id: message_id.to_string(), + }); + self.discard_dlq_message_result + .lock() + .unwrap() + .take() + .unwrap_or(Ok(false)) + } + + async fn dlq_count(&self, topic: &str) -> anyhow::Result { + self.calls.lock().unwrap().push(Call::DlqCount { + topic: topic.to_string(), + }); + self.dlq_count_result + .lock() + .unwrap() + .take() + .unwrap_or(Ok(0)) + } + + async fn dlq_messages(&self, topic: &str, count: usize) -> anyhow::Result> { + self.calls.lock().unwrap().push(Call::DlqMessages { + topic: topic.to_string(), + count, + }); + self.dlq_messages_result + .lock() + .unwrap() + .take() + .unwrap_or(Ok(vec![])) + } + + async fn list_topics(&self) -> anyhow::Result> { + self.calls.lock().unwrap().push(Call::ListTopics); + self.list_topics_result + .lock() + .unwrap() + .take() + .unwrap_or(Ok(vec![])) + } + + async fn topic_stats(&self, topic: &str) -> anyhow::Result { + self.calls.lock().unwrap().push(Call::TopicStats { + topic: topic.to_string(), + }); + self.topic_stats_result + .lock() + .unwrap() + .take() + .unwrap_or(Ok(TopicStats::default())) + } + + async fn shutdown(&self) {} + } + + fn adapter() -> (Arc, Arc) { + let mock = Arc::new(MockAdapter::default()); + let dyn_adapter: Arc = mock.clone(); + (Arc::new(SwappableAdapter::new(dyn_adapter)), mock) + } + + fn job_value(id: &str, payload: Value, attempts: u32) -> Value { + serde_json::to_value(Job { + id: id.to_string(), + payload, + attempts, + enqueued_at_ms: 0, + ready_at_ms: 0, + }) + .unwrap() } #[tokio::test] - async fn publish_enqueues_message() { - let store = store(); + async fn publish_calls_adapter_enqueue() { + let (adapter, mock) = adapter(); publish( - store.clone(), + adapter, PublishInput { topic: "demo".to_string(), data: json!({"hello": "world"}), @@ -358,76 +533,141 @@ mod tests { .await .unwrap(); assert_eq!( - store.dequeue("demo").await.unwrap().payload, - json!({"hello": "world"}) + mock.calls(), + vec![Call::Enqueue { + topic: "demo".to_string(), + data: json!({"hello": "world"}), + }] ); } #[tokio::test] - async fn redrive_moves_dlq_back() { - let store = store(); - move_one_to_dlq(store.clone(), "demo").await; + async fn publish_rejects_empty_topic() { + let (adapter, mock) = adapter(); + let err = publish( + adapter, + PublishInput { + topic: String::new(), + data: Value::Null, + }, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("Topic")); + assert!(mock.calls().is_empty()); + } + + #[tokio::test] + async fn redrive_returns_adapter_count() { + let (adapter, mock) = adapter(); + *mock.redrive_dlq_result.lock().unwrap() = Some(Ok(3)); let result = redrive( - store.clone(), + adapter, RedriveInput { queue: "demo".to_string(), }, ) .await .unwrap(); - assert_eq!(result.redriven, 1); - assert!(store.dequeue("demo").await.is_some()); + assert_eq!(result.redriven, 3); + assert_eq!( + mock.calls(), + vec![Call::RedriveDlq { + topic: "demo".to_string(), + }] + ); + } + + #[tokio::test] + async fn redrive_propagates_adapter_error() { + let (adapter, mock) = adapter(); + *mock.redrive_dlq_result.lock().unwrap() = + Some(Err(anyhow::anyhow!("redis does not support DLQ"))); + let err = redrive( + adapter, + RedriveInput { + queue: "demo".to_string(), + }, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("does not support DLQ")); } #[tokio::test] async fn redrive_message_moves_one_dlq_message() { - let store = store(); - let id = move_one_to_dlq(store.clone(), "demo").await; + let (adapter, mock) = adapter(); + *mock.redrive_dlq_message_result.lock().unwrap() = Some(Ok(true)); let result = redrive_message( - store.clone(), + adapter, RedriveSingleInput { queue: "demo".to_string(), - message_id: id.clone(), + message_id: "m1".to_string(), }, ) .await .unwrap(); assert_eq!(result.redriven, 1); - assert_eq!(store.dequeue("demo").await.unwrap().id, id); + assert_eq!( + mock.calls(), + vec![Call::RedriveDlqMessage { + topic: "demo".to_string(), + message_id: "m1".to_string(), + }] + ); } #[tokio::test] async fn discard_message_purges_one_dlq_message() { - let store = store(); - let id = move_one_to_dlq(store.clone(), "demo").await; + let (adapter, mock) = adapter(); + *mock.discard_dlq_message_result.lock().unwrap() = Some(Ok(true)); let result = discard_message( - store.clone(), + adapter, RedriveSingleInput { queue: "demo".to_string(), - message_id: id, + message_id: "m1".to_string(), }, ) .await .unwrap(); assert_eq!(result.redriven, 1); - assert!(store.dlq_messages("demo", 10).await.is_empty()); + assert_eq!( + mock.calls(), + vec![Call::DiscardDlqMessage { + topic: "demo".to_string(), + message_id: "m1".to_string(), + }] + ); } #[tokio::test] - async fn list_and_stats_return_store_state() { - let store = store(); - store.enqueue("demo", json!("ready")).await.unwrap(); - move_one_to_dlq(store.clone(), "demo").await; + async fn list_and_stats_return_adapter_state() { + let (adapter, _mock) = adapter(); + { + let mock = _mock.clone(); + *mock.list_topics_result.lock().unwrap() = Some(Ok(vec![AdapterTopicInfo { + name: "demo".to_string(), + depth: 1, + }])); + } + let topics = list_topics(adapter.clone()).await.unwrap(); assert_eq!( - list_topics(store.clone()).await.unwrap()[0], + topics[0], TopicInfo { name: "demo".to_string(), broker_type: "builtin".to_string(), subscriber_count: 0, } ); + + *_mock.topic_stats_result.lock().unwrap() = Some(Ok(TopicStats { + depth: 1, + dlq_depth: 1, + delivered: 0, + failed: 1, + })); let stats = topic_stats( - store.clone(), + adapter, TopicStatsInput { topic: "demo".to_string(), }, @@ -439,19 +679,34 @@ mod tests { } #[tokio::test] - async fn dlq_browse_returns_topics_and_messages() { - let store = store(); - let id = move_one_to_dlq(store.clone(), "demo").await; - assert_eq!( - dlq_topics(store.clone()).await.unwrap(), - vec![DlqTopicInfo { - topic: "demo".to_string(), - broker_type: "builtin".to_string(), - message_count: 1, - }] - ); + async fn dlq_topics_only_includes_topics_with_positive_dlq_depth() { + let (adapter, mock) = adapter(); + *mock.list_topics_result.lock().unwrap() = Some(Ok(vec![ + AdapterTopicInfo { + name: "demo".to_string(), + depth: 0, + }, + AdapterTopicInfo { + name: "empty".to_string(), + depth: 0, + }, + ])); + // dlq_count is called once per topic in list order; queue the "demo" + // answer first, then "empty"'s. + *mock.dlq_count_result.lock().unwrap() = Some(Ok(1)); + let result = dlq_topics(adapter.clone()).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].topic, "demo"); + assert_eq!(result[0].message_count, 1); + } + + #[tokio::test] + async fn dlq_browse_returns_messages() { + let (adapter, mock) = adapter(); + *mock.dlq_messages_result.lock().unwrap() = + Some(Ok(vec![job_value("m1", json!({"dead": true}), 1)])); let messages = dlq_messages( - store, + adapter, DlqMessagesInput { topic: "demo".to_string(), offset: 0, @@ -460,7 +715,30 @@ mod tests { ) .await .unwrap(); - assert_eq!(messages[0].id, id); + assert_eq!(messages[0].id, "m1"); assert_eq!(messages[0].retries, 1); + assert_eq!( + mock.calls(), + vec![Call::DlqMessages { + topic: "demo".to_string(), + count: 50, + }] + ); + } + + #[tokio::test] + async fn dlq_messages_rejects_empty_topic() { + let (adapter, _mock) = adapter(); + let err = dlq_messages( + adapter, + DlqMessagesInput { + topic: String::new(), + offset: 0, + limit: 50, + }, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("topic")); } } diff --git a/queue/src/main.rs b/queue/src/main.rs index a2e693bf7..cc6848e74 100644 --- a/queue/src/main.rs +++ b/queue/src/main.rs @@ -89,7 +89,7 @@ async fn main() -> Result<()> { configuration::register_config_trigger( &iii, - boot.store.clone(), + boot.adapter.clone(), boot.trigger_handler.clone(), boot.config.clone(), boot.apply_lock.clone(), diff --git a/queue/src/trigger.rs b/queue/src/trigger.rs index 3fd880aed..757394fae 100644 --- a/queue/src/trigger.rs +++ b/queue/src/trigger.rs @@ -1,8 +1,13 @@ //! Durable subscriber trigger handling. +//! +//! The handler itself no longer owns any consumer loop or store access -- +//! `register_trigger`/`unregister_trigger` translate a `TriggerConfig` into +//! [`crate::adapter::QueueAdapter::subscribe`]/`unsubscribe` calls against the +//! (possibly hot-swapped) [`SwappableAdapter`]. All delivery, retry, and DLQ +//! behavior lives in the adapter (e.g. [`crate::adapters::builtin::BuiltinAdapter`]). use std::collections::HashMap; use std::sync::Arc; -use std::time::Duration; use async_trait::async_trait; use iii_sdk::errors::Error; @@ -13,15 +18,10 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::Value; use tokio::sync::Mutex; -use tokio::task::JoinHandle; -use crate::store::QueueStore; +use crate::adapter::{QueueAdapter, SwappableAdapter}; use crate::subscriber_config::SubscriberQueueConfig; -pub const DEFAULT_MAX_RETRIES: u32 = 3; -pub const DEFAULT_BACKOFF_MS: u64 = 1000; -const IDLE_POLL_MS: u64 = 50; - #[async_trait] pub trait Invoker: Send + Sync + 'static { async fn call(&self, function_id: &str, payload: Value) -> Result, String>; @@ -80,41 +80,48 @@ pub struct RegisteredSubscriber { pub spec: SubscriberSpec, } -struct Consumer { - registration: RegisteredSubscriber, - handle: JoinHandle<()>, +/// Merge the spec's legacy top-level `max_retries`/`backoff_ms` into its +/// `queue_config`. Precedence: explicit top-level field wins, then whatever +/// `queue_config` already carries, then the adapter's own defaults (3 +/// retries / 1000ms backoff) when both are absent -- the adapter applies +/// those defaults itself when the returned fields are `None`. +fn merged_queue_config(spec: &SubscriberSpec) -> SubscriberQueueConfig { + let mut config = spec.queue_config.clone().unwrap_or_default(); + if let Some(max_retries) = spec.max_retries { + config.max_retries = Some(max_retries); + } + if let Some(backoff_ms) = spec.backoff_ms { + config.backoff_delay_ms = Some(backoff_ms); + } + config } #[derive(Clone)] pub struct QueueTriggerHandler { - store: Arc, - invoker: Arc, - consumers: Arc>>, + adapter: Arc, + registrations: Arc>>, } impl QueueTriggerHandler { - pub fn new(store: Arc, invoker: Arc) -> Self { + pub fn new(adapter: Arc) -> Self { Self { - store, - invoker, - consumers: Arc::new(Mutex::new(HashMap::new())), + adapter, + registrations: Arc::new(Mutex::new(HashMap::new())), } } + /// Snapshot of every currently registered subscriber -- consumed by the + /// configuration hot-swap path to re-subscribe against the newly built + /// adapter after a transport swap. pub async fn registrations(&self) -> Vec { - self.consumers - .lock() - .await - .values() - .map(|consumer| consumer.registration.clone()) - .collect() + self.registrations.lock().await.values().cloned().collect() } + /// Clears every tracked registration and tears down the current + /// adapter's active consumers. pub async fn shutdown(&self) { - let consumers = self.consumers.lock().await.drain().collect::>(); - for (_, consumer) in consumers { - consumer.handle.abort(); - } + self.registrations.lock().await.clear(); + self.adapter.shutdown().await; } pub async fn register_subscriber( @@ -125,28 +132,40 @@ impl QueueTriggerHandler { return Err("queue is required for durable:subscriber trigger".to_string()); } - let mut consumers = self.consumers.lock().await; - if consumers.contains_key(®istration.trigger_id) { + let mut registrations = self.registrations.lock().await; + if registrations.contains_key(®istration.trigger_id) { return Err(format!( "durable:subscriber trigger '{}' is already registered", registration.trigger_id )); } - let handle = spawn_consumer_loop( - self.store.clone(), - self.invoker.clone(), - registration.clone(), - ); - consumers.insert( - registration.trigger_id.clone(), - Consumer { - registration, - handle, - }, - ); + let queue_config = merged_queue_config(®istration.spec); + self.adapter + .subscribe( + ®istration.spec.queue, + ®istration.trigger_id, + ®istration.function_id, + registration.spec.condition_function_id.clone(), + Some(queue_config), + ) + .await; + + registrations.insert(registration.trigger_id.clone(), registration); Ok(()) } + + /// Unregister by bare trigger id -- the queue/topic name is recovered + /// from the stored registration, since an unregister `TriggerConfig` + /// carries only the id. + pub async fn unregister(&self, trigger_id: &str) { + let registration = self.registrations.lock().await.remove(trigger_id); + if let Some(registration) = registration { + self.adapter + .unsubscribe(®istration.spec.queue, trigger_id) + .await; + } + } } #[async_trait] @@ -164,98 +183,113 @@ impl TriggerHandler for QueueTriggerHandler { } async fn unregister_trigger(&self, config: TriggerConfig) -> Result<(), Error> { - if let Some(consumer) = self.consumers.lock().await.remove(&config.id) { - consumer.handle.abort(); - } + self.unregister(&config.id).await; Ok(()) } } -fn spawn_consumer_loop( - store: Arc, - invoker: Arc, - registration: RegisteredSubscriber, -) -> JoinHandle<()> { - tokio::spawn(async move { - let queue = registration.spec.queue.clone(); - let max_retries = effective_max_retries(®istration.spec); - let backoff_ms = effective_backoff_ms(®istration.spec); - - loop { - let Some(job) = store.dequeue(&queue).await else { - tokio::time::sleep(Duration::from_millis(IDLE_POLL_MS)).await; - continue; - }; - - if let Some(condition_id) = registration.spec.condition_function_id.as_deref() { - match invoker.call(condition_id, job.payload.clone()).await { - Ok(Some(Value::Bool(false))) => { - store.ack(&queue, &job.id).await; - continue; - } - Ok(_) => {} - Err(_) => { - store.nack(&queue, job, max_retries, backoff_ms).await; - continue; - } - } - } - - match invoker - .call(®istration.function_id, job.payload.clone()) - .await - { - Ok(_) => store.ack(&queue, &job.id).await, - Err(_) => store.nack(&queue, job, max_retries, backoff_ms).await, - } - } - }) -} - -fn effective_max_retries(spec: &SubscriberSpec) -> u32 { - spec.max_retries - .or_else(|| spec.queue_config.as_ref().and_then(|c| c.max_retries)) - .unwrap_or(DEFAULT_MAX_RETRIES) -} - -fn effective_backoff_ms(spec: &SubscriberSpec) -> u64 { - spec.backoff_ms - .or_else(|| spec.queue_config.as_ref().and_then(|c| c.backoff_delay_ms)) - .unwrap_or(DEFAULT_BACKOFF_MS) -} - #[cfg(test)] mod tests { use super::*; - use crate::store::{InMemoryStore, QueueStore}; + use crate::adapter::TopicInfo; + use crate::store::TopicStats; use serde_json::json; - use std::future::Future; - use std::sync::atomic::{AtomicBool, Ordering}; - use tokio::time::{sleep, Duration, Instant}; + use std::sync::Mutex as StdMutex; + + #[derive(Debug, Clone, PartialEq)] + enum Call { + Subscribe { + topic: String, + id: String, + function_id: String, + condition_function_id: Option, + queue_config: Option, + }, + Unsubscribe { + topic: String, + id: String, + }, + } #[derive(Default)] - struct FakeInvoker { - calls: Mutex>, - fail_backend: AtomicBool, - condition_value: Mutex>, + struct MockAdapter { + calls: StdMutex>, + } + + impl MockAdapter { + fn calls(&self) -> Vec { + self.calls.lock().unwrap().clone() + } } #[async_trait] - impl Invoker for FakeInvoker { - async fn call(&self, function_id: &str, payload: Value) -> Result, String> { - self.calls - .lock() - .await - .push((function_id.to_string(), payload)); - if function_id == "condition" { - return Ok(self.condition_value.lock().await.clone()); - } - if self.fail_backend.load(Ordering::SeqCst) { - Err("backend failed".to_string()) - } else { - Ok(Some(json!({"ok": true}))) - } + impl QueueAdapter for MockAdapter { + async fn enqueue( + &self, + _topic: &str, + _data: Value, + _traceparent: Option, + _baggage: Option, + ) { + } + + async fn subscribe( + &self, + topic: &str, + id: &str, + function_id: &str, + condition_function_id: Option, + queue_config: Option, + ) { + self.calls.lock().unwrap().push(Call::Subscribe { + topic: topic.to_string(), + id: id.to_string(), + function_id: function_id.to_string(), + condition_function_id, + queue_config, + }); + } + + async fn unsubscribe(&self, topic: &str, id: &str) { + self.calls.lock().unwrap().push(Call::Unsubscribe { + topic: topic.to_string(), + id: id.to_string(), + }); + } + + async fn redrive_dlq(&self, _topic: &str) -> anyhow::Result { + Ok(0) + } + + async fn redrive_dlq_message( + &self, + _topic: &str, + _message_id: &str, + ) -> anyhow::Result { + Ok(false) + } + + async fn discard_dlq_message( + &self, + _topic: &str, + _message_id: &str, + ) -> anyhow::Result { + Ok(false) } + + async fn dlq_count(&self, _topic: &str) -> anyhow::Result { + Ok(0) + } + + async fn list_topics(&self) -> anyhow::Result> { + Ok(vec![]) + } + + async fn topic_stats(&self, _topic: &str) -> anyhow::Result { + Ok(TopicStats::default()) + } + + async fn shutdown(&self) {} } fn trigger_config(id: &str, function_id: &str, config: Value) -> TriggerConfig { @@ -267,30 +301,16 @@ mod tests { } } - async fn wait_until(mut pred: F) - where - F: FnMut() -> Fut, - Fut: Future, - { - let deadline = Instant::now() + Duration::from_secs(2); - while Instant::now() < deadline { - if pred().await { - return; - } - sleep(Duration::from_millis(20)).await; - } - assert!(pred().await, "condition did not become true before timeout"); + fn handler_with_mock() -> (QueueTriggerHandler, Arc) { + let mock = Arc::new(MockAdapter::default()); + let adapter: Arc = mock.clone(); + let handler = QueueTriggerHandler::new(Arc::new(SwappableAdapter::new(adapter))); + (handler, mock) } #[tokio::test] - async fn register_spawns_loop_and_acks_on_success() { - let store: Arc = Arc::new(InMemoryStore::new()); - let invoker = Arc::new(FakeInvoker::default()); - let handler = QueueTriggerHandler::new(store.clone(), invoker.clone()); - store - .enqueue("demo", json!({"hello": "world"})) - .await - .unwrap(); + async fn register_calls_adapter_subscribe_with_merged_config() { + let (handler, mock) = handler_with_mock(); handler .register_trigger(trigger_config( @@ -301,55 +321,48 @@ mod tests { .await .unwrap(); - wait_until(|| { - let invoker = invoker.clone(); - async move { invoker.calls.lock().await.len() == 1 } - }) - .await; - assert_eq!(store.topic_stats("demo").await.delivered, 1); - assert!(store.dequeue("demo").await.is_none()); - handler.shutdown().await; + assert_eq!( + mock.calls(), + vec![Call::Subscribe { + topic: "demo".to_string(), + id: "t1".to_string(), + function_id: "backend".to_string(), + condition_function_id: None, + queue_config: Some(SubscriberQueueConfig { + max_retries: Some(2), + backoff_delay_ms: Some(1), + ..Default::default() + }), + }] + ); } #[tokio::test] - async fn failed_invocation_nacks_until_dlq() { - let store: Arc = Arc::new(InMemoryStore::new()); - let invoker = Arc::new(FakeInvoker::default()); - invoker.fail_backend.store(true, Ordering::SeqCst); - let handler = QueueTriggerHandler::new(store.clone(), invoker.clone()); - store - .enqueue("demo", json!({"hello": "world"})) - .await - .unwrap(); + async fn register_passes_condition_function_id_through() { + let (handler, mock) = handler_with_mock(); handler .register_trigger(trigger_config( "t1", "backend", - json!({"queue": "demo", "max_retries": 1, "backoff_ms": 1}), + json!({"queue": "demo", "condition_function_id": "condition"}), )) .await .unwrap(); - wait_until(|| { - let invoker = invoker.clone(); - async move { !invoker.calls.lock().await.is_empty() } - }) - .await; - wait_until(|| { - let store = store.clone(); - async move { !store.dlq_messages("demo", 10).await.is_empty() } - }) - .await; - assert_eq!(store.topic_stats("demo").await.failed, 1); - handler.shutdown().await; + let Call::Subscribe { + condition_function_id, + .. + } = &mock.calls()[0] + else { + panic!("expected a Subscribe call"); + }; + assert_eq!(condition_function_id.as_deref(), Some("condition")); } #[tokio::test] async fn register_rejects_empty_queue() { - let store: Arc = Arc::new(InMemoryStore::new()); - let invoker = Arc::new(FakeInvoker::default()); - let handler = QueueTriggerHandler::new(store, invoker); + let (handler, _mock) = handler_with_mock(); let err = handler .register_trigger(trigger_config("t1", "backend", json!({"queue": ""}))) .await @@ -359,66 +372,45 @@ mod tests { #[tokio::test] async fn duplicate_trigger_id_is_rejected() { - let store: Arc = Arc::new(InMemoryStore::new()); - let invoker = Arc::new(FakeInvoker::default()); - let handler = QueueTriggerHandler::new(store, invoker); + let (handler, mock) = handler_with_mock(); let cfg = trigger_config("t1", "backend", json!({"queue": "demo"})); handler.register_trigger(cfg.clone()).await.unwrap(); let err = handler.register_trigger(cfg).await.unwrap_err(); assert!(err.to_string().contains("already registered")); - handler.shutdown().await; + assert_eq!(mock.calls().len(), 1, "adapter.subscribe called only once"); } #[tokio::test] - async fn unregister_by_id_stops_consumption() { - let store: Arc = Arc::new(InMemoryStore::new()); - let invoker = Arc::new(FakeInvoker::default()); - let handler = QueueTriggerHandler::new(store.clone(), invoker.clone()); + async fn unregister_by_id_calls_adapter_unsubscribe() { + let (handler, mock) = handler_with_mock(); handler .register_trigger(trigger_config("t1", "backend", json!({"queue": "demo"}))) .await .unwrap(); + handler .unregister_trigger(trigger_config("t1", "", Value::Null)) .await .unwrap(); - store - .enqueue("demo", json!({"after": "stop"})) - .await - .unwrap(); - sleep(Duration::from_millis(120)).await; - assert!(invoker.calls.lock().await.is_empty()); + + assert_eq!( + mock.calls().last().unwrap(), + &Call::Unsubscribe { + topic: "demo".to_string(), + id: "t1".to_string(), + } + ); + assert!(handler.registrations().await.is_empty()); } #[tokio::test] - async fn explicit_false_condition_skips_and_acks() { - let store: Arc = Arc::new(InMemoryStore::new()); - let invoker = Arc::new(FakeInvoker::default()); - *invoker.condition_value.lock().await = Some(Value::Bool(false)); - let handler = QueueTriggerHandler::new(store.clone(), invoker.clone()); - store - .enqueue("demo", json!({"hello": "world"})) - .await - .unwrap(); - + async fn unregister_unknown_id_is_a_noop() { + let (handler, mock) = handler_with_mock(); handler - .register_trigger(trigger_config( - "t1", - "backend", - json!({"queue": "demo", "condition_function_id": "condition"}), - )) + .unregister_trigger(trigger_config("missing", "", Value::Null)) .await .unwrap(); - - wait_until(|| { - let invoker = invoker.clone(); - async move { invoker.calls.lock().await.len() == 1 } - }) - .await; - assert_eq!(invoker.calls.lock().await[0].0, "condition"); - assert_eq!(store.topic_stats("demo").await.delivered, 1); - assert!(store.dequeue("demo").await.is_none()); - handler.shutdown().await; + assert!(mock.calls().is_empty()); } #[test] @@ -432,7 +424,26 @@ mod tests { })) .unwrap(); assert_eq!(spec.queue, "demo"); - assert_eq!(effective_max_retries(&spec), 9); - assert_eq!(effective_backoff_ms(&spec), 25); + let merged = merged_queue_config(&spec); + assert_eq!(merged.max_retries, Some(9)); + assert_eq!(merged.backoff_delay_ms, Some(25)); + } + + #[test] + fn top_level_overrides_win_over_queue_config() { + let spec = SubscriberSpec { + queue: "demo".to_string(), + max_retries: Some(2), + backoff_ms: Some(1), + condition_function_id: None, + queue_config: Some(SubscriberQueueConfig { + max_retries: Some(9), + backoff_delay_ms: Some(25), + ..Default::default() + }), + }; + let merged = merged_queue_config(&spec); + assert_eq!(merged.max_retries, Some(2)); + assert_eq!(merged.backoff_delay_ms, Some(1)); } } From cb3586bf187a9d7447e2236386932b3eea90535a Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Tue, 7 Jul 2026 05:43:23 -0300 Subject: [PATCH 21/28] feat(queue): redis pub/sub adapter (1:1, DLQ-less like the builtin's) --- queue/Cargo.lock | 211 ++++++++++++++ queue/Cargo.toml | 2 + queue/src/adapters/mod.rs | 1 + queue/src/adapters/redis.rs | 551 +++++++++++++++++++++++++++++++++++ queue/tests/common/docker.rs | 114 ++++++++ queue/tests/common/mod.rs | 1 + queue/tests/e2e_redis.rs | 150 ++++++++++ 7 files changed, 1030 insertions(+) create mode 100644 queue/src/adapters/redis.rs create mode 100644 queue/tests/common/docker.rs create mode 100644 queue/tests/e2e_redis.rs diff --git a/queue/Cargo.lock b/queue/Cargo.lock index 30beba797..db5e5d4c6 100644 --- a/queue/Cargo.lock +++ b/queue/Cargo.lock @@ -67,6 +67,32 @@ version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -84,6 +110,21 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", +] + [[package]] name = "base64" version = "0.22.1" @@ -185,6 +226,29 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -210,6 +274,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "crypto-common" version = "0.1.7" @@ -269,6 +339,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -284,6 +381,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -291,6 +403,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -310,6 +423,12 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + [[package]] name = "futures-macro" version = "0.3.32" @@ -339,10 +458,13 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -647,7 +769,9 @@ dependencies = [ "anyhow", "async-trait", "clap", + "futures", "iii-sdk", + "redis", "schemars", "serde", "serde_json", @@ -802,6 +926,34 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -883,6 +1035,12 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1050,6 +1208,34 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "redis" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fa6f8e4b491d7a8ef3a9550a4d71969bd0064f46e32b8dbbcc7fc60dad94fed" +dependencies = [ + "arc-swap", + "arcstr", + "async-lock", + "backon", + "bytes", + "cfg-if", + "combine", + "futures-channel", + "futures-util", + "itoa", + "num-bigint", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "socket2", + "tokio", + "tokio-util", + "url", + "xxhash-rust", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1370,6 +1556,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1598,6 +1790,19 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "tower" version = "0.5.3" @@ -2215,6 +2420,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xxhash-rust" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" + [[package]] name = "yoke" version = "0.8.3" diff --git a/queue/Cargo.toml b/queue/Cargo.toml index 5a604fded..62a8c315a 100644 --- a/queue/Cargo.toml +++ b/queue/Cargo.toml @@ -30,6 +30,8 @@ tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } clap = { version = "4", features = ["derive"] } schemars = "0.8" uuid = { version = "1", features = ["v4"] } +futures = "0.3" +redis = { version = "1.0.1", features = ["tokio-comp", "connection-manager"] } [dev-dependencies] serial_test = "3" diff --git a/queue/src/adapters/mod.rs b/queue/src/adapters/mod.rs index 951f768a7..743047ae1 100644 --- a/queue/src/adapters/mod.rs +++ b/queue/src/adapters/mod.rs @@ -1,3 +1,4 @@ //! Transport adapters implementing [`crate::adapter::QueueAdapter`]. pub mod builtin; +pub mod redis; diff --git a/queue/src/adapters/redis.rs b/queue/src/adapters/redis.rs new file mode 100644 index 000000000..287a8a7d4 --- /dev/null +++ b/queue/src/adapters/redis.rs @@ -0,0 +1,551 @@ +//! Redis pub/sub transport adapter. +//! +//! 1:1 port of the engine builtin's `RedisAdapter` +//! (`engine/src/workers/queue/adapters/redis_adapter.rs`, whole file), +//! including its limitations: this transport is pub/sub only — it has no +//! DLQ, no retries, no message durability, and delivers each published +//! message to at most one subscriber connection per topic. Seams applied +//! for the standalone worker: +//! - `Arc` + `engine.call(...)` inside the pubsub task -> +//! `Arc` + `invoker.call(function_id, payload)`. +//! - The engine's `check_condition` helper (`engine/src/condition.rs`) is +//! reimplemented locally against `Invoker` with identical semantics: +//! `Ok(Some(v))` -> proceed unless `v` is the JSON boolean `false`, +//! `Ok(None)` -> proceed (with a warn log), `Err` -> skip the message +//! (logged), condition_function_id handling is otherwise untouched. +//! - Telemetry spans (the engine's `tracing::info_span!` + OTel +//! `with_parent_headers`) are dropped; the `__trace` envelope +//! (`traceparent`/`baggage`) is still built on `enqueue` and unwrapped on +//! `subscribe` exactly like the engine, so a future consumer could still +//! read it, it just isn't used to parent a span here. +//! - `QueueAdapter::shutdown` (required by this worker's trait, absent from +//! the engine's) aborts every subscription task, mirroring the engine's +//! "abort consumer tasks in place" behavior at process teardown. +//! +//! Testability note: [`RedisAdapter::new`]/[`RedisAdapter::from_config`] +//! connect to Redis eagerly (engine parity), so unit tests can't construct +//! a live instance without a running Redis. Two pieces of pure logic are +//! therefore extracted so they're covered without a connection: +//! - [`RedisAdapter::resolve_redis_url`] — config parsing. +//! - [`unsubscribe_locked`] — the unsubscribe decision table, taking the +//! already-locked map directly. +//! +//! The DLQ methods need no such extraction: they return a constant error +//! ([`DLQ_NOT_SUPPORTED`]) without touching `self` at all, so the constant +//! itself is the unit under test. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use futures::StreamExt; +use redis::{aio::ConnectionManager, AsyncCommands, Client}; +use serde_json::Value; +use tokio::sync::{mpsc, Mutex, RwLock}; +use tokio::task::JoinHandle; +use tokio::time::timeout; + +use crate::adapter::{QueueAdapter, QueueMessage, TopicInfo}; +use crate::subscriber_config::SubscriberQueueConfig; +use crate::trigger::Invoker; + +/// `engine/src/workers/redis.rs:9`. +const DEFAULT_REDIS_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); +/// `engine/src/workers/queue/adapters/redis_adapter.rs:84` (`make_adapter`). +const DEFAULT_REDIS_URL: &str = "redis://localhost:6379"; +/// Verbatim engine error string +/// (`engine/src/workers/queue/adapters/redis_adapter.rs:338` etc.). +const DLQ_NOT_SUPPORTED: &str = "RedisAdapter does not support DLQ operations (pub/sub only)"; + +pub struct RedisAdapter { + publisher: Arc>, + subscriber: Arc, + subscriptions: Arc>>, + invoker: Arc, +} + +struct SubscriptionInfo { + id: String, + task_handle: JoinHandle<()>, +} + +impl RedisAdapter { + /// Connects to Redis eagerly, exact engine parity: same timeout + /// constant and same error messages + /// (`engine/src/workers/queue/adapters/redis_adapter.rs:48-63`). + pub async fn new(redis_url: String, invoker: Arc) -> anyhow::Result { + let client = Client::open(redis_url.as_str())?; + + let manager = timeout( + DEFAULT_REDIS_CONNECTION_TIMEOUT, + client.get_connection_manager(), + ) + .await + .map_err(|_| { + anyhow::anyhow!( + "Redis connection timed out after {:?}. Please ensure Redis is running at: {}", + DEFAULT_REDIS_CONNECTION_TIMEOUT, + redis_url + ) + })? + .map_err(|e| anyhow::anyhow!("Failed to connect to Redis at {}: {}", redis_url, e))?; + + Ok(Self { + publisher: Arc::new(Mutex::new(manager)), + subscriber: Arc::new(client), + subscriptions: Arc::new(RwLock::new(HashMap::new())), + invoker, + }) + } + + /// Pure config-parsing logic split out of [`Self::from_config`] so it's + /// unit-testable without connecting to Redis. Engine parity: config key + /// `redis_url`, default `redis://localhost:6379` + /// (`engine/src/workers/queue/adapters/redis_adapter.rs:77-88`). + fn resolve_redis_url(config: Option<&Value>) -> String { + config + .and_then(|v| v.get("redis_url")) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .unwrap_or_else(|| DEFAULT_REDIS_URL.to_string()) + } + + /// Factory entry point: parses `config` (`{redis_url}`) and connects. + /// Called by the adapter factory that wires `config.adapter.name == + /// "redis"` into a live `RedisAdapter`. + pub async fn from_config( + config: Option<&Value>, + invoker: Arc, + ) -> anyhow::Result { + let redis_url = Self::resolve_redis_url(config); + Self::new(redis_url, invoker).await + } +} + +/// Port of the engine's `check_condition` (`engine/src/condition.rs`) +/// against `Invoker` instead of `EngineTrait`. Same semantics: `Ok(true)` +/// proceed, `Ok(false)` skip (only when the condition function explicitly +/// returned JSON `false`), `Err` propagates the invocation failure. +async fn check_condition( + invoker: &dyn Invoker, + condition_function_id: &str, + data: Value, +) -> Result { + match invoker.call(condition_function_id, data).await { + Ok(Some(result)) => Ok(result.as_bool() != Some(false)), + Ok(None) => { + tracing::warn!( + condition_function_id = %condition_function_id, + "Condition function returned no result" + ); + Ok(true) + } + Err(e) => Err(e), + } +} + +/// The unsubscribe decision table, given the already write-locked map. +/// Extracted so it's unit-testable without a live Redis connection (a +/// `RedisAdapter` can't be constructed without one). Mirrors +/// `engine/src/workers/queue/adapters/redis_adapter.rs:314-334`. +fn unsubscribe_locked(subs: &mut HashMap, topic: &str, id: &str) { + if let Some(sub_info) = subs.remove(topic) { + if sub_info.id == id { + tracing::debug!(topic = %topic, id = %id, "Unsubscribing from Redis channel"); + sub_info.task_handle.abort(); + } else { + tracing::warn!(topic = %topic, id = %id, "Subscription ID mismatch, not unsubscribing"); + subs.insert(topic.to_string(), sub_info); + } + } else { + tracing::warn!(topic = %topic, id = %id, "No active subscription found for topic"); + } +} + +#[async_trait] +impl QueueAdapter for RedisAdapter { + async fn enqueue( + &self, + topic: &str, + data: Value, + traceparent: Option, + baggage: Option, + ) { + let topic = topic.to_string(); + let publisher = Arc::clone(&self.publisher); + + tracing::debug!(topic = %topic, data = %data, "Publishing to Redis queue"); + + // Wrap data in an envelope that includes trace context — kept + // exactly like the engine even though nothing here parents a span + // off of it (see module doc). + let envelope = serde_json::json!({ + "__trace": { + "traceparent": traceparent, + "baggage": baggage, + }, + "data": data, + }); + + let json = match serde_json::to_string(&envelope) { + Ok(json) => json, + Err(e) => { + tracing::error!(error = %e, topic = %topic, "Failed to serialize queue data"); + return; + } + }; + + let mut conn = publisher.lock().await; + + if let Err(e) = conn.publish::<_, _, ()>(&topic, &json).await { + tracing::error!(error = %e, topic = %topic, "Failed to publish to Redis queue"); + } else { + tracing::debug!(topic = %topic, "Published to Redis queue"); + } + } + + async fn subscribe( + &self, + topic: &str, + id: &str, + function_id: &str, + condition_function_id: Option, + _queue_config: Option, + ) { + let topic = topic.to_string(); + let id = id.to_string(); + let function_id = function_id.to_string(); + let subscriber = Arc::clone(&self.subscriber); + let invoker = Arc::clone(&self.invoker); + let subscriptions = Arc::clone(&self.subscriptions); + + // Check if already subscribed — one subscription per topic per + // adapter instance, same guard as the engine. + let already_subscribed = { + let subs = subscriptions.read().await; + subs.contains_key(&topic) + }; + + if already_subscribed { + tracing::warn!(topic = %topic, id = %id, "Already subscribed to topic"); + return; + } + + let topic_for_task = topic.clone(); + let id_for_task = id.clone(); + let function_id_for_task = function_id.clone(); + let condition_function_id_for_task = condition_function_id.clone(); + + tracing::debug!(topic = %topic_for_task, id = %id_for_task, function_id = %function_id_for_task, "Subscribing to Redis channel"); + + let task_handle = tokio::spawn(async move { + let mut pubsub = match subscriber.get_async_pubsub().await { + Ok(pubsub) => pubsub, + Err(e) => { + tracing::error!(error = %e, topic = %topic_for_task, "Failed to get async pubsub connection"); + return; + } + }; + + if let Err(e) = pubsub.subscribe(&topic_for_task).await { + tracing::error!(error = %e, topic = %topic_for_task, "Failed to subscribe to Redis channel"); + return; + } + + tracing::debug!( + topic = %topic_for_task, + id = %id_for_task, + function_id = %function_id_for_task, + condition_function_id = ?condition_function_id_for_task, + "Subscribed to Redis channel" + ); + + let mut msg = pubsub.into_on_message(); + + while let Some(msg) = msg.next().await { + let payload: String = match msg.get_payload() { + Ok(payload) => payload, + Err(e) => { + tracing::error!(error = %e, topic = %topic_for_task, "Failed to get message payload"); + continue; + } + }; + + tracing::debug!(payload = %payload, "Received message from Redis"); + + let parsed: Value = match serde_json::from_str(&payload) { + Ok(data) => data, + Err(e) => { + tracing::error!(error = %e, topic = %topic_for_task, "Failed to parse message as JSON"); + continue; + } + }; + + // Extract trace context from envelope, with backward + // compatibility. Only treat as envelope when "__trace" is an + // object AND "data" is present, so payloads that merely + // contain a "__trace" key are not silently clobbered — same + // guard as the engine. + let (data, traceparent, baggage) = + if parsed.get("__trace").and_then(|v| v.as_object()).is_some() + && parsed.get("data").is_some() + { + let trace = parsed.get("__trace").unwrap(); + let data = parsed.get("data").cloned().unwrap(); + let traceparent = trace + .get("traceparent") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let baggage = trace + .get("baggage") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + (data, traceparent, baggage) + } else { + // Legacy format or non-envelope payload: use entire + // payload as data. + (parsed, None, None) + }; + + tracing::debug!(topic = %topic_for_task, function_id = %function_id_for_task, traceparent = ?traceparent, baggage = ?baggage, "Received message from Redis queue, invoking function"); + + if let Some(ref condition_id) = condition_function_id_for_task { + tracing::debug!( + condition_function_id = %condition_id, + "Checking trigger conditions" + ); + match check_condition(invoker.as_ref(), condition_id, data.clone()).await { + Ok(true) => {} + Ok(false) => { + tracing::debug!( + function_id = %function_id_for_task, + "Condition check failed, skipping handler" + ); + continue; + } + Err(err) => { + tracing::error!( + condition_function_id = %condition_id, + error = %err, + "Error invoking condition function" + ); + continue; + } + } + } + + let invoker = Arc::clone(&invoker); + let function_id = function_id_for_task.clone(); + let topic_for_call = topic_for_task.clone(); + + tokio::spawn(async move { + if let Err(e) = invoker.call(&function_id, data).await { + tracing::error!( + error = %e, + function_id = %function_id, + topic = %topic_for_call, + "Failed to invoke function for queue job" + ); + } + }); + } + + tracing::debug!(topic = %topic_for_task, id = %id_for_task, "Subscription task ended"); + }); + + tracing::debug!("Subscription task spawned"); + + // Store the subscription. + let mut subs = subscriptions.write().await; + subs.insert(topic, SubscriptionInfo { id, task_handle }); + } + + async fn unsubscribe(&self, topic: &str, id: &str) { + tracing::debug!(topic = %topic, id = %id, "Unsubscribing from Redis channel"); + let mut subs = self.subscriptions.write().await; + unsubscribe_locked(&mut subs, topic, id); + } + + async fn redrive_dlq(&self, _topic: &str) -> anyhow::Result { + Err(anyhow::anyhow!(DLQ_NOT_SUPPORTED)) + } + + async fn redrive_dlq_message(&self, _topic: &str, _message_id: &str) -> anyhow::Result { + Err(anyhow::anyhow!(DLQ_NOT_SUPPORTED)) + } + + async fn discard_dlq_message(&self, _topic: &str, _message_id: &str) -> anyhow::Result { + Err(anyhow::anyhow!(DLQ_NOT_SUPPORTED)) + } + + async fn dlq_count(&self, _topic: &str) -> anyhow::Result { + Err(anyhow::anyhow!(DLQ_NOT_SUPPORTED)) + } + + // `topic_stats` and `dlq_peek` are intentionally NOT overridden: the + // engine's redis adapter doesn't override them either, so both fall + // back to the trait's default bodies (`TopicStats::default()` / + // `Ok(vec![])`) — same as the engine falling back to its own trait + // defaults for this adapter. + + #[allow(clippy::too_many_arguments)] + async fn publish_to_function_queue( + &self, + queue_name: &str, + function_id: &str, + data: Value, + message_id: &str, + _max_retries: u32, + _backoff_ms: u64, + traceparent: Option, + baggage: Option, + // RabbitMQ-only feature; the redis pub/sub adapter ignores it, same + // as the engine. + _priority: Option, + ) { + let channel = format!("__queue::{}", queue_name); + let publisher = Arc::clone(&self.publisher); + + let envelope = serde_json::json!({ + "__trace": { + "traceparent": traceparent, + "baggage": baggage, + }, + "function_id": function_id, + "message_id": message_id, + "data": data, + }); + + let json = match serde_json::to_string(&envelope) { + Ok(json) => json, + Err(e) => { + tracing::error!(error = %e, queue = %queue_name, "Failed to serialize function queue data"); + return; + } + }; + + tracing::debug!(queue = %queue_name, function_id = %function_id, "Publishing to Redis function queue channel"); + + let mut conn = publisher.lock().await; + + if let Err(e) = conn.publish::<_, _, ()>(&channel, &json).await { + tracing::error!(error = %e, queue = %queue_name, "Failed to publish to Redis function queue channel"); + } + } + + async fn consume_function_queue( + &self, + _queue_name: &str, + _prefetch: u32, + ) -> anyhow::Result> { + anyhow::bail!("Redis function queue consumer not yet implemented") + } + + async fn list_topics(&self) -> anyhow::Result> { + // Redis adapter keys subscriptions by topic name directly, and only + // ever allows one subscription per topic (the guard in + // `subscribe`), so each present topic contributes exactly one to + // its own count. This worker's `TopicInfo` has no + // `broker_type`/`subscriber_count` fields (unlike the engine's), so + // the subscriber count is carried in `depth` — the closest + // available field — for lack of a better one. + let subs = self.subscriptions.read().await; + Ok(subs + .keys() + .map(|topic| TopicInfo { + name: topic.clone(), + depth: 1, + }) + .collect()) + } + + async fn shutdown(&self) { + let mut subs = self.subscriptions.write().await; + for (_, sub) in subs.drain() { + sub.task_handle.abort(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn resolve_redis_url_defaults_when_absent() { + assert_eq!(RedisAdapter::resolve_redis_url(None), DEFAULT_REDIS_URL); + } + + #[test] + fn resolve_redis_url_defaults_when_config_has_no_redis_url_key() { + let config = json!({"other": "value"}); + assert_eq!( + RedisAdapter::resolve_redis_url(Some(&config)), + DEFAULT_REDIS_URL + ); + } + + #[test] + fn resolve_redis_url_uses_explicit_url() { + let config = json!({"redis_url": "redis://example.com:1234"}); + assert_eq!( + RedisAdapter::resolve_redis_url(Some(&config)), + "redis://example.com:1234" + ); + } + + #[test] + fn dlq_not_supported_message_is_exact_engine_string() { + // Single source of truth for all four DLQ-family trait methods — + // see the module doc's testability note. + assert_eq!( + DLQ_NOT_SUPPORTED, + "RedisAdapter does not support DLQ operations (pub/sub only)" + ); + } + + #[tokio::test] + async fn unsubscribe_unknown_topic_is_a_noop() { + let mut subs: HashMap = HashMap::new(); + unsubscribe_locked(&mut subs, "missing-topic", "some-id"); + assert!(subs.is_empty()); + } + + #[tokio::test] + async fn unsubscribe_mismatched_id_keeps_subscription() { + let mut subs: HashMap = HashMap::new(); + subs.insert( + "demo".to_string(), + SubscriptionInfo { + id: "owner".to_string(), + task_handle: tokio::spawn(async { + std::future::pending::<()>().await; + }), + }, + ); + unsubscribe_locked(&mut subs, "demo", "someone-else"); + assert!( + subs.contains_key("demo"), + "mismatched id must not remove the subscription" + ); + subs.remove("demo").unwrap().task_handle.abort(); + } + + #[tokio::test] + async fn unsubscribe_matching_id_removes_and_aborts() { + let mut subs: HashMap = HashMap::new(); + subs.insert( + "demo".to_string(), + SubscriptionInfo { + id: "owner".to_string(), + task_handle: tokio::spawn(async { + std::future::pending::<()>().await; + }), + }, + ); + unsubscribe_locked(&mut subs, "demo", "owner"); + assert!(!subs.contains_key("demo")); + } +} diff --git a/queue/tests/common/docker.rs b/queue/tests/common/docker.rs new file mode 100644 index 000000000..56f54a95f --- /dev/null +++ b/queue/tests/common/docker.rs @@ -0,0 +1,114 @@ +#![allow(dead_code)] +//! Docker-backed Redis container for the redis adapter e2e test. +//! +//! Skips (returns `None`) whenever `docker info` fails — no Docker daemon +//! reachable — so CI (which runs without Docker) and casual local runs stay +//! green. The container is started with `-p 0:6379` (an ephemeral host +//! port, so parallel test runs never collide on a fixed port) and torn down +//! via `Drop` (`docker stop`, which also removes it since it's started +//! with `--rm`). + +use std::process::Command; + +pub struct RedisContainer { + container_id: String, + pub host_port: u16, +} + +impl RedisContainer { + pub fn redis_url(&self) -> String { + format!("redis://127.0.0.1:{}", self.host_port) + } +} + +impl Drop for RedisContainer { + fn drop(&mut self) { + let _ = Command::new("docker") + .args(["stop", "-t", "1", &self.container_id]) + .output(); + } +} + +fn docker_available() -> bool { + Command::new("docker") + .arg("info") + .output() + .map(|out| out.status.success()) + .unwrap_or(false) +} + +/// Starts a `redis:7-alpine` container on an ephemeral host port. Returns +/// `None` (the caller should skip the test) when Docker isn't reachable or +/// the container fails to start/become ready. +pub fn start_redis() -> Option { + if !docker_available() { + eprintln!("[skip] docker not reachable — skipping redis adapter e2e test"); + return None; + } + + let run_output = Command::new("docker") + .args(["run", "-d", "--rm", "-p", "0:6379", "redis:7-alpine"]) + .output() + .ok()?; + + if !run_output.status.success() { + eprintln!( + "[skip] `docker run` failed: {}", + String::from_utf8_lossy(&run_output.stderr) + ); + return None; + } + + let container_id = String::from_utf8_lossy(&run_output.stdout) + .trim() + .to_string(); + if container_id.is_empty() { + return None; + } + + let host_port = resolve_mapped_port(&container_id)?; + + let container = RedisContainer { + container_id, + host_port, + }; + + if !wait_until_ready(&container) { + eprintln!("[skip] redis container did not become ready in time"); + return None; + } + + Some(container) +} + +fn resolve_mapped_port(container_id: &str) -> Option { + // `docker port 6379/tcp` prints e.g. `0.0.0.0:54321` (and often a + // second `[::]:54321` line for the IPv6 binding) — take the first + // line's port. + let port_output = Command::new("docker") + .args(["port", container_id, "6379/tcp"]) + .output() + .ok()?; + if !port_output.status.success() { + return None; + } + let stdout = String::from_utf8_lossy(&port_output.stdout); + let first_line = stdout.lines().next()?; + let port_str = first_line.rsplit(':').next()?; + port_str.trim().parse::().ok() +} + +fn wait_until_ready(container: &RedisContainer) -> bool { + use std::net::TcpStream; + use std::time::{Duration, Instant}; + + let addr = format!("127.0.0.1:{}", container.host_port); + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + if TcpStream::connect(&addr).is_ok() { + return true; + } + std::thread::sleep(Duration::from_millis(100)); + } + false +} diff --git a/queue/tests/common/mod.rs b/queue/tests/common/mod.rs index 702e611f1..246c52ab9 100644 --- a/queue/tests/common/mod.rs +++ b/queue/tests/common/mod.rs @@ -1 +1,2 @@ +pub mod docker; pub mod engine; diff --git a/queue/tests/e2e_redis.rs b/queue/tests/e2e_redis.rs new file mode 100644 index 000000000..bcfc4b265 --- /dev/null +++ b/queue/tests/e2e_redis.rs @@ -0,0 +1,150 @@ +mod common; + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use iii_queue::adapter::QueueAdapter; +use iii_queue::adapters::redis::RedisAdapter; +use iii_queue::trigger::IiiInvoker; +use iii_sdk::errors::Error; +use iii_sdk::RegisterFunction; +use serde_json::{json, Value}; +use serial_test::serial; +use tokio::time::Instant; +use uuid::Uuid; + +use common::docker; +use common::engine; + +async fn wait_for_fires(fires: &AtomicUsize, expected: usize) { + let deadline = Instant::now() + Duration::from_secs(3); + while Instant::now() < deadline { + if fires.load(Ordering::SeqCst) >= expected { + return; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + assert!( + fires.load(Ordering::SeqCst) >= expected, + "expected at least {expected} fires" + ); +} + +/// Boots a `RedisAdapter` directly against a live engine connection and a +/// docker-backed Redis, bypassing `boot::start`/`QueueConfig` — the redis +/// adapter isn't wired into the config factory yet (a later task), so this +/// exercises the adapter the way that factory will: `from_config` + a +/// direct `subscribe`/`enqueue` against the trait, instead of going through +/// the `durable:subscriber` trigger type. +#[tokio::test] +#[serial] +async fn publish_delivers_unwrapped_payload_to_subscriber_connect_or_skip() { + let Some(container) = docker::start_redis() else { + return; // skip: docker not reachable + }; + let Some(iii) = engine::connect_fresh().await else { + return; // skip: engine not reachable + }; + + let invoker = Arc::new(IiiInvoker::new(iii.clone())); + let adapter = + RedisAdapter::from_config(Some(&json!({"redis_url": container.redis_url()})), invoker) + .await + .expect("redis adapter should connect"); + + let fires = Arc::new(AtomicUsize::new(0)); + let seen = Arc::new(tokio::sync::Mutex::new(Vec::::new())); + let function_id = format!("queue.e2e.redis.{}", Uuid::new_v4()); + { + let fires = fires.clone(); + let seen = seen.clone(); + iii.register_function( + function_id.as_str(), + RegisterFunction::new_async(move |payload: Value| { + let fires = fires.clone(); + let seen = seen.clone(); + async move { + fires.fetch_add(1, Ordering::SeqCst); + seen.lock().await.push(payload); + Ok::(json!({"ok": true})) + } + }), + ); + } + + let topic = format!("e2e-redis-{}", Uuid::new_v4()); + adapter + .subscribe(&topic, "sub-1", &function_id, None, None) + .await; + // Give the subscription task a beat to actually SUBSCRIBE on the Redis + // connection before the first publish — pub/sub has no buffering for a + // not-yet-subscribed consumer. + tokio::time::sleep(Duration::from_millis(300)).await; + + adapter + .enqueue(&topic, json!({"hello": "world"}), None, None) + .await; + wait_for_fires(&fires, 1).await; + + let seen = seen.lock().await; + // Parity with the builtin (and the pubsub worker's own e2e test): the + // SDK stamps cross-worker invocation payloads with caller metadata + // (e.g. `_caller_worker_id`), so assert the published field is present + // rather than exact-equal on the whole object. The key assertion is + // that this is the RAW data, not the `__trace` envelope + // (`{"__trace": ..., "data": ...}`) — no `__trace`/`data` keys leak + // through. + assert_eq!(seen[0].get("hello"), Some(&json!("world"))); + assert!(seen[0].get("__trace").is_none()); + assert!(seen[0].get("data").is_none()); + drop(seen); + + adapter.unsubscribe(&topic, "sub-1").await; + adapter.shutdown().await; + iii.shutdown_async().await; +} + +/// The redis adapter is pub/sub only: every DLQ-family method must fail +/// with the exact engine error string, never silently succeed. +#[tokio::test] +#[serial] +async fn dlq_operations_return_not_supported_connect_or_skip() { + let Some(container) = docker::start_redis() else { + return; // skip: docker not reachable + }; + let Some(iii) = engine::connect_fresh().await else { + return; // skip: engine not reachable + }; + + let invoker = Arc::new(IiiInvoker::new(iii.clone())); + let adapter = + RedisAdapter::from_config(Some(&json!({"redis_url": container.redis_url()})), invoker) + .await + .expect("redis adapter should connect"); + + const NOT_SUPPORTED: &str = "RedisAdapter does not support DLQ operations (pub/sub only)"; + + let topic = format!("e2e-redis-dlq-{}", Uuid::new_v4()); + + let err = adapter.redrive_dlq(&topic).await.unwrap_err(); + assert_eq!(err.to_string(), NOT_SUPPORTED); + + let err = adapter + .redrive_dlq_message(&topic, "some-id") + .await + .unwrap_err(); + assert_eq!(err.to_string(), NOT_SUPPORTED); + + let err = adapter + .discard_dlq_message(&topic, "some-id") + .await + .unwrap_err(); + assert_eq!(err.to_string(), NOT_SUPPORTED); + + let err = adapter.dlq_count(&topic).await.unwrap_err(); + assert_eq!(err.to_string(), NOT_SUPPORTED); + + adapter.shutdown().await; + iii.shutdown_async().await; +} From 9aaa1a1478832f3ccb73bdbc266fa997d8962a07 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Tue, 7 Jul 2026 05:59:36 -0300 Subject: [PATCH 22/28] feat(queue): rabbitmq types and naming (ported) Port engine/src/workers/queue/adapters/rabbitmq/{types.rs,naming.rs} verbatim (pure logic, no engine dependency) as the first step of the RabbitMQ adapter port. Adds the `rabbitmq` cargo feature (default on, gates `dep:lapin`) and wires queue/src/adapters/mod.rs to include the new module tree only under that feature. --- queue/Cargo.lock | 901 +++++++++++++++++++++++++- queue/Cargo.toml | 5 + queue/src/adapters/mod.rs | 2 + queue/src/adapters/rabbitmq/mod.rs | 10 + queue/src/adapters/rabbitmq/naming.rs | 144 ++++ queue/src/adapters/rabbitmq/types.rs | 231 +++++++ 6 files changed, 1287 insertions(+), 6 deletions(-) create mode 100644 queue/src/adapters/rabbitmq/mod.rs create mode 100644 queue/src/adapters/rabbitmq/naming.rs create mode 100644 queue/src/adapters/rabbitmq/types.rs diff --git a/queue/Cargo.lock b/queue/Cargo.lock index db5e5d4c6..b8556a6e5 100644 --- a/queue/Cargo.lock +++ b/queue/Cargo.lock @@ -2,6 +2,17 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -11,6 +22,58 @@ dependencies = [ "memchr", ] +[[package]] +name = "amq-protocol" +version = "8.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "355603365d2217f7fbc03f0be085ea1440498957890f04276402012cdde445f5" +dependencies = [ + "amq-protocol-tcp", + "amq-protocol-types", + "amq-protocol-uri", + "cookie-factory", + "nom 8.0.0", + "serde", +] + +[[package]] +name = "amq-protocol-tcp" +version = "8.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d7b97a85e08671697e724a6b7f1459ff81603613695e3151764a9529c6fec15" +dependencies = [ + "amq-protocol-uri", + "async-trait", + "cfg-if", + "executor-trait 2.1.2", + "reactor-trait 2.8.0", + "tcp-stream", + "tracing", +] + +[[package]] +name = "amq-protocol-types" +version = "8.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2984a816dba991b5922503921d8f94650792bdeac47c27c83830710d2567f63" +dependencies = [ + "cookie-factory", + "nom 8.0.0", + "serde", + "serde_json", +] + +[[package]] +name = "amq-protocol-uri" +version = "8.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31db8e69d1456ec8ecf6ee598707179cf1d95f34f7d30037b16ad43f0cddcff" +dependencies = [ + "amq-protocol-types", + "percent-encoding", + "url", +] + [[package]] name = "anstream" version = "1.0.0" @@ -82,6 +145,127 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13f937e26114b93193065fd44f507aa2e9169ad0cdabbb996920b1fe1ddea7ba" +dependencies = [ + "async-channel", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-global-executor-trait" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9af57045d58eeb1f7060e7025a1631cbc6399e0a1d10ad6735b3d0ea7f8346ce" +dependencies = [ + "async-global-executor", + "async-trait", + "executor-trait 2.1.2", +] + +[[package]] +name = "async-global-executor-trait" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3727b7da74b92d2d03403cf1142706b53423e5c050791af438f8f50edea057a" +dependencies = [ + "async-global-executor", + "async-global-executor-trait 2.2.0", + "async-trait", + "executor-trait 2.1.2", + "executor-trait 3.1.0", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + [[package]] name = "async-lock" version = "3.4.2" @@ -93,6 +277,24 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-reactor-trait" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ab52004af1f14a170088bd9e10a2d3b2f2307ce04320e58a6ce36ee531be625" +dependencies = [ + "async-io", + "async-trait", + "futures-core", + "reactor-trait 3.1.1", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.89" @@ -116,6 +318,29 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-lc-rs" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "backon" version = "1.6.0" @@ -131,6 +356,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bitflags" version = "2.13.0" @@ -146,6 +377,28 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -158,6 +411,15 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.2.65" @@ -165,6 +427,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -180,6 +444,27 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.6.1" @@ -220,6 +505,27 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cms" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" +dependencies = [ + "const-oid", + "der", + "spki", + "x509-cert", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -249,6 +555,18 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" + [[package]] name = "core-foundation" version = "0.10.1" @@ -274,6 +592,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-utils" version = "0.8.22" @@ -296,6 +623,59 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "der_derive", + "flagset", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "des" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" +dependencies = [ + "cipher", +] + [[package]] name = "digest" version = "0.10.7" @@ -304,6 +684,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -317,6 +698,12 @@ dependencies = [ "syn", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -360,6 +747,24 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "executor-trait" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c39dff9342e4e0e16ce96be751eb21a94e94a87bb2f6e63ad1961c2ce109bf" +dependencies = [ + "async-trait", +] + +[[package]] +name = "executor-trait" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d6a1fc6700fa12782770cb344a29172ae940ea41d5fd5049fdf236dd6eaa92" +dependencies = [ + "async-trait", +] + [[package]] name = "fastrand" version = "2.4.1" @@ -372,6 +777,23 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -381,6 +803,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.32" @@ -429,6 +857,19 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.32" @@ -440,6 +881,17 @@ dependencies = [ "syn", ] +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls", + "rustls-pki-types", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -515,6 +967,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -529,6 +982,27 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "hostname" version = "0.4.2" @@ -771,6 +1245,7 @@ dependencies = [ "clap", "futures", "iii-sdk", + "lapin", "redis", "schemars", "serde", @@ -814,6 +1289,16 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -832,6 +1317,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.103" @@ -843,6 +1338,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lapin" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913a84142a99160ecef997a5c17c53639bcbac4424a0315a5ffe6c8be8e8db86" +dependencies = [ + "amq-protocol", + "async-global-executor-trait 3.1.0", + "async-reactor-trait", + "async-trait", + "backon", + "executor-trait 2.1.2", + "flume", + "futures-core", + "futures-io", + "reactor-trait 2.8.0", + "tracing", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -855,6 +1369,12 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.2" @@ -897,6 +1417,12 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "mio" version = "1.2.1" @@ -908,6 +1434,25 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "ntapi" version = "0.4.3" @@ -936,6 +1481,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.46" @@ -973,6 +1524,15 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1029,12 +1589,35 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand", + "rand 0.9.4", "thiserror", "tokio", "tokio-stream", ] +[[package]] +name = "p12-keystore" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb9bf5222606eb712d3bb30e01bc9420545b00859970897e70c682353a034f2" +dependencies = [ + "base64", + "cbc", + "cms", + "der", + "des", + "hex", + "hmac", + "pkcs12", + "pkcs5", + "rand 0.10.2", + "rc2", + "sha1", + "sha2", + "thiserror", + "x509-parser", +] + [[package]] name = "parking" version = "2.2.1" @@ -1064,6 +1647,25 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1076,6 +1678,67 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "pkcs12" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2" +dependencies = [ + "cms", + "const-oid", + "der", + "digest", + "spki", + "x509-cert", + "zeroize", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -1085,6 +1748,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1132,7 +1801,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -1186,7 +1855,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -1196,7 +1876,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -1208,6 +1888,45 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rc2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62c64daa8e9438b84aaae55010a93f396f8e60e3911590fcba770d04643fc1dd" +dependencies = [ + "cipher", +] + +[[package]] +name = "reactor-trait" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffbbf16bc3e4db5fdcf4b77cebf1313610b54b339712aa90088d2d9b1acb1f1" +dependencies = [ + "async-trait", + "reactor-trait 3.1.1", +] + +[[package]] +name = "reactor-trait" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b1c85237926dd82e8bc3634240ecf2236ea81e904b3d83cdb1df974af9af293" +dependencies = [ + "async-io", + "async-trait", + "executor-trait 2.1.2", + "flume", + "futures-core", + "futures-io", +] + [[package]] name = "redis" version = "1.3.0" @@ -1320,12 +2039,35 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -1334,6 +2076,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-connector" +version = "0.21.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10eb7ce243317e6b6a342ef6bff8c2e0d46d78120a9aeb2ee39693a569615c96" +dependencies = [ + "futures-io", + "futures-rustls", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "rustls-webpki", +] + [[package]] name = "rustls-native-certs" version = "0.8.4" @@ -1346,6 +2103,15 @@ dependencies = [ "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.15.0" @@ -1362,6 +2128,7 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -1379,6 +2146,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "schannel" version = "0.1.29" @@ -1418,6 +2194,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -1552,7 +2339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1562,6 +2349,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1609,6 +2407,25 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1672,6 +2489,20 @@ dependencies = [ "windows", ] +[[package]] +name = "tcp-stream" +version = "0.30.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282ebecea8280bce8b7a0695b5dc93a19839dd445cbba70d3e07c9f6e12c4653" +dependencies = [ + "cfg-if", + "futures-io", + "p12-keystore", + "reactor-trait 2.8.0", + "rustls-connector", + "rustls-pemfile", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -1701,6 +2532,36 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -1926,7 +2787,7 @@ dependencies = [ "http", "httparse", "log", - "rand", + "rand 0.9.4", "rustls", "rustls-pki-types", "sha1", @@ -2420,6 +3281,34 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid", + "der", + "spki", +] + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.3", + "oid-registry", + "rusticata-macros", + "thiserror", + "time", +] + [[package]] name = "xxhash-rust" version = "0.8.16" diff --git a/queue/Cargo.toml b/queue/Cargo.toml index 62a8c315a..1cb61dee7 100644 --- a/queue/Cargo.toml +++ b/queue/Cargo.toml @@ -17,6 +17,10 @@ path = "src/lib.rs" name = "queue" path = "src/main.rs" +[features] +default = ["rabbitmq"] +rabbitmq = ["dep:lapin"] + [dependencies] iii-sdk = "=0.20.0" async-trait = "0.1" @@ -32,6 +36,7 @@ schemars = "0.8" uuid = { version = "1", features = ["v4"] } futures = "0.3" redis = { version = "1.0.1", features = ["tokio-comp", "connection-manager"] } +lapin = { version = "3", optional = true } [dev-dependencies] serial_test = "3" diff --git a/queue/src/adapters/mod.rs b/queue/src/adapters/mod.rs index 743047ae1..add64c86e 100644 --- a/queue/src/adapters/mod.rs +++ b/queue/src/adapters/mod.rs @@ -1,4 +1,6 @@ //! Transport adapters implementing [`crate::adapter::QueueAdapter`]. pub mod builtin; +#[cfg(feature = "rabbitmq")] +pub mod rabbitmq; pub mod redis; diff --git a/queue/src/adapters/rabbitmq/mod.rs b/queue/src/adapters/rabbitmq/mod.rs new file mode 100644 index 000000000..f85a9840e --- /dev/null +++ b/queue/src/adapters/rabbitmq/mod.rs @@ -0,0 +1,10 @@ +//! RabbitMQ transport adapter. +//! +//! Ported from the engine builtin `iii-queue` worker's RabbitMQ adapter +//! (`engine/src/workers/queue/adapters/rabbitmq/`, all 8 modules). Gated +//! behind the `rabbitmq` cargo feature (on by default) — the whole module +//! tree, and every individual file (`#![cfg(feature = "rabbitmq")]`), compile +//! out entirely with `--no-default-features`. + +pub mod naming; +pub mod types; diff --git a/queue/src/adapters/rabbitmq/naming.rs b/queue/src/adapters/rabbitmq/naming.rs new file mode 100644 index 000000000..b01286b2c --- /dev/null +++ b/queue/src/adapters/rabbitmq/naming.rs @@ -0,0 +1,144 @@ +//! RabbitMQ exchange/queue naming scheme. +//! +//! 1:1 port of `engine/src/workers/queue/adapters/rabbitmq/naming.rs` — pure +//! string formatting, no engine dependency, ports verbatim. + +#![cfg(feature = "rabbitmq")] + +pub const EXCHANGE_PREFIX: &str = "iii"; + +pub struct RabbitNames { + pub topic: String, +} + +impl RabbitNames { + pub fn new(topic: impl Into) -> Self { + Self { + topic: topic.into(), + } + } + + pub fn exchange(&self) -> String { + format!("{}.{}.exchange", EXCHANGE_PREFIX, self.topic) + } + + pub fn queue(&self) -> String { + format!("{}.{}.queue", EXCHANGE_PREFIX, self.topic) + } + + pub fn function_queue(&self, function_id: &str) -> String { + format!("{}.{}.{}.queue", EXCHANGE_PREFIX, self.topic, function_id) + } + + pub fn function_dlq(&self, function_id: &str) -> String { + format!("{}.{}.{}.dlq", EXCHANGE_PREFIX, self.topic, function_id) + } + + pub fn dlq(&self) -> String { + format!("{}.{}.dlq", EXCHANGE_PREFIX, self.topic) + } +} + +pub struct FnQueueNames { + pub name: String, +} + +impl FnQueueNames { + pub fn new(name: impl Into) -> Self { + Self { name: name.into() } + } + + pub fn exchange(&self) -> String { + format!("{}.__fn_queue::{}", EXCHANGE_PREFIX, self.name) + } + + pub fn queue(&self) -> String { + format!("{}.__fn_queue::{}.queue", EXCHANGE_PREFIX, self.name) + } + + pub fn retry_exchange(&self) -> String { + format!("{}.__fn_queue::{}::retry", EXCHANGE_PREFIX, self.name) + } + + pub fn retry_queue(&self) -> String { + format!("{}.__fn_queue::{}::retry.queue", EXCHANGE_PREFIX, self.name) + } + + pub fn dlq_exchange(&self) -> String { + format!("{}.__fn_queue::{}::dlq", EXCHANGE_PREFIX, self.name) + } + + pub fn dlq(&self) -> String { + format!("{}.__fn_queue::{}::dlq.queue", EXCHANGE_PREFIX, self.name) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rabbit_names() { + let names = RabbitNames::new("user.created"); + + assert_eq!(names.exchange(), "iii.user.created.exchange"); + assert_eq!(names.queue(), "iii.user.created.queue"); + assert_eq!(names.dlq(), "iii.user.created.dlq"); + } + + #[test] + fn test_fn_queue_names_exchange() { + let names = FnQueueNames::new("orders"); + assert_eq!(names.exchange(), "iii.__fn_queue::orders"); + } + + #[test] + fn test_fn_queue_names_queue() { + let names = FnQueueNames::new("orders"); + assert_eq!(names.queue(), "iii.__fn_queue::orders.queue"); + } + + #[test] + fn test_fn_queue_names_retry_exchange() { + let names = FnQueueNames::new("orders"); + assert_eq!(names.retry_exchange(), "iii.__fn_queue::orders::retry"); + } + + #[test] + fn test_fn_queue_names_retry_queue() { + let names = FnQueueNames::new("orders"); + assert_eq!(names.retry_queue(), "iii.__fn_queue::orders::retry.queue"); + } + + #[test] + fn test_fn_queue_names_dlq_exchange() { + let names = FnQueueNames::new("orders"); + assert_eq!(names.dlq_exchange(), "iii.__fn_queue::orders::dlq"); + } + + #[test] + fn test_fn_queue_names_dlq() { + let names = FnQueueNames::new("orders"); + assert_eq!(names.dlq(), "iii.__fn_queue::orders::dlq.queue"); + } + + #[test] + fn test_fn_queue_names_with_dots() { + let names = FnQueueNames::new("payment.processing"); + assert_eq!(names.exchange(), "iii.__fn_queue::payment.processing"); + assert_eq!(names.queue(), "iii.__fn_queue::payment.processing.queue"); + assert_eq!( + names.retry_exchange(), + "iii.__fn_queue::payment.processing::retry" + ); + assert_eq!( + names.retry_queue(), + "iii.__fn_queue::payment.processing::retry.queue" + ); + assert_eq!( + names.dlq_exchange(), + "iii.__fn_queue::payment.processing::dlq" + ); + assert_eq!(names.dlq(), "iii.__fn_queue::payment.processing::dlq.queue"); + } +} diff --git a/queue/src/adapters/rabbitmq/types.rs b/queue/src/adapters/rabbitmq/types.rs new file mode 100644 index 000000000..0d6283238 --- /dev/null +++ b/queue/src/adapters/rabbitmq/types.rs @@ -0,0 +1,231 @@ +//! Core RabbitMQ types: `Job` envelope, priority resolution, queue mode, and +//! adapter config parsing. +//! +//! 1:1 port of the engine's `engine/src/workers/queue/adapters/rabbitmq/types.rs` +//! — no engine-specific dependencies here (no `Arc`, no telemetry), so +//! this file ports verbatim. + +#![cfg(feature = "rabbitmq")] + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::str::FromStr; +use std::time::{SystemTime, UNIX_EPOCH}; +use uuid::Uuid; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Job { + pub id: String, + pub topic: String, + pub data: Value, + pub attempts_made: u32, + pub max_attempts: u32, + pub created_at: u64, + #[serde(default)] + pub traceparent: Option, + #[serde(default)] + pub baggage: Option, + /// AMQP message priority (0..=queue `x-max-priority`). Carried on the job so + /// it survives requeue and DLQ-redrive republishes; stamped onto + /// `BasicProperties` at publish time. `None` means default priority. + #[serde(default)] + pub priority: Option, +} + +impl Job { + pub fn new( + topic: impl Into, + data: Value, + max_attempts: u32, + traceparent: Option, + baggage: Option, + ) -> Self { + Self { + id: Uuid::new_v4().to_string(), + topic: topic.into(), + data, + attempts_made: 0, + max_attempts, + created_at: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as u64, + traceparent, + baggage, + priority: None, + } + } + + /// Attach a priority to the job (builder style). `None` leaves it at the + /// default priority. + pub fn with_priority(mut self, priority: Option) -> Self { + self.priority = priority; + self + } + + pub fn increment_attempts(&mut self) { + self.attempts_made += 1; + } + + pub fn is_exhausted(&self) -> bool { + self.attempts_made >= self.max_attempts + } +} + +/// Resolve an AMQP message priority from `data` given the configured field +/// name. Returns `None` when no field is configured, the field is absent, or +/// its value is not a non-negative integer. Values above 255 saturate to 255; +/// the broker further clamps to each queue's `x-max-priority`. +pub fn priority_from_data(data: &Value, priority_field: Option<&str>) -> Option { + let field = priority_field?; + let value = data.get(field)?; + let n = value.as_u64()?; + Some(n.min(u8::MAX as u64) as u8) +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub enum QueueMode { + #[default] + Standard, + Fifo, +} + +impl FromStr for QueueMode { + type Err = (); + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "fifo" => Ok(QueueMode::Fifo), + "standard" => Ok(QueueMode::Standard), + _ => Ok(QueueMode::default()), + } + } +} + +#[derive(Debug, Clone)] +pub struct RabbitMQConfig { + pub amqp_url: String, + pub max_attempts: u32, + pub prefetch_count: u16, + pub queue_mode: QueueMode, + /// Field in the message data whose integer value sets the priority of + /// messages published to topics (pub/sub fanout). `None` disables priority + /// stamping on the topic path. + pub priority_field: Option, +} + +impl Default for RabbitMQConfig { + fn default() -> Self { + Self { + amqp_url: "amqp://localhost:5672".to_string(), + max_attempts: 3, + prefetch_count: 10, + queue_mode: QueueMode::default(), + priority_field: None, + } + } +} + +impl RabbitMQConfig { + pub fn from_value(config: Option<&Value>) -> Self { + let mut cfg = Self::default(); + + if let Some(config) = config { + if let Some(url) = config.get("amqp_url").and_then(|v| v.as_str()) { + cfg.amqp_url = url.to_string(); + } + if let Some(attempts) = config.get("max_attempts").and_then(|v| v.as_u64()) { + cfg.max_attempts = attempts as u32; + } + if let Some(prefetch) = config.get("prefetch_count").and_then(|v| v.as_u64()) { + cfg.prefetch_count = prefetch as u16; + } + if let Some(mode) = config.get("queue_mode").and_then(|v| v.as_str()) { + cfg.queue_mode = QueueMode::from_str(mode).unwrap_or_default(); + } + if let Some(field) = config.get("priority_field").and_then(|v| v.as_str()) { + cfg.priority_field = Some(field.to_string()); + } + } + + cfg + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_job_creation() { + let job = Job::new( + "test.topic", + serde_json::json!({"key": "value"}), + 3, + None, + None, + ); + assert_eq!(job.topic, "test.topic"); + assert_eq!(job.attempts_made, 0); + assert_eq!(job.max_attempts, 3); + assert!(!job.is_exhausted()); + assert_eq!(job.priority, None); + } + + #[test] + fn test_job_with_priority() { + let job = Job::new("t", serde_json::json!({}), 1, None, None).with_priority(Some(7)); + assert_eq!(job.priority, Some(7)); + } + + #[test] + fn test_priority_from_data() { + let data = serde_json::json!({ "priority": 9, "other": "x" }); + // No field configured -> None. + assert_eq!(priority_from_data(&data, None), None); + // Field present and integer. + assert_eq!(priority_from_data(&data, Some("priority")), Some(9)); + // Field absent. + assert_eq!(priority_from_data(&data, Some("missing")), None); + // Non-integer value -> None. + let str_data = serde_json::json!({ "priority": "high" }); + assert_eq!(priority_from_data(&str_data, Some("priority")), None); + // Values above u8::MAX saturate. + let big = serde_json::json!({ "priority": 300 }); + assert_eq!(priority_from_data(&big, Some("priority")), Some(255)); + } + + #[test] + fn test_config_from_value_defaults() { + let cfg = RabbitMQConfig::from_value(None); + assert_eq!(cfg.amqp_url, "amqp://localhost:5672"); + assert_eq!(cfg.max_attempts, 3); + assert_eq!(cfg.prefetch_count, 10); + assert!(matches!(cfg.queue_mode, QueueMode::Standard)); + assert_eq!(cfg.priority_field, None); + } + + #[test] + fn test_config_from_value_overrides() { + let value = serde_json::json!({ + "amqp_url": "amqp://example.com:5672", + "max_attempts": 7, + "prefetch_count": 20, + "queue_mode": "fifo", + "priority_field": "priority", + }); + let cfg = RabbitMQConfig::from_value(Some(&value)); + assert_eq!(cfg.amqp_url, "amqp://example.com:5672"); + assert_eq!(cfg.max_attempts, 7); + assert_eq!(cfg.prefetch_count, 20); + assert!(matches!(cfg.queue_mode, QueueMode::Fifo)); + assert_eq!(cfg.priority_field, Some("priority".to_string())); + } + + #[test] + fn test_config_from_value_unknown_queue_mode_defaults_to_standard() { + let value = serde_json::json!({ "queue_mode": "not-a-mode" }); + let cfg = RabbitMQConfig::from_value(Some(&value)); + assert!(matches!(cfg.queue_mode, QueueMode::Standard)); + } +} From 22347e93e432c7e60dd9c54aefd0d91d658fa2c4 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Tue, 7 Jul 2026 06:05:25 -0300 Subject: [PATCH 23/28] wip(queue): rabbitmq topology, retry, publisher, consumer, worker, adapter Port the remaining engine/src/workers/queue/adapters/rabbitmq modules (topology.rs, retry.rs, publisher.rs, consumer.rs, worker.rs, adapter.rs) implementing the full QueueAdapter trait. Compiles and passes clippy under both --features rabbitmq (default) and --no-default-features. Adds a `max_priority: Option` field to the worker's minimal FunctionQueueConfig (queue/src/adapter.rs) so setup_function_queue can pass priority-queue declaration through, matching the engine's config shape. --- queue/src/adapter.rs | 8 + queue/src/adapters/rabbitmq/adapter.rs | 1200 ++++++++++++++++++++++ queue/src/adapters/rabbitmq/consumer.rs | 70 ++ queue/src/adapters/rabbitmq/mod.rs | 10 +- queue/src/adapters/rabbitmq/publisher.rs | 176 ++++ queue/src/adapters/rabbitmq/retry.rs | 88 ++ queue/src/adapters/rabbitmq/topology.rs | 276 +++++ queue/src/adapters/rabbitmq/worker.rs | 286 ++++++ 8 files changed, 2113 insertions(+), 1 deletion(-) create mode 100644 queue/src/adapters/rabbitmq/adapter.rs create mode 100644 queue/src/adapters/rabbitmq/consumer.rs create mode 100644 queue/src/adapters/rabbitmq/publisher.rs create mode 100644 queue/src/adapters/rabbitmq/retry.rs create mode 100644 queue/src/adapters/rabbitmq/topology.rs create mode 100644 queue/src/adapters/rabbitmq/worker.rs diff --git a/queue/src/adapter.rs b/queue/src/adapter.rs index 6c6564c62..92a2ac4e3 100644 --- a/queue/src/adapter.rs +++ b/queue/src/adapter.rs @@ -67,6 +67,14 @@ pub struct FunctionQueueConfig { pub concurrency: u32, /// Base delay in milliseconds for the exponential retry backoff. pub backoff_ms: u64, + /// Declares the queue as a RabbitMQ priority queue with this many + /// priority levels (`x-max-priority`). `None` means not a priority + /// queue. RabbitMQ-only; other adapters ignore it. Added (rather than + /// part of the original minimal port) so + /// [`QueueAdapter::setup_function_queue`] can pass it through to the + /// RabbitMQ adapter's topology setup. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_priority: Option, } /// Transport abstraction implemented by every queue backend adapter diff --git a/queue/src/adapters/rabbitmq/adapter.rs b/queue/src/adapters/rabbitmq/adapter.rs new file mode 100644 index 000000000..0f533d99a --- /dev/null +++ b/queue/src/adapters/rabbitmq/adapter.rs @@ -0,0 +1,1200 @@ +//! RabbitMQ transport adapter: implements +//! [`crate::adapter::QueueAdapter`] on top of the fanout-exchange + +//! per-function-queue topology (`topology.rs`), the job envelope +//! (`types.rs`), and the retry/DLQ decision table (`retry.rs`). +//! +//! Port of `engine/src/workers/queue/adapters/rabbitmq/adapter.rs`. Seams +//! applied for the standalone worker (mirrors every other adapter in this +//! crate, e.g. `adapters/redis.rs`): +//! - `Arc` -> `Arc`. +//! - The engine's `make_adapter`/`register_adapter!` factory-registration +//! macro (which plugs into the engine's own adapter registry) has no +//! equivalent here; instead this module exposes +//! [`RabbitMQAdapter::from_config`], matching the +//! `RedisAdapter::from_config(config, invoker)` convention already +//! established by `adapters/redis.rs`. +//! - No `Clone` impl on the adapter itself (the engine's exists so the +//! factory can hand owned clones around before wrapping in `Arc`); this +//! worker always shares the adapter via `Arc` / +//! `SwappableAdapter`, so it isn't needed. +//! - Telemetry (OTel spans) dropped, same as every other adapter port in +//! this crate. +//! - Return-type shape deviations, forced by this worker's simpler +//! `crate::adapter::TopicInfo` (`{name, depth}` vs the engine's `{name, +//! broker_type, subscriber_count}`) and `crate::store::TopicStats` +//! (`{depth, dlq_depth, delivered, failed}` vs the engine's `{depth, +//! consumer_count, dlq_depth, config}`) and the trait's `dlq_peek` +//! returning `Vec` instead of the engine's richer `Vec`: +//! - `list_topics`: `depth` carries the per-topic subscriber count (same +//! repurposing `adapters/redis.rs::list_topics` already documents for +//! its own `TopicInfo`, since this worker's struct has no +//! `subscriber_count` field). +//! - `topic_stats`: `consumer_count` is dropped (no field to put it in); +//! `delivered`/`failed` are not tracked by this adapter (RabbitMQ is the +//! source of truth for queue depth, not per-message delivery counters) +//! and are always `0`, same as the engine's rabbitmq `topic_stats` never +//! populating equivalent fields either. +//! - `dlq_peek`: builds the same fields the engine's `DlqMessage` carries +//! (`id`, `payload`, `error`, `failed_at`, `retries`, `size_bytes`) as a +//! JSON object per message, merging the engine's separate +//! `dlq_messages`/`dlq_peek` methods into this trait's single paginated +//! `dlq_peek(topic, offset, limit)` (the engine's `dlq_messages` is +//! effectively `dlq_peek(topic, 0, count)` with a lighter-weight +//! rawvalue-only return; this worker's trait only exposes the +//! paginated shape and defaults `dlq_messages` to calling it). + +#![cfg(feature = "rabbitmq")] + +use std::{ + collections::HashMap, + str::FromStr, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, +}; + +use async_trait::async_trait; +use lapin::{message::Delivery, options::*, Channel, Connection, ConnectionProperties}; +use serde_json::Value; +use tokio::{sync::RwLock, task::JoinHandle}; +use uuid::Uuid; + +use crate::adapter::{FunctionQueueConfig, QueueAdapter, QueueMessage, TopicInfo}; +use crate::store::TopicStats; +use crate::subscriber_config::SubscriberQueueConfig; +use crate::trigger::Invoker; + +use super::naming::{FnQueueNames, RabbitNames}; +use super::publisher::Publisher; +use super::retry::RetryHandler; +use super::topology::TopologyManager; +use super::types::{priority_from_data, Job, QueueMode, RabbitMQConfig}; +use super::worker::Worker; + +pub struct RabbitMQAdapter { + publisher: Arc, + retry_handler: Arc, + topology: Arc, + channel: Arc, + subscriptions: Arc>>, + invoker: Arc, + config: RabbitMQConfig, + delivery_map: Arc>>, + delivery_counter: Arc, +} + +struct SubscriptionInfo { + id: String, + consumer_tag: String, + task_handle: JoinHandle<()>, +} + +impl RabbitMQAdapter { + /// Resolve the DLQ queue name for a given topic. Handles both function + /// queues (`__fn_queue::` prefix) and topic-based queues. + fn resolve_dlq_name(topic: &str) -> (String, bool) { + if let Some(queue_name) = topic.strip_prefix("__fn_queue::") { + (FnQueueNames::new(queue_name).dlq(), true) + } else { + (RabbitNames::new(topic).dlq(), false) + } + } + + /// Scan a DLQ for a specific message by ID, applying `on_found` when the + /// target is located. Non-target messages are nacked back to the queue. + /// + /// CONCURRENCY NOTE: this uses `basic_get` + `nack(requeue)`, which is + /// not atomic. Under concurrent DLQ operations, messages may be + /// reordered. The iteration is bounded by the queue depth snapshot to + /// prevent infinite loops from requeued messages, but concurrent + /// producers may cause messages to be missed. + async fn find_dlq_message( + &self, + topic: &str, + message_id: &str, + on_found: F, + ) -> anyhow::Result + where + F: FnOnce(&Delivery, bool, &str) -> Fut, + Fut: std::future::Future>, + { + let (dlq_name, is_fn_queue) = Self::resolve_dlq_name(topic); + + let queue_info = self + .channel + .queue_declare( + &dlq_name, + lapin::options::QueueDeclareOptions { + passive: true, + ..Default::default() + }, + lapin::types::FieldTable::default(), + ) + .await + .map_err(|e| anyhow::anyhow!("Failed to get DLQ info: {}", e))?; + + let count = queue_info.message_count(); + + for _ in 0..count { + let get_result = self + .channel + .basic_get(&dlq_name, BasicGetOptions { no_ack: false }) + .await + .map_err(|e| anyhow::anyhow!("Failed to get message from DLQ: {}", e))?; + + let Some(delivery) = get_result else { break }; + + // Determine if this is the target message. + // Function queue DLQ: stable ID from AMQP message_id or body hash + // Topic DLQ: ID is in the JSON payload at job.id + let is_target = if is_fn_queue { + delivery_stable_id(&delivery.delivery) == message_id + } else { + let dlq_payload: Value = + serde_json::from_slice(&delivery.delivery.data).unwrap_or(Value::Null); + dlq_payload + .get("job") + .and_then(|j| j.get("id")) + .and_then(|id| id.as_str()) + == Some(message_id) + }; + + if is_target { + let queue_name = topic.strip_prefix("__fn_queue::").unwrap_or(topic); + on_found(&delivery.delivery, is_fn_queue, queue_name).await?; + + delivery + .delivery + .ack(BasicAckOptions { multiple: false }) + .await + .map_err(|e| anyhow::anyhow!("Failed to ack target message: {}", e))?; + + return Ok(true); + } + + // Not the target -- put it back + delivery + .delivery + .nack(BasicNackOptions { + requeue: true, + multiple: false, + }) + .await + .map_err(|e| anyhow::anyhow!("Failed to nack non-target message: {}", e))?; + } + + Ok(false) + } + + pub async fn new(config: RabbitMQConfig, invoker: Arc) -> anyhow::Result { + let connection = Connection::connect(&config.amqp_url, ConnectionProperties::default()) + .await + .map_err(|e| { + anyhow::anyhow!( + "Failed to connect to RabbitMQ at {}: {}", + config.amqp_url, + e + ) + })?; + + let channel = connection + .create_channel() + .await + .map_err(|e| anyhow::anyhow!("Failed to create RabbitMQ channel: {}", e))?; + + let effective_prefetch = match config.queue_mode { + QueueMode::Fifo => 1, + QueueMode::Standard => config.prefetch_count, + }; + + channel + .basic_qos(effective_prefetch, BasicQosOptions::default()) + .await + .map_err(|e| anyhow::anyhow!("Failed to set QoS: {}", e))?; + + let channel = Arc::new(channel); + let publisher = Arc::new(Publisher::new(Arc::clone(&channel))); + let topology = Arc::new(TopologyManager::new(Arc::clone(&channel))); + let retry_handler = Arc::new(RetryHandler::new(Arc::clone(&publisher))); + + Ok(Self { + publisher, + retry_handler, + topology, + channel, + subscriptions: Arc::new(RwLock::new(HashMap::new())), + invoker, + config, + delivery_map: Arc::new(RwLock::new(HashMap::new())), + delivery_counter: Arc::new(AtomicU64::new(0)), + }) + } + + /// Factory entry point: parses `config` (`{amqp_url, max_attempts, + /// prefetch_count, queue_mode, priority_field}`, see + /// [`RabbitMQConfig::from_value`]) and connects. Mirrors + /// `RedisAdapter::from_config`'s convention for this crate's other + /// adapters. + pub async fn from_config( + config: Option<&Value>, + invoker: Arc, + ) -> anyhow::Result { + let config = RabbitMQConfig::from_value(config); + Self::new(config, invoker).await + } +} + +/// Extracts a stable message ID from a DLQ delivery. Tries the AMQP +/// `message_id` property first, then falls back to a hash of the raw body +/// so that `dlq_peek` and `redrive_dlq_message` always agree on the ID. +fn delivery_stable_id(delivery: &Delivery) -> String { + if let Some(mid) = delivery.properties.message_id() { + let s = mid.as_str(); + if !s.is_empty() { + return s.to_string(); + } + } + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + delivery.data.hash(&mut hasher); + format!("dlq-{:016x}", hasher.finish()) +} + +#[async_trait] +impl QueueAdapter for RabbitMQAdapter { + async fn enqueue( + &self, + topic: &str, + data: Value, + traceparent: Option, + baggage: Option, + ) { + // Topic fanout publishes one message to every bound subscriber queue, so + // the priority is resolved once here from the adapter-level + // `priority_field`. Each subscriber queue honors it only if declared with + // `max_priority`, and clamps it to its own `x-max-priority`. + let priority = priority_from_data(&data, self.config.priority_field.as_deref()); + let job = Job::new(topic, data, self.config.max_attempts, traceparent, baggage) + .with_priority(priority); + + if let Err(e) = self.topology.setup_topic(topic).await { + tracing::error!( + error = ?e, + topic = %topic, + "Failed to setup RabbitMQ topology" + ); + return; + } + + if let Err(e) = self.publisher.publish(topic, &job).await { + tracing::error!( + error = ?e, + topic = %topic, + "Failed to publish to RabbitMQ" + ); + } else { + tracing::debug!( + topic = %topic, + job_id = %job.id, + "Published to RabbitMQ queue" + ); + } + } + + async fn subscribe( + &self, + topic: &str, + id: &str, + function_id: &str, + condition_function_id: Option, + queue_config: Option, + ) { + let topic = topic.to_string(); + let id = id.to_string(); + let function_id = function_id.to_string(); + let subscriptions = Arc::clone(&self.subscriptions); + + let already_subscribed = { + let subs = subscriptions.read().await; + subs.contains_key(&format!("{}:{}", topic, id)) + }; + + if already_subscribed { + tracing::warn!(topic = %topic, id = %id, "Already subscribed to topic"); + return; + } + + if let Err(e) = self.topology.setup_topic(&topic).await { + tracing::error!( + error = ?e, + topic = %topic, + "Failed to setup RabbitMQ fanout exchange" + ); + return; + } + + let subscriber_max_priority = queue_config.as_ref().and_then(|c| c.max_priority); + if let Err(e) = self + .topology + .setup_subscriber_queue(&topic, &function_id, subscriber_max_priority) + .await + { + tracing::error!( + error = ?e, + topic = %topic, + function_id = %function_id, + "Failed to setup RabbitMQ per-function queue" + ); + return; + } + + let names = RabbitNames::new(&topic); + let per_function_queue = names.function_queue(&function_id); + let consumer_tag = format!("consumer-{}", Uuid::new_v4()); + + let effective_queue_mode = queue_config + .as_ref() + .and_then(|c| c.queue_mode.as_ref()) + .map(|mode| QueueMode::from_str(mode).unwrap_or_default()) + .unwrap_or_else(|| self.config.queue_mode.clone()); + + let effective_prefetch_count = queue_config + .as_ref() + .and_then(|c| c.concurrency) + .map(|c| c as u16) + .unwrap_or(self.config.prefetch_count); + + let worker = Arc::new(Worker::new( + Arc::clone(&self.channel), + Arc::clone(&self.retry_handler), + Arc::clone(&self.invoker), + effective_queue_mode, + effective_prefetch_count, + )); + + let topic_clone = topic.clone(); + let function_id_clone = function_id.clone(); + let consumer_tag_clone = consumer_tag.clone(); + let queue_name_clone = per_function_queue.clone(); + + let task_handle = tokio::spawn(async move { + worker + .run( + topic_clone, + function_id_clone, + condition_function_id, + consumer_tag_clone, + queue_name_clone, + ) + .await; + }); + + let mut subs = subscriptions.write().await; + subs.insert( + format!("{}:{}", topic, id), + SubscriptionInfo { + id, + consumer_tag, + task_handle, + }, + ); + + tracing::debug!( + topic = %topic, + function_id = %function_id, + queue = %per_function_queue, + "Subscribed to RabbitMQ per-function queue" + ); + } + + async fn unsubscribe(&self, topic: &str, id: &str) { + let subscriptions = Arc::clone(&self.subscriptions); + let key = format!("{}:{}", topic, id); + + let mut subs = subscriptions.write().await; + + if let Some(sub_info) = subs.remove(&key) { + if sub_info.id == id { + tracing::debug!( + topic = %topic, + id = %id, + "Unsubscribing from RabbitMQ queue" + ); + + if let Err(e) = self + .channel + .basic_cancel(&sub_info.consumer_tag, BasicCancelOptions::default()) + .await + { + tracing::error!( + error = ?e, + topic = %topic, + consumer_tag = %sub_info.consumer_tag, + "Failed to cancel consumer" + ); + } + + sub_info.task_handle.abort(); + } else { + tracing::warn!( + topic = %topic, + id = %id, + "Subscription ID mismatch, not unsubscribing" + ); + subs.insert(key, sub_info); + } + } else { + tracing::warn!( + topic = %topic, + id = %id, + "No active subscription found for topic" + ); + } + } + + async fn redrive_dlq(&self, topic: &str) -> anyhow::Result { + let (dlq_name, is_fn_queue) = Self::resolve_dlq_name(topic); + let mut count: u64 = 0; + + loop { + let get_result = self + .channel + .basic_get(&dlq_name, BasicGetOptions { no_ack: false }) + .await + .map_err(|e| anyhow::anyhow!("Failed to get message from DLQ: {}", e))?; + + match get_result { + Some(delivery) => { + let republish_result: anyhow::Result<()> = if is_fn_queue { + // Function queue DLQ: raw data payload, republish directly + let queue_name = topic.strip_prefix("__fn_queue::").unwrap(); + let names = FnQueueNames::new(queue_name); + + let mut headers = delivery + .delivery + .properties + .headers() + .clone() + .unwrap_or_default(); + headers.insert("x-attempt".into(), lapin::types::AMQPValue::LongUInt(0)); + + let properties = lapin::BasicProperties::default() + .with_content_type("application/json".into()) + .with_delivery_mode(2) + .with_headers(headers); + + let properties = + if let Some(mid) = delivery.delivery.properties.message_id() { + properties.with_message_id(mid.clone()) + } else { + properties + }; + + self.channel + .basic_publish( + &names.exchange(), + queue_name, + BasicPublishOptions::default(), + &delivery.delivery.data, + properties, + ) + .await + .map_err(|e| anyhow::anyhow!("Failed to republish: {}", e))? + .await + .map_err(|e| anyhow::anyhow!("Failed to confirm: {}", e))?; + Ok(()) + } else { + // Topic DLQ: wrapped payload with job/error/exhausted_at + let dlq_payload: Value = serde_json::from_slice(&delivery.delivery.data) + .map_err(|e| anyhow::anyhow!("Failed to parse DLQ message: {}", e))?; + + let job: Job = serde_json::from_value( + dlq_payload + .get("job") + .ok_or_else(|| anyhow::anyhow!("DLQ message missing 'job' field"))? + .clone(), + ) + .map_err(|e| anyhow::anyhow!("Failed to parse job: {}", e))?; + + let mut redriven = job; + redriven.attempts_made = 0; + + self.publisher + .publish(topic, &redriven) + .await + .map_err(|e| anyhow::anyhow!("Failed to republish: {}", e))?; + Ok(()) + }; + + if let Err(e) = republish_result { + tracing::error!(error = ?e, "Failed to republish DLQ message"); + delivery + .delivery + .nack(BasicNackOptions { + requeue: true, + multiple: false, + }) + .await + .map_err(|e| anyhow::anyhow!("Failed to nack message: {}", e))?; + break; + } + + delivery + .delivery + .ack(BasicAckOptions { multiple: false }) + .await + .map_err(|e| anyhow::anyhow!("Failed to ack message: {}", e))?; + + count += 1; + } + None => break, + } + } + + Ok(count) + } + + async fn redrive_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { + let publisher = self.publisher.clone(); + let channel = Arc::clone(&self.channel); + let topic_owned = topic.to_string(); + + self.find_dlq_message(topic, message_id, |delivery, is_fn_queue, queue_name| { + let delivery_data = delivery.data.clone(); + let delivery_props = delivery.properties.clone(); + let queue_name = queue_name.to_string(); + + async move { + if is_fn_queue { + // Function queue: republish raw data to the function queue exchange + let names = FnQueueNames::new(&queue_name); + + // Preserve original headers, reset attempt counter + let mut headers = delivery_props.headers().clone().unwrap_or_default(); + headers.insert("x-attempt".into(), lapin::types::AMQPValue::LongUInt(0)); + + let properties = lapin::BasicProperties::default() + .with_content_type("application/json".into()) + .with_delivery_mode(2) + .with_headers(headers); + + // Copy message_id if present + let properties = if let Some(mid) = delivery_props.message_id() { + properties.with_message_id(mid.clone()) + } else { + properties + }; + + channel + .basic_publish( + &names.exchange(), + &queue_name, + BasicPublishOptions::default(), + &delivery_data, + properties, + ) + .await + .map_err(|e| { + anyhow::anyhow!("Failed to republish to function queue: {}", e) + })? + .await + .map_err(|e| anyhow::anyhow!("Failed to confirm republish: {}", e))?; + } else { + // Topic queue: parse job, reset attempts, republish via Publisher + let dlq_payload: Value = serde_json::from_slice(&delivery_data) + .map_err(|e| anyhow::anyhow!("Failed to parse DLQ message: {}", e))?; + + let job_value = dlq_payload + .get("job") + .ok_or_else(|| anyhow::anyhow!("DLQ message missing 'job' field"))?; + + let mut job: Job = serde_json::from_value(job_value.clone()) + .map_err(|e| anyhow::anyhow!("Failed to parse job: {}", e))?; + + job.attempts_made = 0; + + publisher + .publish(&topic_owned, &job) + .await + .map_err(|e| anyhow::anyhow!("Failed to republish message: {}", e))?; + } + + Ok(()) + } + }) + .await + } + + async fn discard_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { + // Discard = just ack (handled by find_dlq_message after on_found returns) + self.find_dlq_message( + topic, + message_id, + |_delivery, _is_fn_queue, _queue_name| async { Ok(()) }, + ) + .await + } + + async fn dlq_count(&self, topic: &str) -> anyhow::Result { + // Function queues use FnQueueNames (e.g., __fn_queue::orders -> ::dlq.queue), + // while topic-based queues use RabbitNames (e.g., user.created -> .dlq). + let (dlq_name, _is_fn_queue) = Self::resolve_dlq_name(topic); + + let queue = self + .channel + .queue_declare( + &dlq_name, + lapin::options::QueueDeclareOptions { + passive: true, + ..Default::default() + }, + lapin::types::FieldTable::default(), + ) + .await + .map_err(|e| anyhow::anyhow!("Failed to get DLQ info: {}", e))?; + + Ok(queue.message_count() as u64) + } + + /// Merges the engine's separate `dlq_messages`/`dlq_peek` methods (see + /// the module doc) into this trait's single paginated shape, building + /// the same fields the engine's `DlqMessage` struct carries as a JSON + /// object per entry instead of a dedicated type. + async fn dlq_peek(&self, topic: &str, offset: u64, limit: u64) -> anyhow::Result> { + let (dlq_name, is_fn_queue) = Self::resolve_dlq_name(topic); + + let queue_depth = match self + .channel + .queue_declare( + &dlq_name, + lapin::options::QueueDeclareOptions { + passive: true, + ..Default::default() + }, + lapin::types::FieldTable::default(), + ) + .await + { + Ok(info) => info.message_count() as u64, + Err(_) => return Ok(vec![]), + }; + + let fetch_count = (offset + limit).min(queue_depth) as usize; + let mut results = Vec::new(); + let mut deliveries_to_nack = Vec::new(); + + for i in 0..fetch_count { + let get_result = self + .channel + .basic_get(&dlq_name, BasicGetOptions { no_ack: false }) + .await + .map_err(|e| anyhow::anyhow!("Failed to get message from DLQ: {}", e))?; + + let Some(delivery) = get_result else { break }; + + if (i as u64) >= offset { + let raw_data = &delivery.delivery.data; + let payload: Value = serde_json::from_slice(raw_data).unwrap_or(Value::Null); + + let dlq_value = if is_fn_queue { + // Function queue DLQ: stable ID from AMQP properties or body hash + let id = delivery_stable_id(&delivery.delivery); + + let function_id = delivery + .delivery + .properties + .headers() + .as_ref() + .and_then(|h| h.inner().get("function_id")) + .and_then(|v| match v { + lapin::types::AMQPValue::LongString(s) => Some(s.to_string()), + _ => None, + }) + .unwrap_or_default(); + + let retries = delivery + .delivery + .properties + .headers() + .as_ref() + .and_then(|h| h.inner().get("x-attempt")) + .and_then(|v| match v { + lapin::types::AMQPValue::LongUInt(n) => Some(*n), + _ => None, + }) + .unwrap_or(0); + + serde_json::json!({ + "id": id, + "payload": payload, + "error": format!("Function {} exhausted retries", function_id), + "failed_at": 0, + "retries": retries, + "size_bytes": raw_data.len() as u64, + }) + } else { + // Topic-based DLQ: wrapped payload with job/error/exhausted_at + let id = payload + .get("job") + .and_then(|j| j.get("id")) + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(); + + let job_payload = payload + .get("job") + .and_then(|j| j.get("data")) + .cloned() + .unwrap_or(Value::Null); + + let error = payload + .get("error") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + let failed_at = payload + .get("exhausted_at") + .and_then(|v| v.as_u64()) + .map(|ms| ms / 1000) + .unwrap_or(0); + + let retries = payload + .get("job") + .and_then(|j| j.get("attempts_made")) + .and_then(|v| v.as_u64()) + .unwrap_or(0); + + serde_json::json!({ + "id": id, + "payload": job_payload, + "error": error, + "failed_at": failed_at, + "retries": retries, + "size_bytes": raw_data.len() as u64, + }) + }; + + results.push(dlq_value); + } + + deliveries_to_nack.push(delivery); + } + + // Nack all back to DLQ (peek, not consume) + for delivery in deliveries_to_nack { + let _ = delivery + .delivery + .nack(BasicNackOptions { + requeue: true, + multiple: false, + }) + .await; + } + + Ok(results) + } + + #[allow(clippy::too_many_arguments)] + async fn publish_to_function_queue( + &self, + queue_name: &str, + function_id: &str, + data: Value, + message_id: &str, + _max_retries: u32, + _backoff_ms: u64, + traceparent: Option, + baggage: Option, + priority: Option, + ) { + let names = FnQueueNames::new(queue_name); + + let payload = match serde_json::to_vec(&data) { + Ok(p) => p, + Err(e) => { + tracing::error!(error = %e, "Failed to serialize data"); + return; + } + }; + + let mut headers = lapin::types::FieldTable::default(); + headers.insert( + "function_id".into(), + lapin::types::AMQPValue::LongString(function_id.into()), + ); + if let Some(tp) = &traceparent { + headers.insert( + "traceparent".into(), + lapin::types::AMQPValue::LongString(tp.as_str().into()), + ); + } + if let Some(bg) = &baggage { + headers.insert( + "baggage".into(), + lapin::types::AMQPValue::LongString(bg.as_str().into()), + ); + } + + let mut properties = lapin::BasicProperties::default() + .with_content_type("application/json".into()) + .with_delivery_mode(2) + .with_message_id(message_id.into()) + .with_headers(headers); + if let Some(p) = priority { + properties = properties.with_priority(p); + } + + match self + .channel + .basic_publish( + &names.exchange(), + queue_name, + BasicPublishOptions::default(), + &payload, + properties, + ) + .await + { + Ok(confirm) => { + if let Err(e) = confirm.await { + tracing::error!(error = %e, queue = %queue_name, "Failed to confirm publish to function queue"); + } + } + Err(e) => { + tracing::error!(error = %e, queue = %queue_name, "Failed to publish to function queue"); + } + } + } + + async fn setup_function_queue( + &self, + queue_name: &str, + config: &FunctionQueueConfig, + ) -> anyhow::Result<()> { + self.topology + .setup_function_queue(queue_name, config.backoff_ms, config.max_priority) + .await + .map_err(|e| anyhow::anyhow!("Failed to setup function queue topology: {}", e)) + } + + async fn consume_function_queue( + &self, + queue_name: &str, + prefetch: u32, + ) -> anyhow::Result> { + use futures::StreamExt; + + let names = FnQueueNames::new(queue_name); + + self.channel + .basic_qos(prefetch as u16, BasicQosOptions::default()) + .await + .map_err(|e| anyhow::anyhow!("Failed to set QoS: {}", e))?; + + let consumer_tag = format!("fn-queue-{}-{}", queue_name, Uuid::new_v4()); + let mut consumer = self + .channel + .basic_consume( + &names.queue(), + &consumer_tag, + BasicConsumeOptions::default(), + lapin::types::FieldTable::default(), + ) + .await + .map_err(|e| anyhow::anyhow!("Failed to create consumer: {}", e))?; + + let (tx, rx) = tokio::sync::mpsc::channel(prefetch as usize); + let delivery_map = Arc::clone(&self.delivery_map); + let delivery_counter = Arc::clone(&self.delivery_counter); + + tokio::spawn(async move { + while let Some(delivery_result) = consumer.next().await { + match delivery_result { + Ok(delivery) => { + let delivery_id = delivery_counter.fetch_add(1, Ordering::SeqCst); + + let data: Value = match serde_json::from_slice(&delivery.data) { + Ok(v) => v, + Err(e) => { + tracing::error!(error = %e, "Failed to parse function queue message"); + let _ = delivery + .nack(BasicNackOptions { + requeue: false, + ..Default::default() + }) + .await; + continue; + } + }; + + let headers = delivery.properties.headers().as_ref(); + + let function_id = headers + .and_then(|h| h.inner().get("function_id")) + .and_then(|v| match v { + lapin::types::AMQPValue::LongString(s) => Some(s.to_string()), + _ => None, + }) + .unwrap_or_default(); + + let traceparent = headers + .and_then(|h| h.inner().get("traceparent")) + .and_then(|v| match v { + lapin::types::AMQPValue::LongString(s) => Some(s.to_string()), + _ => None, + }); + + let baggage = + headers + .and_then(|h| h.inner().get("baggage")) + .and_then(|v| match v { + lapin::types::AMQPValue::LongString(s) => Some(s.to_string()), + _ => None, + }); + + let attempt = headers + .and_then(|h| h.inner().get("x-attempt")) + .and_then(|v| match v { + lapin::types::AMQPValue::LongUInt(n) => Some(*n), + _ => None, + }) + .unwrap_or(0); + + let message_id = delivery + .properties + .message_id() + .as_ref() + .map(|s| s.to_string()); + + delivery_map.write().await.insert(delivery_id, delivery); + + let msg = QueueMessage { + delivery_id, + function_id, + data, + attempt, + message_id, + traceparent, + baggage, + }; + + if tx.send(msg).await.is_err() { + // Receiver dropped -- nack the delivery we just stored so + // RabbitMQ requeues it rather than leaving it stranded. + if let Some(d) = delivery_map.write().await.remove(&delivery_id) { + let _ = d + .nack(BasicNackOptions { + requeue: true, + ..Default::default() + }) + .await; + } + break; + } + } + Err(e) => { + tracing::error!(error = %e, "Error receiving delivery from function queue"); + } + } + } + }); + + Ok(rx) + } + + async fn ack_function_queue(&self, _queue_name: &str, delivery_id: u64) -> anyhow::Result<()> { + let delivery = self.delivery_map.write().await.remove(&delivery_id); + if let Some(delivery) = delivery { + delivery + .ack(BasicAckOptions::default()) + .await + .map_err(|e| anyhow::anyhow!("Failed to ack: {}", e))?; + } + Ok(()) + } + + async fn nack_function_queue( + &self, + queue_name: &str, + delivery_id: u64, + attempt: u32, + max_retries: u32, + ) -> anyhow::Result<()> { + let delivery = self.delivery_map.write().await.remove(&delivery_id); + let delivery = match delivery { + Some(d) => d, + None => return Ok(()), + }; + + if attempt < max_retries { + // Retry: ack the original delivery, republish to the retry exchange. + // The retry queue has a TTL; after expiry RabbitMQ dead-letters the + // message back to the main exchange (configured via + // x-dead-letter-exchange on the retry queue). + delivery + .ack(BasicAckOptions::default()) + .await + .map_err(|e| anyhow::anyhow!("Failed to ack for retry: {}", e))?; + + let names = FnQueueNames::new(queue_name); + + let mut headers = delivery.properties.headers().clone().unwrap_or_default(); + + // Increment our own attempt counter so classic queues (which do not + // populate x-delivery-count) can still track retry depth. + let current_attempt = headers + .inner() + .get("x-attempt") + .and_then(|v| match v { + lapin::types::AMQPValue::LongUInt(n) => Some(*n), + _ => None, + }) + .unwrap_or(0); + headers.insert( + "x-attempt".into(), + lapin::types::AMQPValue::LongUInt(current_attempt + 1), + ); + + let mut properties = lapin::BasicProperties::default() + .with_content_type("application/json".into()) + .with_delivery_mode(2) + .with_headers(headers); + + // Preserve message_id through retries so DLQ messages remain identifiable + if let Some(mid) = delivery.properties.message_id() { + properties = properties.with_message_id(mid.clone()); + } + + self.channel + .basic_publish( + &names.retry_exchange(), + queue_name, + BasicPublishOptions::default(), + &delivery.data, + properties, + ) + .await + .map_err(|e| anyhow::anyhow!("Failed to publish to retry exchange: {}", e))? + .await + .map_err(|e| anyhow::anyhow!("Failed to confirm retry publish: {}", e))?; + + tracing::debug!( + queue = %queue_name, + attempt = attempt, + max_retries = max_retries, + "Message sent to retry queue" + ); + } else { + // Exhausted: nack without requeue. The main queue's DLX points to the + // DLQ exchange, so RabbitMQ routes the message there automatically. + delivery + .nack(BasicNackOptions { + requeue: false, + ..Default::default() + }) + .await + .map_err(|e| anyhow::anyhow!("Failed to nack to DLQ: {}", e))?; + + tracing::warn!( + queue = %queue_name, + attempt = attempt, + max_retries = max_retries, + "Message exhausted retries, routed to DLQ" + ); + } + + Ok(()) + } + + /// Deviation from the engine (see module doc): `consumer_count` is + /// dropped (no field for it in this worker's `TopicStats`); + /// `delivered`/`failed` are always `0` (not tracked by this adapter). + /// + /// Inherits one engine quirk verbatim: the "main queue depth" is looked + /// up via `RabbitNames::new(topic).queue()` (`iii.{topic}.queue`), a + /// queue name that topic-fanout subscriptions never actually declare + /// (`subscribe` declares per-function queues via + /// `RabbitNames::function_queue`, never the bare `.queue()` name) -- so + /// for topic-based (non-`__fn_queue::`) topics this always resolves to + /// depth `0` in both the engine and this port. + async fn topic_stats(&self, topic: &str) -> anyhow::Result { + let (queue_name, dlq_name) = if let Some(name) = topic.strip_prefix("__fn_queue::") { + let names = FnQueueNames::new(name); + (names.queue(), names.dlq()) + } else { + let names = RabbitNames::new(topic); + (names.queue(), names.dlq()) + }; + + let depth = match self + .channel + .queue_declare( + &queue_name, + lapin::options::QueueDeclareOptions { + passive: true, + ..Default::default() + }, + lapin::types::FieldTable::default(), + ) + .await + { + Ok(info) => info.message_count() as u64, + Err(_) => 0, + }; + + let dlq_depth = match self + .channel + .queue_declare( + &dlq_name, + lapin::options::QueueDeclareOptions { + passive: true, + ..Default::default() + }, + lapin::types::FieldTable::default(), + ) + .await + { + Ok(info) => info.message_count() as u64, + Err(_) => 0, + }; + + Ok(TopicStats { + depth, + dlq_depth, + delivered: 0, + failed: 0, + }) + } + + /// Deviation from the engine (see module doc): `depth` carries the + /// per-topic subscriber count (this worker's `TopicInfo` has no + /// `subscriber_count` field), same repurposing `adapters/redis.rs` + /// already documents for its own `list_topics`. + async fn list_topics(&self) -> anyhow::Result> { + let subs = self.subscriptions.read().await; + let mut topics: HashMap = HashMap::new(); + for key in subs.keys() { + // subscription key format is "topic:id" + if let Some(topic) = key.split(':').next() { + *topics.entry(topic.to_string()).or_insert(0u64) += 1; + } + } + Ok(topics + .into_iter() + .map(|(name, count)| TopicInfo { name, depth: count }) + .collect()) + } + + async fn shutdown(&self) { + let mut subs = self.subscriptions.write().await; + for (_, sub) in subs.drain() { + let _ = self + .channel + .basic_cancel(&sub.consumer_tag, BasicCancelOptions::default()) + .await; + sub.task_handle.abort(); + } + } +} diff --git a/queue/src/adapters/rabbitmq/consumer.rs b/queue/src/adapters/rabbitmq/consumer.rs new file mode 100644 index 000000000..01250bc51 --- /dev/null +++ b/queue/src/adapters/rabbitmq/consumer.rs @@ -0,0 +1,70 @@ +//! Parses a `Job` out of an inbound AMQP delivery, recovering the +//! attempt count from the `x-iii-attempts` header (falling back to the +//! serialized `Job.attempts_made` when absent). +//! +//! 1:1 port of `engine/src/workers/queue/adapters/rabbitmq/consumer.rs` — no +//! engine dependency, ports verbatim. + +#![cfg(feature = "rabbitmq")] + +use lapin::{ + message::Delivery, + types::{AMQPValue, FieldTable}, +}; + +use super::naming::EXCHANGE_PREFIX; +use super::types::Job; + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub enum ConsumerError { + Serialization(serde_json::Error), +} + +impl std::fmt::Display for ConsumerError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ConsumerError::Serialization(e) => write!(f, "Serialization error: {}", e), + } + } +} + +impl std::error::Error for ConsumerError {} + +impl From for ConsumerError { + fn from(err: serde_json::Error) -> Self { + ConsumerError::Serialization(err) + } +} + +pub struct JobParser; + +impl JobParser { + pub fn parse_from_delivery(delivery: &Delivery) -> Result { + let mut job: Job = serde_json::from_slice(&delivery.data)?; + + if let Some(headers) = &delivery.properties.headers() { + if let Some(attempts) = Self::extract_attempts(headers)? { + job.attempts_made = attempts; + } + } + + Ok(job) + } + + fn extract_attempts(headers: &FieldTable) -> Result> { + let header_name = format!("x-{}-attempts", EXCHANGE_PREFIX); + if let Some(value) = headers.inner().get(header_name.as_str()) { + match value { + AMQPValue::LongUInt(v) => Ok(Some(*v)), + AMQPValue::ShortUInt(v) => Ok(Some(*v as u32)), + AMQPValue::LongInt(v) => Ok(Some(*v as u32)), + AMQPValue::ShortInt(v) => Ok(Some(*v as u32)), + _ => Ok(None), + } + } else { + Ok(None) + } + } +} diff --git a/queue/src/adapters/rabbitmq/mod.rs b/queue/src/adapters/rabbitmq/mod.rs index f85a9840e..b4ff8cc1f 100644 --- a/queue/src/adapters/rabbitmq/mod.rs +++ b/queue/src/adapters/rabbitmq/mod.rs @@ -6,5 +6,13 @@ //! tree, and every individual file (`#![cfg(feature = "rabbitmq")]`), compile //! out entirely with `--no-default-features`. -pub mod naming; +mod adapter; +mod consumer; +mod naming; +mod publisher; +mod retry; +mod topology; pub mod types; +mod worker; + +pub use adapter::RabbitMQAdapter; diff --git a/queue/src/adapters/rabbitmq/publisher.rs b/queue/src/adapters/rabbitmq/publisher.rs new file mode 100644 index 000000000..8e8f1d47e --- /dev/null +++ b/queue/src/adapters/rabbitmq/publisher.rs @@ -0,0 +1,176 @@ +//! Publishes `Job`s to fanout exchanges, per-function subscriber queues, and +//! DLQs. +//! +//! 1:1 port of `engine/src/workers/queue/adapters/rabbitmq/publisher.rs` — no +//! engine dependency (pure `lapin::Channel` operations), ports verbatim. + +#![cfg(feature = "rabbitmq")] + +use std::sync::Arc; + +use lapin::{ + options::*, + types::{AMQPValue, FieldTable}, + Channel, +}; + +use super::naming::{RabbitNames, EXCHANGE_PREFIX}; +use super::types::Job; + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub enum PublisherError { + Lapin(lapin::Error), + Serialization(serde_json::Error), +} + +impl std::fmt::Display for PublisherError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PublisherError::Lapin(e) => write!(f, "RabbitMQ error: {}", e), + PublisherError::Serialization(e) => write!(f, "Serialization error: {}", e), + } + } +} + +impl std::error::Error for PublisherError {} + +impl From for PublisherError { + fn from(err: lapin::Error) -> Self { + PublisherError::Lapin(err) + } +} + +impl From for PublisherError { + fn from(err: serde_json::Error) -> Self { + PublisherError::Serialization(err) + } +} + +pub struct Publisher { + channel: Arc, +} + +impl Publisher { + pub fn new(channel: Arc) -> Self { + Self { channel } + } + + pub async fn publish(&self, topic: &str, job: &Job) -> Result<()> { + let names = RabbitNames::new(topic); + let headers = self.build_headers(job); + self.publish_to_exchange(&names.exchange(), topic, job, Some(headers)) + .await + } + + pub async fn requeue(&self, topic: &str, job: &Job, function_id: Option<&str>) -> Result<()> { + if let Some(fid) = function_id { + // Per-function subscriber queue: publish directly to the function's + // queue (default exchange) to avoid re-fanning out to all subscribers. + let names = RabbitNames::new(topic); + let queue_name = names.function_queue(fid); + let headers = self.build_headers(job); + self.publish_to_exchange("", &queue_name, job, Some(headers)) + .await + } else { + self.publish(topic, job).await + } + } + + pub async fn publish_to_dlq( + &self, + topic: &str, + job: &Job, + error: &str, + function_id: Option<&str>, + ) -> Result<()> { + let names = RabbitNames::new(topic); + let dlq_name = if let Some(fid) = function_id { + names.function_dlq(fid) + } else { + names.dlq() + }; + + let payload = serde_json::to_vec(&serde_json::json!({ + "job": job, + "error": error, + "exhausted_at": std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64, + }))?; + + let properties = lapin::BasicProperties::default() + .with_content_type("application/json".into()) + .with_delivery_mode(2); + + self.channel + .basic_publish( + "", + &dlq_name, + BasicPublishOptions::default(), + &payload, + properties, + ) + .await? + .await?; + + Ok(()) + } + + async fn publish_to_exchange( + &self, + exchange: &str, + routing_key: &str, + job: &Job, + headers: Option, + ) -> Result<()> { + let payload = serde_json::to_vec(job)?; + + let mut properties = lapin::BasicProperties::default() + .with_content_type("application/json".into()) + .with_delivery_mode(2) + .with_headers(headers.unwrap_or_default()); + // Carry the job's priority through fanout publishes, requeues, and + // DLQ-redrive republishes so ordering survives retries on priority + // queues. No-op on queues without `x-max-priority`. + if let Some(p) = job.priority { + properties = properties.with_priority(p); + } + + self.channel + .basic_publish( + exchange, + routing_key, + BasicPublishOptions::default(), + &payload, + properties, + ) + .await? + .await?; + + Ok(()) + } + + fn build_headers(&self, job: &Job) -> FieldTable { + let mut headers = FieldTable::default(); + headers.insert( + format!("x-{}-job-id", EXCHANGE_PREFIX).into(), + AMQPValue::LongString(job.id.clone().into()), + ); + headers.insert( + format!("x-{}-attempts", EXCHANGE_PREFIX).into(), + AMQPValue::LongUInt(job.attempts_made), + ); + headers.insert( + format!("x-{}-max-attempts", EXCHANGE_PREFIX).into(), + AMQPValue::LongUInt(job.max_attempts), + ); + headers.insert( + format!("x-{}-created-at", EXCHANGE_PREFIX).into(), + AMQPValue::LongString(job.created_at.to_string().into()), + ); + headers + } +} diff --git a/queue/src/adapters/rabbitmq/retry.rs b/queue/src/adapters/rabbitmq/retry.rs new file mode 100644 index 000000000..ff4e97a3a --- /dev/null +++ b/queue/src/adapters/rabbitmq/retry.rs @@ -0,0 +1,88 @@ +//! Retry/DLQ decision logic for topic-fanout jobs: requeue while attempts +//! remain, publish to DLQ once exhausted. +//! +//! 1:1 port of `engine/src/workers/queue/adapters/rabbitmq/retry.rs` — no +//! engine dependency, ports verbatim. + +#![cfg(feature = "rabbitmq")] + +use std::sync::Arc; + +use super::publisher::Publisher; +use super::types::Job; + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub enum RetryError { + Publisher(super::publisher::PublisherError), +} + +impl std::fmt::Display for RetryError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RetryError::Publisher(e) => write!(f, "Publisher error: {}", e), + } + } +} + +impl std::error::Error for RetryError {} + +impl From for RetryError { + fn from(err: super::publisher::PublisherError) -> Self { + RetryError::Publisher(err) + } +} + +pub struct RetryHandler { + publisher: Arc, +} + +impl RetryHandler { + pub fn new(publisher: Arc) -> Self { + Self { publisher } + } + + pub async fn handle_success(&self, topic: &str, job: &Job) -> Result<()> { + tracing::debug!( + job_id = %job.id, + topic = %topic, + "Job completed successfully" + ); + Ok(()) + } + + pub async fn handle_failure( + &self, + topic: &str, + job: &mut Job, + error: &str, + function_id: Option<&str>, + ) -> Result<()> { + job.increment_attempts(); + + if job.is_exhausted() { + self.publisher + .publish_to_dlq(topic, job, error, function_id) + .await?; + + tracing::warn!( + job_id = %job.id, + topic = %topic, + attempts = job.attempts_made, + "Job exhausted retries, moved to DLQ" + ); + } else { + self.publisher.requeue(topic, job, function_id).await?; + + tracing::debug!( + job_id = %job.id, + topic = %topic, + attempts = job.attempts_made, + "Job requeued for retry" + ); + } + + Ok(()) + } +} diff --git a/queue/src/adapters/rabbitmq/topology.rs b/queue/src/adapters/rabbitmq/topology.rs new file mode 100644 index 000000000..da44c8aee --- /dev/null +++ b/queue/src/adapters/rabbitmq/topology.rs @@ -0,0 +1,276 @@ +//! RabbitMQ topology setup: fanout exchanges, per-function subscriber +//! queues, and the retry/DLX topology for function queues. +//! +//! 1:1 port of `engine/src/workers/queue/adapters/rabbitmq/topology.rs` — no +//! engine dependency (pure `lapin::Channel` operations), ports verbatim. + +#![cfg(feature = "rabbitmq")] + +use std::sync::Arc; + +use lapin::{ + options::*, + types::{AMQPValue, FieldTable}, + Channel, +}; + +use super::naming::{FnQueueNames, RabbitNames}; + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub enum TopologyError { + Lapin(lapin::Error), +} + +impl std::fmt::Display for TopologyError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TopologyError::Lapin(e) => write!(f, "RabbitMQ error: {}", e), + } + } +} + +impl std::error::Error for TopologyError {} + +impl From for TopologyError { + fn from(err: lapin::Error) -> Self { + TopologyError::Lapin(err) + } +} + +pub struct TopologyManager { + channel: Arc, +} + +impl TopologyManager { + pub fn new(channel: Arc) -> Self { + Self { channel } + } + + pub async fn setup_topic(&self, topic: &str) -> Result<()> { + let names = RabbitNames::new(topic); + + self.channel + .exchange_declare( + &names.exchange(), + lapin::ExchangeKind::Fanout, + ExchangeDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await?; + + tracing::debug!(topic = %topic, "RabbitMQ fanout exchange setup complete"); + Ok(()) + } + + pub async fn setup_subscriber_queue( + &self, + topic: &str, + function_id: &str, + max_priority: Option, + ) -> Result<()> { + let names = RabbitNames::new(topic); + + let queue_name = names.function_queue(function_id); + let dlq_name = names.function_dlq(function_id); + + self.channel + .queue_declare( + &dlq_name, + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await?; + + let mut queue_args = FieldTable::default(); + with_max_priority(&mut queue_args, max_priority); + + self.channel + .queue_declare( + &queue_name, + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + queue_args, + ) + .await?; + + self.channel + .queue_bind( + &queue_name, + &names.exchange(), + "", + QueueBindOptions::default(), + FieldTable::default(), + ) + .await?; + + tracing::debug!( + topic = %topic, + function_id = %function_id, + queue = %queue_name, + "RabbitMQ per-function queue setup complete" + ); + Ok(()) + } + + pub async fn setup_function_queue( + &self, + queue_name: &str, + backoff_ms: u64, + max_priority: Option, + ) -> Result<()> { + let names = FnQueueNames::new(queue_name); + + // Main exchange + queue with DLX to retry + self.channel + .exchange_declare( + &names.exchange(), + lapin::ExchangeKind::Direct, + ExchangeDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await?; + + // DLX points to DLQ exchange: nack(requeue=false) sends exhausted + // messages to DLQ automatically. Retry is handled explicitly by + // the adapter (ack + publish to retry exchange). + let mut main_queue_args = FieldTable::default(); + main_queue_args.insert( + "x-dead-letter-exchange".into(), + AMQPValue::LongString(names.dlq_exchange().into()), + ); + // A priority queue must be declared with `x-max-priority`; declare the + // retry queue with it too so retried messages keep their ordering (the + // `priority` property survives dead-lettering). The DLQ stays plain. + with_max_priority(&mut main_queue_args, max_priority); + + self.channel + .queue_declare( + &names.queue(), + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + main_queue_args, + ) + .await?; + + self.channel + .queue_bind( + &names.queue(), + &names.exchange(), + queue_name, + QueueBindOptions::default(), + FieldTable::default(), + ) + .await?; + + // Retry exchange + queue (TTL -> back to main) + self.channel + .exchange_declare( + &names.retry_exchange(), + lapin::ExchangeKind::Direct, + ExchangeDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await?; + + let mut retry_queue_args = FieldTable::default(); + retry_queue_args.insert( + "x-message-ttl".into(), + AMQPValue::LongUInt(backoff_ms as u32), + ); + retry_queue_args.insert( + "x-dead-letter-exchange".into(), + AMQPValue::LongString(names.exchange().into()), + ); + retry_queue_args.insert( + "x-dead-letter-routing-key".into(), + AMQPValue::LongString(queue_name.into()), + ); + with_max_priority(&mut retry_queue_args, max_priority); + + self.channel + .queue_declare( + &names.retry_queue(), + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + retry_queue_args, + ) + .await?; + + self.channel + .queue_bind( + &names.retry_queue(), + &names.retry_exchange(), + queue_name, + QueueBindOptions::default(), + FieldTable::default(), + ) + .await?; + + // DLQ exchange + queue + self.channel + .exchange_declare( + &names.dlq_exchange(), + lapin::ExchangeKind::Direct, + ExchangeDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await?; + + self.channel + .queue_declare( + &names.dlq(), + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await?; + + self.channel + .queue_bind( + &names.dlq(), + &names.dlq_exchange(), + queue_name, + QueueBindOptions::default(), + FieldTable::default(), + ) + .await?; + + tracing::debug!(queue = %queue_name, "Function queue RabbitMQ topology setup complete"); + Ok(()) + } +} + +/// Insert the `x-max-priority` declaration argument when a priority level is +/// configured, turning the queue into a RabbitMQ priority queue. Declared as a +/// signed integer (`AMQPValue::LongInt`) to match the canonical client encoding; +/// the value is fixed at queue-creation time and cannot be changed later. +fn with_max_priority(args: &mut FieldTable, max_priority: Option) { + if let Some(max) = max_priority { + args.insert("x-max-priority".into(), AMQPValue::LongInt(max as i32)); + } +} diff --git a/queue/src/adapters/rabbitmq/worker.rs b/queue/src/adapters/rabbitmq/worker.rs new file mode 100644 index 000000000..7596a91d5 --- /dev/null +++ b/queue/src/adapters/rabbitmq/worker.rs @@ -0,0 +1,286 @@ +//! Per-subscriber consumer loop: pulls deliveries off a function's queue, +//! invokes the target function (through an optional condition-function +//! gate), and hands the result to the [`RetryHandler`] for ack/requeue/DLQ. +//! +//! Port of `engine/src/workers/queue/adapters/rabbitmq/worker.rs`. Seams +//! applied for the standalone worker: +//! - `Arc` and `engine.call(...)` become `Arc` and `invoker.call(function_id, payload)` +//! (same seam as every other adapter in this crate, e.g. +//! `adapters/redis.rs`). +//! - The engine's `condition::check_condition` helper and its telemetry span +//! (`tracing::info_span!` + OTel `SpanExt::with_parent_headers` + +//! `Instrument`) are dropped; `check_condition` is reimplemented locally +//! against `Invoker` with identical semantics (same duplication pattern as +//! `adapters/redis.rs::check_condition`) and the worker emits plain +//! `tracing` events instead of a parented span. + +#![cfg(feature = "rabbitmq")] + +use std::sync::Arc; + +use futures::StreamExt; +use lapin::{message::Delivery, options::*, Channel}; +use serde_json::Value; +use tokio::sync::Semaphore; + +use crate::trigger::Invoker; + +use super::consumer::JobParser; +use super::retry::RetryHandler; +use super::types::{Job, QueueMode}; + +/// Port of the engine's `check_condition` (`engine/src/condition.rs`) against +/// `Invoker` instead of `EngineTrait` — same semantics as +/// `adapters/redis.rs::check_condition`: `Ok(Some(v))` proceeds unless `v` is +/// the JSON boolean `false`, `Ok(None)` proceeds (with a warn log), `Err` +/// propagates the invocation failure. +async fn check_condition( + invoker: &dyn Invoker, + condition_function_id: &str, + data: Value, +) -> Result { + match invoker.call(condition_function_id, data).await { + Ok(Some(result)) => Ok(result.as_bool() != Some(false)), + Ok(None) => { + tracing::warn!( + condition_function_id = %condition_function_id, + "Condition function returned no result" + ); + Ok(true) + } + Err(e) => Err(e), + } +} + +pub struct Worker { + channel: Arc, + retry_handler: Arc, + invoker: Arc, + semaphore: Option>, + queue_mode: QueueMode, + prefetch_count: u16, +} + +impl Worker { + pub fn new( + channel: Arc, + retry_handler: Arc, + invoker: Arc, + queue_mode: QueueMode, + prefetch_count: u16, + ) -> Self { + let semaphore = match queue_mode { + QueueMode::Fifo => None, + QueueMode::Standard => Some(Arc::new(Semaphore::new(prefetch_count as usize))), + }; + + Self { + channel, + retry_handler, + invoker, + semaphore, + queue_mode, + prefetch_count, + } + } + + pub async fn run( + self: Arc, + topic: String, + function_id: String, + condition_function_id: Option, + consumer_tag: String, + queue_name: String, + ) { + // Apply per-consumer prefetch (QoS) so the broker keeps a backlog in the + // queue instead of shipping everything to this consumer at once. Without + // it a priority subscriber queue can't reorder (the broker only orders + // messages it has yet to deliver), and the queue loses backpressure. The + // function-queue consumer sets QoS the same way. + if let Err(e) = self + .channel + .basic_qos(self.prefetch_count, BasicQosOptions::default()) + .await + { + tracing::error!(topic = %topic, error = ?e, "Failed to set consumer QoS"); + return; + } + + let mut consumer = match self + .channel + .basic_consume( + &queue_name, + &consumer_tag, + BasicConsumeOptions::default(), + lapin::types::FieldTable::default(), + ) + .await + { + Ok(consumer) => consumer, + Err(e) => { + tracing::error!( + topic = %topic, + error = ?e, + "Failed to create consumer" + ); + return; + } + }; + + while let Some(delivery_result) = consumer.next().await { + match delivery_result { + Ok(delivery) => { + let worker = Arc::clone(&self); + let topic_clone = topic.clone(); + let function_id_clone = function_id.clone(); + let condition_function_id_clone = condition_function_id.clone(); + + match self.queue_mode { + QueueMode::Fifo => { + if let Err(e) = worker + .process_delivery( + delivery, + &topic_clone, + &function_id_clone, + condition_function_id_clone.as_deref(), + ) + .await + { + tracing::error!( + topic = %topic_clone, + error = ?e, + "Failed to process delivery" + ); + } + } + QueueMode::Standard => { + let semaphore = self.semaphore.as_ref().map(Arc::clone); + tokio::spawn(async move { + let _permit = if let Some(ref sem) = semaphore { + Some(sem.acquire().await.unwrap()) + } else { + None + }; + + if let Err(e) = worker + .process_delivery( + delivery, + &topic_clone, + &function_id_clone, + condition_function_id_clone.as_deref(), + ) + .await + { + tracing::error!( + topic = %topic_clone, + error = ?e, + "Failed to process delivery" + ); + } + }); + } + } + } + Err(e) => { + tracing::error!( + topic = %topic, + error = ?e, + "Error receiving delivery" + ); + } + } + } + + tracing::warn!(topic = %topic, "Consumer stream ended"); + } + + async fn process_delivery( + &self, + delivery: Delivery, + topic: &str, + function_id: &str, + condition_function_id: Option<&str>, + ) -> Result<(), Box> { + let mut job = JobParser::parse_from_delivery(&delivery)?; + + match self + .process_job(&job, function_id, condition_function_id) + .await + { + Ok(_) => { + delivery + .ack(BasicAckOptions::default()) + .await + .map_err(|e| format!("Failed to ack message: {}", e))?; + + self.retry_handler + .handle_success(topic, &job) + .await + .map_err(|e| format!("Failed to handle success: {}", e))?; + } + Err(e) => { + delivery + .nack(BasicNackOptions { + requeue: false, + ..Default::default() + }) + .await + .map_err(|e| format!("Failed to nack message: {}", e))?; + + self.retry_handler + .handle_failure(topic, &mut job, &format!("{:?}", e), Some(function_id)) + .await + .map_err(|e| format!("Failed to handle failure: {}", e))?; + } + } + + Ok(()) + } + + async fn process_job( + &self, + job: &Job, + function_id: &str, + condition_function_id: Option<&str>, + ) -> Result<(), Box> { + let invoker = Arc::clone(&self.invoker); + let data = job.data.clone(); + + if let Some(condition_path) = condition_function_id { + tracing::debug!( + condition_function_id = %condition_path, + "Checking trigger conditions" + ); + match check_condition(invoker.as_ref(), condition_path, data.clone()).await { + Ok(true) => {} + Ok(false) => { + tracing::debug!( + function_id = %function_id, + "Condition check failed, skipping handler" + ); + return Ok(()); + } + Err(err) => { + tracing::error!( + condition_function_id = %condition_path, + error = %err, + "Error invoking condition function" + ); + return Err(format!("Condition function error: {:?}", err).into()); + } + } + } + + match invoker.call(function_id, data).await { + Ok(_) => { + tracing::debug!(job_id = %job.id, "Job processed successfully"); + Ok(()) + } + Err(e) => { + tracing::error!(job_id = %job.id, error = ?e, "Job processing failed"); + Err(format!("Job processing error: {:?}", e).into()) + } + } + } +} From 8a352787c009bdea242a95d56c7a3787f54c5610 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Tue, 7 Jul 2026 06:23:36 -0300 Subject: [PATCH 24/28] feat(queue): rabbitmq adapter with retry, DLQ, priority parity Complete the RabbitMQ transport adapter port (types/naming from the first commit; topology/retry/publisher/consumer/worker/adapter from the second) with e2e coverage: - queue/tests/common/docker.rs: add a rabbitmq:3-alpine container starter (start_rabbitmq), generalizing the existing redis port-resolution helper. Readiness is polled via a real lapin AMQP connect/close loop (RabbitMQ opens its TCP port well before the AMQP handshake is actually served), not a bare TCP check. - queue/tests/e2e_rabbitmq.rs: four connect-or-skip scenarios against a live iii engine + docker RabbitMQ -- basic delivery; priority ordering (3 messages published highest-priority-last, pre-declared+pre-published before any consumer attaches, delivered 9/5/1); fifo mode (10 messages delivered in publish order); and retry-then-DLQ-then-redrive exercised through the function-queue trait methods directly (setup/publish/consume/ ack/nack_function_queue), since `resolve_dlq_name` for a bare (non-`__fn_queue::`) topic resolves to a queue name `subscribe()` never actually declares -- a mismatch inherited verbatim from the engine (its own rabbitmq_queue_integration.rs test suite never exercises DLQ ops against a bare subscribed topic either). All gates pass: cargo test (default + --no-default-features), cargo fmt --check, cargo clippy --all-targets (default + --no-default-features) -D warnings. e2e_rabbitmq's 4 scenarios verified passing against a real `iii` engine and a real rabbitmq:3-alpine container, not just compiling. --- queue/tests/common/docker.rs | 123 ++++++++- queue/tests/e2e_rabbitmq.rs | 500 +++++++++++++++++++++++++++++++++++ 2 files changed, 609 insertions(+), 14 deletions(-) create mode 100644 queue/tests/e2e_rabbitmq.rs diff --git a/queue/tests/common/docker.rs b/queue/tests/common/docker.rs index 56f54a95f..3ac8a07d8 100644 --- a/queue/tests/common/docker.rs +++ b/queue/tests/common/docker.rs @@ -1,12 +1,13 @@ #![allow(dead_code)] -//! Docker-backed Redis container for the redis adapter e2e test. +//! Docker-backed Redis/RabbitMQ containers for the redis/rabbitmq adapter +//! e2e tests. //! //! Skips (returns `None`) whenever `docker info` fails — no Docker daemon //! reachable — so CI (which runs without Docker) and casual local runs stay -//! green. The container is started with `-p 0:6379` (an ephemeral host -//! port, so parallel test runs never collide on a fixed port) and torn down -//! via `Drop` (`docker stop`, which also removes it since it's started -//! with `--rm`). +//! green. Each container is started with an ephemeral host port (`-p 0:`, +//! so parallel test runs never collide on a fixed port) and torn down via +//! `Drop` (`docker stop`, which also removes it since it's started with +//! `--rm`). use std::process::Command; @@ -66,14 +67,14 @@ pub fn start_redis() -> Option { return None; } - let host_port = resolve_mapped_port(&container_id)?; + let host_port = resolve_mapped_port(&container_id, "6379/tcp")?; let container = RedisContainer { container_id, host_port, }; - if !wait_until_ready(&container) { + if !wait_until_tcp_ready(container.host_port) { eprintln!("[skip] redis container did not become ready in time"); return None; } @@ -81,12 +82,12 @@ pub fn start_redis() -> Option { Some(container) } -fn resolve_mapped_port(container_id: &str) -> Option { - // `docker port 6379/tcp` prints e.g. `0.0.0.0:54321` (and often a - // second `[::]:54321` line for the IPv6 binding) — take the first - // line's port. +/// `docker port ` prints e.g. `0.0.0.0:54321` (and often a +/// second `[::]:54321` line for the IPv6 binding) — take the first line's +/// port. +fn resolve_mapped_port(container_id: &str, port_spec: &str) -> Option { let port_output = Command::new("docker") - .args(["port", container_id, "6379/tcp"]) + .args(["port", container_id, port_spec]) .output() .ok()?; if !port_output.status.success() { @@ -98,11 +99,11 @@ fn resolve_mapped_port(container_id: &str) -> Option { port_str.trim().parse::().ok() } -fn wait_until_ready(container: &RedisContainer) -> bool { +fn wait_until_tcp_ready(host_port: u16) -> bool { use std::net::TcpStream; use std::time::{Duration, Instant}; - let addr = format!("127.0.0.1:{}", container.host_port); + let addr = format!("127.0.0.1:{}", host_port); let deadline = Instant::now() + Duration::from_secs(10); while Instant::now() < deadline { if TcpStream::connect(&addr).is_ok() { @@ -112,3 +113,97 @@ fn wait_until_ready(container: &RedisContainer) -> bool { } false } + +/// Docker-backed RabbitMQ container for the rabbitmq adapter e2e tests. +/// +/// RabbitMQ boots noticeably slower than Redis (~10-20s): the TCP port opens +/// well before the AMQP protocol handshake is actually served, so readiness +/// is checked with a real `lapin::Connection::connect` retry loop instead of +/// a bare TCP connect (see `wait_until_tcp_ready`, used for Redis). +#[cfg(feature = "rabbitmq")] +pub struct RabbitMqContainer { + container_id: String, + pub host_port: u16, +} + +#[cfg(feature = "rabbitmq")] +impl RabbitMqContainer { + pub fn amqp_url(&self) -> String { + format!("amqp://127.0.0.1:{}/%2f", self.host_port) + } +} + +#[cfg(feature = "rabbitmq")] +impl Drop for RabbitMqContainer { + fn drop(&mut self) { + let _ = Command::new("docker") + .args(["stop", "-t", "1", &self.container_id]) + .output(); + } +} + +/// Starts a `rabbitmq:3-alpine` container on an ephemeral host port. Returns +/// `None` (the caller should skip the test) when Docker isn't reachable or +/// the container fails to start/become ready within the deadline. +#[cfg(feature = "rabbitmq")] +pub async fn start_rabbitmq() -> Option { + if !docker_available() { + eprintln!("[skip] docker not reachable — skipping rabbitmq adapter e2e test"); + return None; + } + + let run_output = Command::new("docker") + .args(["run", "-d", "--rm", "-p", "0:5672", "rabbitmq:3-alpine"]) + .output() + .ok()?; + + if !run_output.status.success() { + eprintln!( + "[skip] `docker run` failed: {}", + String::from_utf8_lossy(&run_output.stderr) + ); + return None; + } + + let container_id = String::from_utf8_lossy(&run_output.stdout) + .trim() + .to_string(); + if container_id.is_empty() { + return None; + } + + let host_port = resolve_mapped_port(&container_id, "5672/tcp")?; + let container = RabbitMqContainer { + container_id, + host_port, + }; + + if !wait_until_amqp_ready(&container).await { + eprintln!("[skip] rabbitmq container did not become ready in time"); + return None; + } + + Some(container) +} + +#[cfg(feature = "rabbitmq")] +async fn wait_until_amqp_ready(container: &RabbitMqContainer) -> bool { + use tokio::time::{sleep, Duration, Instant}; + + let deadline = Instant::now() + Duration::from_secs(20); + while Instant::now() < deadline { + match lapin::Connection::connect( + &container.amqp_url(), + lapin::ConnectionProperties::default(), + ) + .await + { + Ok(connection) => { + let _ = connection.close(200, "readiness check").await; + return true; + } + Err(_) => sleep(Duration::from_millis(300)).await, + } + } + false +} diff --git a/queue/tests/e2e_rabbitmq.rs b/queue/tests/e2e_rabbitmq.rs new file mode 100644 index 000000000..17f48aebe --- /dev/null +++ b/queue/tests/e2e_rabbitmq.rs @@ -0,0 +1,500 @@ +#![cfg(feature = "rabbitmq")] + +mod common; + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use iii_queue::adapter::{FunctionQueueConfig, QueueAdapter}; +use iii_queue::adapters::rabbitmq::RabbitMQAdapter; +use iii_queue::subscriber_config::SubscriberQueueConfig; +use iii_queue::trigger::{IiiInvoker, Invoker}; +use iii_sdk::errors::Error; +use iii_sdk::RegisterFunction; +use lapin::options::{ExchangeDeclareOptions, QueueBindOptions, QueueDeclareOptions}; +use lapin::types::{AMQPValue, FieldTable}; +use lapin::{Channel, Connection, ConnectionProperties, ExchangeKind}; +use serde_json::{json, Value}; +use serial_test::serial; +use tokio::sync::Mutex; +use tokio::time::Instant; +use uuid::Uuid; + +use common::docker; +use common::engine; + +/// Polls `pred` until it returns `true` or `timeout` elapses, then asserts +/// once more (for a useful panic message) -- same pattern as this crate's +/// `adapters::builtin` unit tests and `e2e_redis.rs`. +async fn wait_until(mut pred: F, timeout: Duration) +where + F: FnMut() -> Fut, + Fut: std::future::Future, +{ + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if pred().await { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!(pred().await, "condition did not become true before timeout"); +} + +// -- Naming helpers mirroring `adapters::rabbitmq::naming::RabbitNames` +// (private to the crate) -- needed here only to pre-declare a subscriber's +// topology by hand for the priority test, see its comment below. +fn exchange_name(topic: &str) -> String { + format!("iii.{topic}.exchange") +} +fn function_queue_name(topic: &str, function_id: &str) -> String { + format!("iii.{topic}.{function_id}.queue") +} +fn function_dlq_name(topic: &str, function_id: &str) -> String { + format!("iii.{topic}.{function_id}.dlq") +} + +/// Declares the exact same topology `RabbitMQAdapter::subscribe` would +/// declare for `(topic, function_id)` -- fanout exchange, per-function DLQ, +/// per-function priority queue (`x-max-priority`), and the binding -- via a +/// raw `lapin` channel, so messages can be published into the queue BEFORE +/// any adapter-owned consumer exists. All declarations use identical +/// arguments to what `topology::TopologyManager` declares, so the later, +/// real `subscribe()` call's redeclare is a idempotent no-op (AMQP rejects a +/// redeclare with mismatched arguments). +async fn predeclare_priority_subscriber_queue( + channel: &Channel, + topic: &str, + function_id: &str, + max_priority: i32, +) { + channel + .exchange_declare( + &exchange_name(topic), + ExchangeKind::Fanout, + ExchangeDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .expect("declare fanout exchange"); + + channel + .queue_declare( + &function_dlq_name(topic, function_id), + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .expect("declare dlq"); + + let mut args = FieldTable::default(); + args.insert("x-max-priority".into(), AMQPValue::LongInt(max_priority)); + channel + .queue_declare( + &function_queue_name(topic, function_id), + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + args, + ) + .await + .expect("declare priority queue"); + + channel + .queue_bind( + &function_queue_name(topic, function_id), + &exchange_name(topic), + "", + QueueBindOptions::default(), + FieldTable::default(), + ) + .await + .expect("bind priority queue"); +} + +/// Never actually invoked -- used for the function-queue DLQ/redrive test, +/// which drives ack/nack directly through the `QueueAdapter` trait instead +/// of going through a live engine + registered function. +struct NoopInvoker; + +#[async_trait] +impl Invoker for NoopInvoker { + async fn call(&self, _function_id: &str, _payload: Value) -> Result, String> { + panic!("NoopInvoker::call should never be invoked in this test") + } +} + +/// (a) Basic delivery: subscribe, enqueue, the registered function receives +/// the raw data. +#[tokio::test] +#[serial] +async fn basic_delivery_connect_or_skip() { + let Some(container) = docker::start_rabbitmq().await else { + return; // skip: docker not reachable + }; + let Some(iii) = engine::connect_fresh().await else { + return; // skip: engine not reachable + }; + + let invoker = Arc::new(IiiInvoker::new(iii.clone())); + let adapter = + RabbitMQAdapter::from_config(Some(&json!({"amqp_url": container.amqp_url()})), invoker) + .await + .expect("rabbitmq adapter should connect"); + + let seen = Arc::new(Mutex::new(Vec::::new())); + let function_id = format!("queue.e2e.rabbitmq.basic.{}", Uuid::new_v4()); + { + let seen = seen.clone(); + iii.register_function( + function_id.as_str(), + RegisterFunction::new_async(move |payload: Value| { + let seen = seen.clone(); + async move { + seen.lock().await.push(payload); + Ok::(json!({"ok": true})) + } + }), + ); + } + + let topic = format!("e2e-rmq-basic-{}", Uuid::new_v4()); + adapter + .subscribe(&topic, "sub-1", &function_id, None, None) + .await; + // Give the consumer task a beat to actually attach before the first + // publish. + tokio::time::sleep(Duration::from_millis(300)).await; + + adapter + .enqueue(&topic, json!({"hello": "world"}), None, None) + .await; + + wait_until( + || { + let seen = seen.clone(); + async move { !seen.lock().await.is_empty() } + }, + Duration::from_secs(10), + ) + .await; + + let seen = seen.lock().await; + assert_eq!(seen[0].get("hello"), Some(&json!("world"))); + drop(seen); + + adapter.unsubscribe(&topic, "sub-1").await; + adapter.shutdown().await; + iii.shutdown_async().await; +} + +/// (b) Failing handler retries per `max_attempts`, lands in the DLQ, and +/// `redrive_dlq` redelivers it so it can succeed. +/// +/// Exercised through the function-queue trait methods +/// (`setup_function_queue`/`publish_to_function_queue`/ +/// `consume_function_queue`/`ack_function_queue`/`nack_function_queue`) +/// rather than through `subscribe`'s topic-fanout path. Discovered while +/// writing this test: `RabbitMQAdapter::resolve_dlq_name` (and therefore +/// `dlq_count`/`redrive_dlq`/`redrive_dlq_message`/`discard_dlq_message`) +/// resolves a bare (non-`__fn_queue::`) topic to `RabbitNames::dlq()` +/// (`iii.{topic}.dlq`) -- a queue name that `subscribe`'s +/// `setup_subscriber_queue` never actually declares (it declares +/// `RabbitNames::function_dlq(function_id)`, i.e. +/// `iii.{topic}.{function_id}.dlq`, a *different* name). That mismatch is +/// inherited verbatim from the engine +/// (`engine/src/workers/queue/adapters/rabbitmq/adapter.rs`) -- the engine's +/// own `rabbitmq_queue_integration.rs` test suite never calls +/// `dlq_count`/`redrive_dlq` against a bare subscribed topic either (only +/// against `__fn_queue::`-prefixed function-queue names, which resolve +/// correctly). A passive `queue_declare` against a queue that doesn't exist +/// is a channel-level AMQP error, which would poison this adapter's shared +/// channel for every other operation -- so this test deliberately exercises +/// the function-queue path, which resolves correctly end-to-end, instead of +/// tripping that pre-existing engine gap. +#[tokio::test] +#[serial] +async fn function_queue_retry_then_dlq_then_redrive_connect_or_skip() { + let Some(container) = docker::start_rabbitmq().await else { + return; // skip: docker not reachable + }; + + let invoker: Arc = Arc::new(NoopInvoker); + let adapter = + RabbitMQAdapter::from_config(Some(&json!({"amqp_url": container.amqp_url()})), invoker) + .await + .expect("rabbitmq adapter should connect"); + + let queue_name = format!("e2e-rmq-fnq-{}", Uuid::new_v4()); + let dlq_topic = format!("__fn_queue::{queue_name}"); + + adapter + .setup_function_queue( + &queue_name, + &FunctionQueueConfig { + max_retries: 1, + concurrency: 1, + backoff_ms: 200, + max_priority: None, + }, + ) + .await + .expect("setup_function_queue should succeed"); + + adapter + .publish_to_function_queue( + &queue_name, + "target-fn", + json!({"n": 1}), + "msg-1", + 1, + 200, + None, + None, + None, + ) + .await; + + let mut rx = adapter + .consume_function_queue(&queue_name, 10) + .await + .expect("consume_function_queue should succeed"); + + let msg1 = tokio::time::timeout(Duration::from_secs(5), rx.recv()) + .await + .expect("first delivery should arrive") + .expect("channel should stay open"); + assert_eq!(msg1.attempt, 0); + adapter + .nack_function_queue(&queue_name, msg1.delivery_id, msg1.attempt, 1) + .await + .expect("nack (retry) should succeed"); + + let msg2 = tokio::time::timeout(Duration::from_secs(5), rx.recv()) + .await + .expect("retried delivery should arrive") + .expect("channel should stay open"); + assert_eq!(msg2.attempt, 1, "attempt should be incremented on retry"); + adapter + .nack_function_queue(&queue_name, msg2.delivery_id, msg2.attempt, 1) + .await + .expect("nack (exhausted) should succeed"); + + wait_until( + || { + let adapter = &adapter; + let dlq_topic = dlq_topic.clone(); + async move { adapter.dlq_count(&dlq_topic).await.unwrap_or(0) >= 1 } + }, + Duration::from_secs(5), + ) + .await; + assert_eq!(adapter.dlq_count(&dlq_topic).await.unwrap(), 1); + + let redriven = adapter + .redrive_dlq(&dlq_topic) + .await + .expect("redrive_dlq should succeed"); + assert_eq!(redriven, 1); + assert_eq!(adapter.dlq_count(&dlq_topic).await.unwrap(), 0); + + let msg3 = tokio::time::timeout(Duration::from_secs(5), rx.recv()) + .await + .expect("redriven delivery should arrive") + .expect("channel should stay open"); + assert_eq!(msg3.attempt, 0, "redrive resets the attempt counter"); + adapter + .ack_function_queue(&queue_name, msg3.delivery_id) + .await + .expect("ack should succeed"); + + adapter.shutdown().await; +} + +/// (c) Priority ordering: 3 messages, published with `priority_field` +/// values 1, 9, 5 -- in that order -- onto a subscriber queue declared with +/// `max_priority: 10`, all BEFORE any consumer attaches (the topology is +/// pre-declared by hand via a raw `lapin` channel so the fanout-published +/// messages have somewhere to land). `subscribe()` is only called after all +/// three are confirmed-published, so the very first delivery already +/// reflects priority order: 9, 5, 1. +#[tokio::test] +#[serial] +async fn priority_ordering_connect_or_skip() { + let Some(container) = docker::start_rabbitmq().await else { + return; // skip: docker not reachable + }; + let Some(iii) = engine::connect_fresh().await else { + return; // skip: engine not reachable + }; + + let invoker = Arc::new(IiiInvoker::new(iii.clone())); + let adapter = RabbitMQAdapter::from_config( + Some(&json!({"amqp_url": container.amqp_url(), "priority_field": "priority"})), + invoker, + ) + .await + .expect("rabbitmq adapter should connect"); + + let function_id = format!("queue.e2e.rabbitmq.priority.{}", Uuid::new_v4()); + let order = Arc::new(Mutex::new(Vec::::new())); + { + let order = order.clone(); + iii.register_function( + function_id.as_str(), + RegisterFunction::new_async(move |payload: Value| { + let order = order.clone(); + async move { + let marker = payload.get("marker").and_then(|v| v.as_u64()).unwrap_or(0); + order.lock().await.push(marker); + Ok::(json!({"ok": true})) + } + }), + ); + } + + let topic = format!("e2e-rmq-priority-{}", Uuid::new_v4()); + let sub_id = "sub-priority-1"; + + let raw_connection = + Connection::connect(&container.amqp_url(), ConnectionProperties::default()) + .await + .expect("raw amqp connection"); + let raw_channel = raw_connection + .create_channel() + .await + .expect("raw amqp channel"); + predeclare_priority_subscriber_queue(&raw_channel, &topic, &function_id, 10).await; + + for p in [1u64, 9, 5] { + adapter + .enqueue(&topic, json!({"marker": p, "priority": p}), None, None) + .await; + } + + // No consumer exists yet at this point -- all three publishes above were + // already broker-confirmed (`Publisher::publish` awaits the publish + // confirm) before `subscribe` is called below. + adapter + .subscribe( + &topic, + sub_id, + &function_id, + None, + Some(SubscriberQueueConfig { + max_priority: Some(10), + concurrency: Some(1), + ..Default::default() + }), + ) + .await; + + wait_until( + || { + let order = order.clone(); + async move { order.lock().await.len() >= 3 } + }, + Duration::from_secs(10), + ) + .await; + + assert_eq!( + *order.lock().await, + vec![9, 5, 1], + "priority queue should drain highest-priority-first" + ); + + adapter.unsubscribe(&topic, sub_id).await; + adapter.shutdown().await; + let _ = raw_connection.close(200, "test done").await; + iii.shutdown_async().await; +} + +/// (d) Fifo mode: 10 messages published in order are delivered in the same +/// order. Handler sleeps a small jittered delay so a broken fifo consumer +/// (e.g. one that spawns handlers concurrently instead of processing them +/// one at a time) would visibly scramble the order. +#[tokio::test] +#[serial] +async fn fifo_mode_preserves_order_connect_or_skip() { + let Some(container) = docker::start_rabbitmq().await else { + return; // skip: docker not reachable + }; + let Some(iii) = engine::connect_fresh().await else { + return; // skip: engine not reachable + }; + + let invoker = Arc::new(IiiInvoker::new(iii.clone())); + let adapter = RabbitMQAdapter::from_config( + Some(&json!({"amqp_url": container.amqp_url(), "queue_mode": "fifo"})), + invoker, + ) + .await + .expect("rabbitmq adapter should connect"); + + let function_id = format!("queue.e2e.rabbitmq.fifo.{}", Uuid::new_v4()); + let order = Arc::new(Mutex::new(Vec::::new())); + { + let order = order.clone(); + iii.register_function( + function_id.as_str(), + RegisterFunction::new_async(move |payload: Value| { + let order = order.clone(); + async move { + let n = payload.get("n").and_then(|v| v.as_u64()).unwrap_or(0); + tokio::time::sleep(Duration::from_millis((n % 3) * 5)).await; + order.lock().await.push(n); + Ok::(json!({"ok": true})) + } + }), + ); + } + + let topic = format!("e2e-rmq-fifo-{}", Uuid::new_v4()); + adapter + .subscribe( + &topic, + "sub-fifo-1", + &function_id, + None, + Some(SubscriberQueueConfig { + queue_mode: Some("fifo".to_string()), + concurrency: Some(1), + ..Default::default() + }), + ) + .await; + tokio::time::sleep(Duration::from_millis(300)).await; + + for n in 0..10u64 { + adapter.enqueue(&topic, json!({"n": n}), None, None).await; + } + + wait_until( + || { + let order = order.clone(); + async move { order.lock().await.len() >= 10 } + }, + Duration::from_secs(15), + ) + .await; + + let expected: Vec = (0..10).collect(); + assert_eq!( + *order.lock().await, + expected, + "fifo mode should preserve publish order" + ); + + adapter.unsubscribe(&topic, "sub-fifo-1").await; + adapter.shutdown().await; + iii.shutdown_async().await; +} From 6a775fc8d868bef8473bd5c70f31c96ce04398a7 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Tue, 7 Jul 2026 08:46:58 -0300 Subject: [PATCH 25/28] feat(queue): adapter factory, memory test adapter, hot-swap re-subscribe --- queue/Cargo.lock | 1 + queue/Cargo.toml | 9 ++ queue/src/adapter.rs | 31 ++++- queue/src/adapters/memory.rs | 222 ++++++++++++++++++++++++++++++++ queue/src/adapters/mod.rs | 2 + queue/src/boot.rs | 221 +++++++++++++++++++++++++++++++- queue/src/config.rs | 54 ++++++++ queue/src/configuration.rs | 242 +++++++++++++++++++++++++++++++---- queue/src/functions.rs | 11 +- queue/src/trigger.rs | 79 +++++++++++- 10 files changed, 834 insertions(+), 38 deletions(-) create mode 100644 queue/src/adapters/memory.rs diff --git a/queue/Cargo.lock b/queue/Cargo.lock index b8556a6e5..ace37ddcc 100644 --- a/queue/Cargo.lock +++ b/queue/Cargo.lock @@ -1244,6 +1244,7 @@ dependencies = [ "async-trait", "clap", "futures", + "iii-queue", "iii-sdk", "lapin", "redis", diff --git a/queue/Cargo.toml b/queue/Cargo.toml index 1cb61dee7..3cd262736 100644 --- a/queue/Cargo.toml +++ b/queue/Cargo.toml @@ -20,6 +20,11 @@ path = "src/main.rs" [features] default = ["rabbitmq"] rabbitmq = ["dep:lapin"] +# In-process test-only transport (`adapters::memory::MemoryAdapter`), +# registered under the adapter name "memory". Off in normal builds; turned on +# for this crate's own test targets via the dev-dependency self-reference +# below (mirrors `engine/Cargo.toml:168-171`). +test-adapters = [] [dependencies] iii-sdk = "=0.20.0" @@ -40,3 +45,7 @@ lapin = { version = "3", optional = true } [dev-dependencies] serial_test = "3" +# Self-dependency trick: turns on `test-adapters` for this crate's own test +# targets (the `memory` adapter the config hot-swap tests swap to) without +# enabling it for a normal build. Mirrors `engine/Cargo.toml:168-171`. +iii-queue = { path = ".", features = ["test-adapters"] } diff --git a/queue/src/adapter.rs b/queue/src/adapter.rs index 92a2ac4e3..9d2ae507c 100644 --- a/queue/src/adapter.rs +++ b/queue/src/adapter.rs @@ -229,12 +229,19 @@ pub trait QueueAdapter: Send + Sync + 'static { #[derive(Clone)] pub struct SwappableAdapter { inner: Arc>>, + /// The active adapter's identity (e.g. `"builtin"`, `"redis"`, + /// `"rabbitmq"`), set by the factory (`crate::boot::build_adapter`'s + /// caller) at construction/replace time. Exposed via [`Self::current_name`] + /// for service functions (`list_topics`) that report which transport + /// backs a topic. + name: Arc>, } impl SwappableAdapter { - pub fn new(adapter: Arc) -> Self { + pub fn new(adapter: Arc, name: impl Into) -> Self { Self { inner: Arc::new(RwLock::new(adapter)), + name: Arc::new(RwLock::new(name.into())), } } @@ -242,8 +249,13 @@ impl SwappableAdapter { self.inner.read().await.clone() } - pub async fn replace(&self, adapter: Arc) { + pub async fn current_name(&self) -> String { + self.name.read().await.clone() + } + + pub async fn replace(&self, adapter: Arc, name: impl Into) { *self.inner.write().await = adapter; + *self.name.write().await = name.into(); } } @@ -457,15 +469,26 @@ mod tests { #[tokio::test] async fn current_reflects_replace() { let first = Arc::new(CountingAdapter::default()); - let swappable = SwappableAdapter::new(first.clone()); + let swappable = SwappableAdapter::new(first.clone(), "first"); swappable.enqueue("demo", Value::Null, None, None).await; assert_eq!(first.enqueue_calls.load(Ordering::SeqCst), 1); let second = Arc::new(CountingAdapter::default()); - swappable.replace(second.clone()).await; + swappable.replace(second.clone(), "second").await; swappable.enqueue("demo", Value::Null, None, None).await; assert_eq!(first.enqueue_calls.load(Ordering::SeqCst), 1); assert_eq!(second.enqueue_calls.load(Ordering::SeqCst), 1); } + + #[tokio::test] + async fn current_name_reflects_replace() { + let first = Arc::new(CountingAdapter::default()); + let swappable = SwappableAdapter::new(first, "builtin"); + assert_eq!(swappable.current_name().await, "builtin"); + + let second = Arc::new(CountingAdapter::default()); + swappable.replace(second, "redis").await; + assert_eq!(swappable.current_name().await, "redis"); + } } diff --git a/queue/src/adapters/memory.rs b/queue/src/adapters/memory.rs new file mode 100644 index 000000000..62421e8a4 --- /dev/null +++ b/queue/src/adapters/memory.rs @@ -0,0 +1,222 @@ +//! In-process test-only transport, registered under the adapter name +//! `"memory"`. +//! +//! Ported from the engine builtin's `memory_adapter` +//! (`engine/src/workers/queue/adapters/memory_adapter.rs`): the engine's +//! version is mechanically just `builtin::make_adapter` re-registered under +//! a second name so the config hot-swap test suite can swap to a *second* +//! dependency-free backend (the real alternatives, `redis`/`rabbitmq`, need a +//! live server). This port keeps that spirit as a distinct type — rather +//! than re-registering the same constructor under two names (this crate's +//! factory dispatches on a `match` arm per adapter, not a name registry) — +//! wrapping a fresh [`BuiltinAdapter`] over its own [`InMemoryStore`] and +//! delegating every [`QueueAdapter`] method straight through. Only compiled +//! for this crate's own test targets (`#[cfg(feature = "test-adapters")]`, +//! enabled via the dev-dependency self-reference in `Cargo.toml`), never for +//! a normal build. + +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::Value; +use tokio::sync::mpsc; + +use crate::adapter::{FunctionQueueConfig, QueueAdapter, QueueMessage, TopicInfo}; +use crate::store::{InMemoryStore, TopicStats}; +use crate::subscriber_config::SubscriberQueueConfig; +use crate::trigger::Invoker; + +use super::builtin::BuiltinAdapter; + +/// In-process transport identical to [`BuiltinAdapter`] over a fresh +/// [`InMemoryStore`], registered under its own adapter name so hot-swap +/// tests can exercise a swap between two dependency-free backends. +pub struct MemoryAdapter { + inner: BuiltinAdapter, +} + +impl MemoryAdapter { + pub fn new(invoker: Arc) -> Self { + Self { + inner: BuiltinAdapter::new(Arc::new(InMemoryStore::new()), invoker), + } + } +} + +#[async_trait] +impl QueueAdapter for MemoryAdapter { + async fn enqueue( + &self, + topic: &str, + data: Value, + traceparent: Option, + baggage: Option, + ) { + self.inner.enqueue(topic, data, traceparent, baggage).await; + } + + async fn subscribe( + &self, + topic: &str, + id: &str, + function_id: &str, + condition_function_id: Option, + queue_config: Option, + ) { + self.inner + .subscribe(topic, id, function_id, condition_function_id, queue_config) + .await; + } + + async fn unsubscribe(&self, topic: &str, id: &str) { + self.inner.unsubscribe(topic, id).await; + } + + async fn redrive_dlq(&self, topic: &str) -> anyhow::Result { + self.inner.redrive_dlq(topic).await + } + + async fn redrive_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { + self.inner.redrive_dlq_message(topic, message_id).await + } + + async fn discard_dlq_message(&self, topic: &str, message_id: &str) -> anyhow::Result { + self.inner.discard_dlq_message(topic, message_id).await + } + + async fn dlq_count(&self, topic: &str) -> anyhow::Result { + self.inner.dlq_count(topic).await + } + + async fn dlq_peek(&self, topic: &str, offset: u64, limit: u64) -> anyhow::Result> { + self.inner.dlq_peek(topic, offset, limit).await + } + + async fn list_topics(&self) -> anyhow::Result> { + self.inner.list_topics().await + } + + async fn topic_stats(&self, topic: &str) -> anyhow::Result { + self.inner.topic_stats(topic).await + } + + async fn shutdown(&self) { + self.inner.shutdown().await; + } + + // -- Function queue transport methods: same defaults as `BuiltinAdapter` + // (unimplemented/no-op) -- delegated for completeness rather than relying + // on the trait's own defaults, so a future override on `BuiltinAdapter` + // is inherited here automatically. + + async fn publish_to_function_queue( + &self, + queue_name: &str, + function_id: &str, + data: Value, + message_id: &str, + max_retries: u32, + backoff_ms: u64, + traceparent: Option, + baggage: Option, + priority: Option, + ) { + self.inner + .publish_to_function_queue( + queue_name, + function_id, + data, + message_id, + max_retries, + backoff_ms, + traceparent, + baggage, + priority, + ) + .await; + } + + async fn setup_function_queue( + &self, + queue_name: &str, + config: &FunctionQueueConfig, + ) -> anyhow::Result<()> { + self.inner.setup_function_queue(queue_name, config).await + } + + async fn consume_function_queue( + &self, + queue_name: &str, + prefetch: u32, + ) -> anyhow::Result> { + self.inner + .consume_function_queue(queue_name, prefetch) + .await + } + + async fn ack_function_queue(&self, queue_name: &str, delivery_id: u64) -> anyhow::Result<()> { + self.inner.ack_function_queue(queue_name, delivery_id).await + } + + async fn nack_function_queue( + &self, + queue_name: &str, + delivery_id: u64, + attempt: u32, + max_retries: u32, + ) -> anyhow::Result<()> { + self.inner + .nack_function_queue(queue_name, delivery_id, attempt, max_retries) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::trigger::Invoker; + use async_trait::async_trait; + use serde_json::json; + use std::time::Duration; + use tokio::time::Instant; + + #[derive(Default)] + struct RecordingInvoker { + calls: tokio::sync::Mutex>, + } + + #[async_trait] + impl Invoker for RecordingInvoker { + async fn call(&self, function_id: &str, payload: Value) -> Result, String> { + self.calls + .lock() + .await + .push((function_id.to_string(), payload)); + Ok(None) + } + } + + #[tokio::test] + async fn enqueue_and_subscribe_deliver_like_builtin() { + let invoker = Arc::new(RecordingInvoker::default()); + let adapter = MemoryAdapter::new(invoker.clone()); + + adapter.subscribe("demo", "sub-1", "fn-1", None, None).await; + adapter + .enqueue("demo", json!({"hello": "world"}), None, None) + .await; + + let deadline = Instant::now() + Duration::from_secs(2); + loop { + if !invoker.calls.lock().await.is_empty() { + break; + } + assert!(Instant::now() < deadline, "message was never delivered"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + + let calls = invoker.calls.lock().await; + assert_eq!(calls[0].0, "fn-1"); + assert_eq!(calls[0].1, json!({"hello": "world"})); + } +} diff --git a/queue/src/adapters/mod.rs b/queue/src/adapters/mod.rs index add64c86e..181aab699 100644 --- a/queue/src/adapters/mod.rs +++ b/queue/src/adapters/mod.rs @@ -1,6 +1,8 @@ //! Transport adapters implementing [`crate::adapter::QueueAdapter`]. pub mod builtin; +#[cfg(feature = "test-adapters")] +pub mod memory; #[cfg(feature = "rabbitmq")] pub mod rabbitmq; pub mod redis; diff --git a/queue/src/boot.rs b/queue/src/boot.rs index 1292d6790..b104ba7a9 100644 --- a/queue/src/boot.rs +++ b/queue/src/boot.rs @@ -7,11 +7,14 @@ use iii_sdk::{IIIClient, RegisterTriggerType}; use serde::Deserialize; use tokio::sync::{Mutex, RwLock}; -use crate::adapter::SwappableAdapter; +use crate::adapter::{QueueAdapter, SwappableAdapter}; use crate::adapters::builtin::BuiltinAdapter; -use crate::config::QueueConfig; +#[cfg(feature = "rabbitmq")] +use crate::adapters::rabbitmq::RabbitMQAdapter; +use crate::adapters::redis::RedisAdapter; +use crate::config::{adapter_config_value, QueueConfig}; use crate::store::{FileStore, InMemoryStore, QueueStore}; -use crate::trigger::{IiiInvoker, QueueTriggerHandler, SubscriberSpec}; +use crate::trigger::{IiiInvoker, Invoker, QueueTriggerHandler, SubscriberSpec}; use crate::TRIGGER_TYPE; const LIST_WORKERS_FUNCTION_ID: &str = "engine::workers::list"; @@ -36,11 +39,12 @@ impl BootHandle { pub async fn start(iii: Arc, config: QueueConfig) -> anyhow::Result { guard_against_builtin_iii_queue(&iii).await?; - let store = build_store(&config).await?; let invoker = Arc::new(IiiInvoker::new(iii.clone())); - let adapter = Arc::new(SwappableAdapter::new(Arc::new(BuiltinAdapter::new( - store, invoker, - )))); + // No fallback to builtin on a bad config: an adapter that fails to build + // at boot (e.g. redis/rabbitmq unreachable) must fail the boot itself, + // matching the engine's `make_adapter` behavior for `iii-queue`. + let built = build_adapter(&config, invoker).await?; + let adapter = Arc::new(SwappableAdapter::new(built, adapter_identity_name(&config))); let config = Arc::new(RwLock::new(Arc::new(config.normalized()))); let apply_lock = Arc::new(Mutex::new(())); @@ -64,6 +68,69 @@ pub async fn start(iii: Arc, config: QueueConfig) -> anyhow::Result, +) -> anyhow::Result> { + let name = config.effective_adapter_name(); + match name { + // Legacy worker aliases keep working: they select the builtin store + // backend (`build_store` already dispatches `store_method` between + // in-memory and file-backed). + "builtin" | "in_memory" | "file_based" => { + let store = build_store(config).await?; + Ok(Arc::new(BuiltinAdapter::new(store, invoker))) + } + "redis" => { + let adapter_cfg = adapter_config_value(config); + let adapter = RedisAdapter::from_config(adapter_cfg.as_ref(), invoker).await?; + Ok(Arc::new(adapter)) + } + #[cfg(feature = "rabbitmq")] + "rabbitmq" => { + let adapter_cfg = adapter_config_value(config); + let adapter = RabbitMQAdapter::from_config(adapter_cfg.as_ref(), invoker).await?; + Ok(Arc::new(adapter)) + } + #[cfg(not(feature = "rabbitmq"))] + "rabbitmq" => { + anyhow::bail!( + "queue adapter 'rabbitmq' requires the standalone queue worker to be built \ + with the `rabbitmq` feature enabled" + ) + } + #[cfg(feature = "test-adapters")] + "memory" => Ok(Arc::new(crate::adapters::memory::MemoryAdapter::new( + invoker, + ))), + other => anyhow::bail!( + "queue adapter '{other}' is not implemented by the standalone queue worker yet" + ), + } +} + +/// The transport identity behind `config`'s effective adapter, for +/// [`SwappableAdapter::new`]/[`SwappableAdapter::replace`]. Distinct from +/// `config.effective_adapter_name()`: the legacy `in_memory`/`file_based` +/// aliases both select the `builtin` transport, so they report as `"builtin"` +/// here too (matching the identity `build_adapter` actually constructs). +pub fn adapter_identity_name(config: &QueueConfig) -> &str { + match config.effective_adapter_name() { + "in_memory" | "file_based" => "builtin", + other => other, + } +} + pub async fn build_store(config: &QueueConfig) -> anyhow::Result> { let adapter_name = config.effective_adapter_name(); if adapter_name != "builtin" && adapter_name != "file_based" && adapter_name != "in_memory" { @@ -228,4 +295,144 @@ mod tests { assert!(dir.join("queue_store.json").exists()); let _ = std::fs::remove_dir_all(dir); } + + struct NoopInvoker; + + #[async_trait::async_trait] + impl Invoker for NoopInvoker { + async fn call( + &self, + _function_id: &str, + _payload: serde_json::Value, + ) -> Result, String> { + Ok(None) + } + } + + fn noop_invoker() -> Arc { + Arc::new(NoopInvoker) + } + + #[tokio::test] + async fn build_adapter_defaults_to_builtin_without_config() { + let adapter = build_adapter(&QueueConfig::default(), noop_invoker()) + .await + .unwrap(); + adapter + .enqueue("demo", json!({"ok": true}), None, None) + .await; + } + + #[tokio::test] + async fn build_adapter_accepts_legacy_in_memory_alias() { + let config = QueueConfig { + adapter: Some(AdapterEntry { + name: "in_memory".to_string(), + config: None, + }), + }; + build_adapter(&config, noop_invoker()).await.unwrap(); + } + + #[tokio::test] + async fn build_adapter_accepts_legacy_file_based_alias() { + let dir = std::env::temp_dir().join(format!("queue_boot_adapter_{}", Uuid::new_v4())); + let config = QueueConfig { + adapter: Some(AdapterEntry { + name: "file_based".to_string(), + config: Some(json!({ + "file_path": dir.to_string_lossy(), + "save_interval_ms": 5 + })), + }), + }; + build_adapter(&config, noop_invoker()).await.unwrap(); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn build_adapter_rejects_unknown_adapter() { + let config = QueueConfig { + adapter: Some(AdapterEntry { + name: "sqs".to_string(), + config: None, + }), + }; + let err = match build_adapter(&config, noop_invoker()).await { + Ok(_) => panic!("expected 'sqs' to be rejected"), + Err(err) => err, + }; + assert_eq!( + err.to_string(), + "queue adapter 'sqs' is not implemented by the standalone queue worker yet" + ); + } + + #[cfg(not(feature = "rabbitmq"))] + #[tokio::test] + async fn build_adapter_rabbitmq_without_feature_mentions_the_feature() { + let config = QueueConfig { + adapter: Some(AdapterEntry { + name: "rabbitmq".to_string(), + config: None, + }), + }; + let err = match build_adapter(&config, noop_invoker()).await { + Ok(_) => panic!("expected 'rabbitmq' to be rejected without the feature"), + Err(err) => err, + }; + assert!( + err.to_string().contains("rabbitmq"), + "error should mention the missing feature: {err}" + ); + assert!( + err.to_string().contains("feature"), + "error should mention the missing feature: {err}" + ); + } + + #[cfg(feature = "test-adapters")] + #[tokio::test] + async fn build_adapter_selects_memory_test_adapter() { + let config = QueueConfig { + adapter: Some(AdapterEntry { + name: "memory".to_string(), + config: None, + }), + }; + let adapter = build_adapter(&config, noop_invoker()).await.unwrap(); + adapter + .enqueue("demo", json!({"ok": true}), None, None) + .await; + } + + #[test] + fn adapter_identity_name_maps_legacy_aliases_to_builtin() { + let in_memory = QueueConfig { + adapter: Some(AdapterEntry { + name: "in_memory".to_string(), + config: None, + }), + }; + let file_based = QueueConfig { + adapter: Some(AdapterEntry { + name: "file_based".to_string(), + config: None, + }), + }; + assert_eq!(adapter_identity_name(&in_memory), "builtin"); + assert_eq!(adapter_identity_name(&file_based), "builtin"); + assert_eq!(adapter_identity_name(&QueueConfig::default()), "builtin"); + } + + #[test] + fn adapter_identity_name_passes_through_other_names() { + let redis = QueueConfig { + adapter: Some(AdapterEntry { + name: "redis".to_string(), + config: None, + }), + }; + assert_eq!(adapter_identity_name(&redis), "redis"); + } } diff --git a/queue/src/config.rs b/queue/src/config.rs index b1ca1863e..5f76ec4c9 100644 --- a/queue/src/config.rs +++ b/queue/src/config.rs @@ -49,6 +49,19 @@ impl QueueConfig { } } +/// The active adapter's `config` object, Null-safe: an absent +/// `adapter`/`adapter.config`, and an explicit JSON `null`, both come back as +/// `None` rather than `Some(Value::Null)` -- adapter constructors +/// (`RedisAdapter::from_config`, `RabbitMQAdapter::from_config`, ...) only +/// ever need to distinguish "no config" from "an actual object". +pub fn adapter_config_value(config: &QueueConfig) -> Option { + config + .adapter + .as_ref() + .and_then(|adapter| adapter.config.clone()) + .filter(|value| !value.is_null()) +} + #[cfg(test)] mod tests { use super::*; @@ -80,4 +93,45 @@ mod tests { let back = QueueConfig::from_json(&c.to_json()).unwrap(); assert_eq!(back.effective_adapter_name(), "file_based"); } + + #[test] + fn adapter_config_value_none_without_adapter() { + assert_eq!(adapter_config_value(&QueueConfig::default()), None); + } + + #[test] + fn adapter_config_value_none_without_config() { + let c = QueueConfig { + adapter: Some(AdapterEntry { + name: "redis".to_string(), + config: None, + }), + }; + assert_eq!(adapter_config_value(&c), None); + } + + #[test] + fn adapter_config_value_none_for_explicit_null() { + let c = QueueConfig { + adapter: Some(AdapterEntry { + name: "redis".to_string(), + config: Some(serde_json::Value::Null), + }), + }; + assert_eq!(adapter_config_value(&c), None); + } + + #[test] + fn adapter_config_value_some_for_object() { + let c = QueueConfig { + adapter: Some(AdapterEntry { + name: "redis".to_string(), + config: Some(serde_json::json!({"redis_url": "redis://x"})), + }), + }; + assert_eq!( + adapter_config_value(&c), + Some(serde_json::json!({"redis_url": "redis://x"})) + ); + } } diff --git a/queue/src/configuration.rs b/queue/src/configuration.rs index 24e38c577..00cf42aac 100644 --- a/queue/src/configuration.rs +++ b/queue/src/configuration.rs @@ -9,10 +9,9 @@ use iii_sdk::{IIIClient, RegisterFunction}; use serde_json::{json, Value}; use crate::adapter::SwappableAdapter; -use crate::adapters::builtin::BuiltinAdapter; -use crate::boot::{ApplyLock, ConfigCell}; +use crate::boot::{self, ApplyLock, ConfigCell}; use crate::config::QueueConfig; -use crate::trigger::{IiiInvoker, QueueTriggerHandler}; +use crate::trigger::{IiiInvoker, Invoker, QueueTriggerHandler}; pub const CONFIG_ID: &str = "queue"; pub const CONFIG_FN_ID: &str = "queue::on-config-change"; @@ -110,28 +109,25 @@ async fn on_config_change( let old = config.read().await.clone(); if swap_needed(&old, &next) { - let registrations = trigger_handler.registrations().await; - let new_store = match crate::boot::build_store(&next).await { - Ok(store) => store, - Err(err) => { - tracing::error!(error = %err, "queue config-change: failed to build replacement store; keeping previous config"); - return; + let invoker = Arc::new(IiiInvoker::new(iii.clone())); + match swap_adapter(&adapter, &trigger_handler, invoker, &next).await { + Ok(()) => { + *config.write().await = Arc::new(next); + tracing::info!("queue transport hot-swapped after configuration change"); } - }; - let new_adapter = Arc::new(BuiltinAdapter::new( - new_store, - Arc::new(IiiInvoker::new(iii.clone())), - )); - trigger_handler.shutdown().await; - adapter.replace(new_adapter).await; - *config.write().await = Arc::new(next); - - for registration in registrations { - if let Err(err) = trigger_handler.register_subscriber(registration).await { - tracing::error!(error = %err, "queue config-change: failed to restart subscriber"); + Err(err) => { + // Mirrors the engine's policy (`engine/src/workers/queue/queue.rs:1002-1019` + // builds the replacement adapter first and only swaps in the + // successful result): a failed hot-swap must never kill a + // live worker, so the previous adapter and config are left + // running untouched on error. + tracing::error!( + error = %err, + "queue config-change: failed to build replacement adapter; keeping the \ + previous adapter running" + ); } } - tracing::info!("queue transport hot-swapped after configuration change"); return; } @@ -139,6 +135,30 @@ async fn on_config_change( tracing::info!("queue configuration reloaded without transport swap"); } +/// Fallible core of a transport hot-swap. Builds the new adapter for `next` +/// via [`crate::boot::build_adapter`] *before* touching anything else: only +/// once that succeeds does this replace the live adapter and re-subscribe +/// every tracked trigger registration onto it. A `build_adapter` failure +/// returns `Err` with the old adapter, its subscriptions, and the config +/// cell completely untouched -- same policy as the engine's `apply_config` +/// (`engine/src/workers/queue/queue.rs:1002-1019`): never tear down a live +/// transport for a replacement that didn't pan out. +async fn swap_adapter( + adapter: &Arc, + trigger_handler: &QueueTriggerHandler, + invoker: Arc, + next: &QueueConfig, +) -> anyhow::Result<()> { + let new_adapter = boot::build_adapter(next, invoker).await?; + let old_adapter = adapter.current().await; + adapter + .replace(new_adapter, boot::adapter_identity_name(next)) + .await; + old_adapter.shutdown().await; + trigger_handler.resubscribe_all().await; + Ok(()) +} + pub fn swap_needed(old: &QueueConfig, new: &QueueConfig) -> bool { effective_adapter(old) != effective_adapter(new) } @@ -221,7 +241,185 @@ pub struct ConfigChangeRequest {} #[cfg(test)] mod tests { use super::*; + use crate::adapter::{QueueAdapter, TopicInfo}; use crate::config::AdapterEntry; + use crate::store::TopicStats; + use crate::trigger::{RegisteredSubscriber, SubscriberSpec}; + use async_trait::async_trait; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + + #[derive(Default)] + struct MockAdapter { + subscribe_calls: AtomicUsize, + enqueue_calls: AtomicUsize, + shutdown_calls: AtomicU64, + } + + #[async_trait] + impl QueueAdapter for MockAdapter { + async fn enqueue( + &self, + _topic: &str, + _data: Value, + _traceparent: Option, + _baggage: Option, + ) { + self.enqueue_calls.fetch_add(1, Ordering::SeqCst); + } + + async fn subscribe( + &self, + _topic: &str, + _id: &str, + _function_id: &str, + _condition_function_id: Option, + _queue_config: Option, + ) { + self.subscribe_calls.fetch_add(1, Ordering::SeqCst); + } + + async fn unsubscribe(&self, _topic: &str, _id: &str) {} + + async fn redrive_dlq(&self, _topic: &str) -> anyhow::Result { + Ok(0) + } + + async fn redrive_dlq_message( + &self, + _topic: &str, + _message_id: &str, + ) -> anyhow::Result { + Ok(false) + } + + async fn discard_dlq_message( + &self, + _topic: &str, + _message_id: &str, + ) -> anyhow::Result { + Ok(false) + } + + async fn dlq_count(&self, _topic: &str) -> anyhow::Result { + Ok(0) + } + + async fn list_topics(&self) -> anyhow::Result> { + Ok(vec![]) + } + + async fn topic_stats(&self, _topic: &str) -> anyhow::Result { + Ok(TopicStats::default()) + } + + async fn shutdown(&self) { + self.shutdown_calls.fetch_add(1, Ordering::SeqCst); + } + } + + struct NoopInvoker; + + #[async_trait] + impl Invoker for NoopInvoker { + async fn call(&self, _function_id: &str, _payload: Value) -> Result, String> { + Ok(None) + } + } + + fn noop_invoker() -> Arc { + Arc::new(NoopInvoker) + } + + #[tokio::test] + async fn swap_adapter_keeps_old_adapter_running_when_build_fails() { + let mock = Arc::new(MockAdapter::default()); + let dyn_adapter: Arc = mock.clone(); + let adapter = Arc::new(SwappableAdapter::new(dyn_adapter, "mock")); + let trigger_handler = QueueTriggerHandler::new(adapter.clone()); + trigger_handler + .register_subscriber(RegisteredSubscriber { + trigger_id: "t1".to_string(), + function_id: "backend".to_string(), + spec: SubscriberSpec { + queue: "demo".to_string(), + max_retries: None, + backoff_ms: None, + condition_function_id: None, + queue_config: None, + }, + }) + .await + .unwrap(); + assert_eq!(mock.subscribe_calls.load(Ordering::SeqCst), 1); + + let bad_config = QueueConfig { + adapter: Some(AdapterEntry { + name: "not-a-real-adapter".to_string(), + config: None, + }), + }; + + let err = swap_adapter(&adapter, &trigger_handler, noop_invoker(), &bad_config) + .await + .unwrap_err(); + assert!(err.to_string().contains("not implemented")); + + // The old adapter must never be shut down, and it must still be the + // one live behind the `SwappableAdapter`. + assert_eq!(mock.shutdown_calls.load(Ordering::SeqCst), 0); + assert_eq!(adapter.current_name().await, "mock"); + adapter.enqueue("demo", Value::Null, None, None).await; + assert_eq!(mock.enqueue_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn swap_adapter_replaces_transport_and_resubscribes_on_success() { + let old_mock = Arc::new(MockAdapter::default()); + let old_dyn: Arc = old_mock.clone(); + let adapter = Arc::new(SwappableAdapter::new(old_dyn, "mock")); + let trigger_handler = QueueTriggerHandler::new(adapter.clone()); + trigger_handler + .register_subscriber(RegisteredSubscriber { + trigger_id: "t1".to_string(), + function_id: "backend".to_string(), + spec: SubscriberSpec { + queue: "demo".to_string(), + max_retries: None, + backoff_ms: None, + condition_function_id: None, + queue_config: None, + }, + }) + .await + .unwrap(); + assert_eq!(old_mock.subscribe_calls.load(Ordering::SeqCst), 1); + + // A real (dependency-free) config change: builtin -> in-memory, + // which `build_adapter` resolves successfully. + let next = QueueConfig { + adapter: Some(AdapterEntry { + name: "in_memory".to_string(), + config: None, + }), + }; + + swap_adapter(&adapter, &trigger_handler, noop_invoker(), &next) + .await + .unwrap(); + + assert_eq!( + old_mock.shutdown_calls.load(Ordering::SeqCst), + 1, + "old adapter must be shut down once the new one is live" + ); + assert_eq!(adapter.current_name().await, "builtin"); + // The registration is still tracked and was re-subscribed onto the + // new adapter (a builtin adapter has no externally observable + // subscribe counter, so this is asserted end-to-end in + // `trigger::tests::resubscribe_all_attaches_every_registration_to_the_new_adapter` + // -- here we only assert the registration survived the swap). + assert_eq!(trigger_handler.registrations().await.len(), 1); + } #[test] fn swap_needed_ignores_implicit_builtin_default() { diff --git a/queue/src/functions.rs b/queue/src/functions.rs index 1f5776766..a75eac5bf 100644 --- a/queue/src/functions.rs +++ b/queue/src/functions.rs @@ -249,6 +249,7 @@ pub async fn discard_message( } pub async fn list_topics(adapter: Arc) -> Result, Error> { + let broker_type = adapter.current_name().await; let topics = adapter .list_topics() .await @@ -257,7 +258,7 @@ pub async fn list_topics(adapter: Arc) -> Result) -> Result, Error> { + let broker_type = adapter.current_name().await; let topics = adapter .list_topics() .await @@ -300,7 +302,7 @@ pub async fn dlq_topics(adapter: Arc) -> Result 0 { dlq_topics.push(DlqTopicInfo { topic: topic.name, - broker_type: "builtin".to_string(), + broker_type: broker_type.clone(), message_count: dlq_count, }); } @@ -506,7 +508,7 @@ mod tests { fn adapter() -> (Arc, Arc) { let mock = Arc::new(MockAdapter::default()); let dyn_adapter: Arc = mock.clone(); - (Arc::new(SwappableAdapter::new(dyn_adapter)), mock) + (Arc::new(SwappableAdapter::new(dyn_adapter, "mock")), mock) } fn job_value(id: &str, payload: Value, attempts: u32) -> Value { @@ -655,7 +657,7 @@ mod tests { topics[0], TopicInfo { name: "demo".to_string(), - broker_type: "builtin".to_string(), + broker_type: "mock".to_string(), subscriber_count: 0, } ); @@ -698,6 +700,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].topic, "demo"); assert_eq!(result[0].message_count, 1); + assert_eq!(result[0].broker_type, "mock"); } #[tokio::test] diff --git a/queue/src/trigger.rs b/queue/src/trigger.rs index 757394fae..9b5c038f1 100644 --- a/queue/src/trigger.rs +++ b/queue/src/trigger.rs @@ -124,6 +124,28 @@ impl QueueTriggerHandler { self.adapter.shutdown().await; } + /// Re-subscribe every currently tracked registration directly against + /// the adapter, bypassing [`Self::register_subscriber`]'s duplicate-id + /// check and leaving the registrations map untouched. Used by the + /// configuration hot-swap path (`crate::configuration::swap_adapter`) + /// *after* [`SwappableAdapter::replace`] has pointed `self.adapter` at + /// the newly built transport, so these `subscribe` calls attach + /// consumers to the new adapter rather than the one being retired. + pub async fn resubscribe_all(&self) { + for registration in self.registrations().await { + let queue_config = merged_queue_config(®istration.spec); + self.adapter + .subscribe( + ®istration.spec.queue, + ®istration.trigger_id, + ®istration.function_id, + registration.spec.condition_function_id.clone(), + Some(queue_config), + ) + .await; + } + } + pub async fn register_subscriber( &self, registration: RegisteredSubscriber, @@ -304,7 +326,7 @@ mod tests { fn handler_with_mock() -> (QueueTriggerHandler, Arc) { let mock = Arc::new(MockAdapter::default()); let adapter: Arc = mock.clone(); - let handler = QueueTriggerHandler::new(Arc::new(SwappableAdapter::new(adapter))); + let handler = QueueTriggerHandler::new(Arc::new(SwappableAdapter::new(adapter, "mock"))); (handler, mock) } @@ -403,6 +425,61 @@ mod tests { assert!(handler.registrations().await.is_empty()); } + #[tokio::test] + async fn resubscribe_all_attaches_every_registration_to_the_new_adapter() { + let (handler, old_mock) = handler_with_mock(); + handler + .register_trigger(trigger_config( + "t1", + "backend-1", + json!({"queue": "demo-1", "max_retries": 2}), + )) + .await + .unwrap(); + handler + .register_trigger(trigger_config( + "t2", + "backend-2", + json!({"queue": "demo-2"}), + )) + .await + .unwrap(); + assert_eq!(old_mock.calls().len(), 2, "initial registration subscribes"); + + // Simulate a hot-swap: the adapter behind the handler's + // `SwappableAdapter` is replaced with a fresh mock. + let new_mock = Arc::new(MockAdapter::default()); + let new_adapter: Arc = new_mock.clone(); + handler.adapter.replace(new_adapter, "new-mock").await; + + handler.resubscribe_all().await; + + // The old adapter never sees another call post-swap. + assert_eq!(old_mock.calls().len(), 2); + // The new adapter is subscribed once per tracked registration. + let new_calls = new_mock.calls(); + assert_eq!(new_calls.len(), 2); + assert!(new_calls.contains(&Call::Subscribe { + topic: "demo-1".to_string(), + id: "t1".to_string(), + function_id: "backend-1".to_string(), + condition_function_id: None, + queue_config: Some(SubscriberQueueConfig { + max_retries: Some(2), + ..Default::default() + }), + })); + assert!(new_calls.contains(&Call::Subscribe { + topic: "demo-2".to_string(), + id: "t2".to_string(), + function_id: "backend-2".to_string(), + condition_function_id: None, + queue_config: Some(SubscriberQueueConfig::default()), + })); + // The registrations map itself is untouched by the swap. + assert_eq!(handler.registrations().await.len(), 2); + } + #[tokio::test] async fn unregister_unknown_id_is_a_noop() { let (handler, mock) = handler_with_mock(); From 43252e71b63e86b12049f4bc3204e150831714d4 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Tue, 7 Jul 2026 09:10:04 -0300 Subject: [PATCH 26/28] docs(queue): adapter matrix, parity notes, and config examples for all transports --- queue/README.md | 124 ++++++++++++++++++++++++++++++++++++++++-- queue/iii.worker.yaml | 25 +++++++++ queue/skills/SKILL.md | 38 +++++++++++-- queue/src/boot.rs | 10 ++++ 4 files changed, 186 insertions(+), 11 deletions(-) diff --git a/queue/README.md b/queue/README.md index f91729fa0..c327646b2 100644 --- a/queue/README.md +++ b/queue/README.md @@ -47,6 +47,23 @@ Configuration is owned by the `configuration` worker under id `queue`. Seed it once with `--config .yaml`; runtime edits come from the configuration worker after that. +`adapter.name` selects the transport: `builtin` (default), `redis`, or +`rabbitmq`. Changing the adapter config hot-swaps the transport and +restarts every consumer — the new adapter is built first, so a bad config +leaves the previous adapter serving. Pending in-memory jobs are lost on +swap/restart; file-backed jobs survive. An unreachable `redis`/`rabbitmq` +target at boot fails the boot itself (no fallback to `builtin`). + +### Adapters + +#### `builtin` (default) + +In-process, single-worker transport. Full fan-out (every subscriber on a +topic receives every published message), retries, DLQ, redrive/discard, +and both `fifo` and `concurrent` subscriber modes. The legacy aliases +`in_memory` and `file_based` are also accepted as `adapter.name` and both +resolve to this transport. + ```yaml adapter: name: builtin @@ -58,14 +75,84 @@ adapter: | Field | Default | Description | |---|---|---| -| `adapter.name` | `builtin` | In-process queue transport. `redis` and `rabbitmq` are follow-up transports. | | `adapter.config.store_method` | `in_memory` | `in_memory` or `file_based`. | | `adapter.config.file_path` | `queue_store_data` | Directory used by `file_based`. | -| `adapter.config.save_interval_ms` | `5000` | Accepted for parity; this worker persists on mutation. | +| `adapter.config.save_interval_ms` | `5000` | Accepted for parity; this worker persists on mutation rather than on an interval. | + +#### `redis` -Changing the adapter config hot-swaps the transport and restarts every -consumer. Pending in-memory jobs are lost on swap/restart, matching the -built-in adapter's in-process durability profile. File-backed jobs survive. +Pub/sub only — 1:1 port of the engine builtin's `RedisAdapter`, including +its limitations. There is no DLQ, no retries, and no message durability: a +message published with no subscriber connected is lost. + +```yaml +adapter: + name: redis + config: + redis_url: redis://localhost:6379 +``` + +| Field | Default | Description | +|---|---|---| +| `adapter.config.redis_url` | `redis://localhost:6379` | Redis connection string. | + +Every DLQ operation (`redrive`, `redrive_message`, `discard_message`, +`dlq_count`, `dlq_peek`) returns the error +`RedisAdapter does not support DLQ operations (pub/sub only)`, verbatim +from the engine. + +#### `rabbitmq` + +Full-featured transport: retries, DLQ, redrive, message priority, and both +standard and FIFO queue modes. On by default (the `rabbitmq` cargo feature +is feature-default-on); building with `--no-default-features` drops it, +and selecting `adapter.name: rabbitmq` in that build fails boot with an +error naming the missing feature. + +```yaml +adapter: + name: rabbitmq + config: + amqp_url: amqp://localhost:5672 + max_attempts: 3 + prefetch_count: 10 + queue_mode: standard + priority_field: priority +``` + +| Field | Default | Description | +|---|---|---| +| `adapter.config.amqp_url` | `amqp://localhost:5672` | AMQP connection string. | +| `adapter.config.max_attempts` | `3` | Delivery attempts before a message moves to DLQ (adapter-level default; per-subscriber `queue_config.maxRetries` overrides it). | +| `adapter.config.prefetch_count` | `10` | Consumer prefetch (QoS), used in `standard` queue mode. | +| `adapter.config.queue_mode` | `standard` | `standard` or `fifo`. Unrecognized values fall back to `standard`. | +| `adapter.config.priority_field` | none | JSON field read from each published message's payload to set the AMQP message priority. Only applies to subscribers whose `queue_config.maxPriority` declares the queue as a priority queue (`x-max-priority`). | + +Per-subscriber queue tuning uses the trigger's `queue_config` below; +`maxPriority` is RabbitMQ-only and ignored by the other adapters. + +#### `memory` (dev/test only) + +An in-process, test-only transport (`adapters::memory::MemoryAdapter`) +used by this worker's own hot-swap tests. It is gated behind the +`test-adapters` cargo feature, off in normal builds, and not a supported +`adapter.name` value for real deployments. + +### Trigger `queue_config` + +The `durable:subscriber` trigger's `queue_config` accepts the full builtin +`SubscriberQueueConfig` shape: + +| Field | Default | Description | +|---|---|---| +| `type` | `concurrent` | `fifo` (strictly serial) or `concurrent`. | +| `maxRetries` | `3` | Failed deliveries before DLQ, per subscriber. | +| `concurrency` | `10` (builtin) | In-flight invocations for `concurrent` mode. `0` pauses consumption. Ignored by `fifo` (always serial — this worker has no grouped-fifo partitioning; see Known Gaps). | +| `visibilityTimeout` | — | Accepted for parity. | +| `delaySeconds` | — | Accepted for parity. | +| `backoffType` | — | Accepted for parity. | +| `backoffDelayMs` | `1000` | Base retry delay; exponential backoff. | +| `maxPriority` | none | RabbitMQ-only: declares the subscriber's queue as an AMQP priority queue with this many levels (1-255). | ## Requires Removing The Built-In `iii-queue` Worker @@ -86,9 +173,34 @@ On boot, this worker queries `engine::workers::list` and refuses to start if | DLQ | after retries exhausted; redrive/redrive_message/discard | same | | Restart survival | file-backed store | same (`file_based`) | | In-memory mode | jobs lost on restart/swap | same | -| Transports | builtin/memory/redis/rabbitmq | builtin memory/file; redis+rabbitmq follow-up | +| Transports | builtin (memory/file), redis (pub/sub, no DLQ), rabbitmq (full: retry/DLQ/priority/fifo) | same as builtin | +| bridge adapter | engine-internal (`TriggerAction::Enqueue`) | N/A — engine `QueueEnqueuer` cut (MOT-3829) | | Enqueue failure when worker offline | n/a, in-process | invocation fails explicitly once the engine remote-enqueue cut lands | | Latency benchmark | in-process baseline required | tracked in `queue/benches/enqueue_latency.md`; final budget is a pre-deprecation gate | The full engine `TriggerAction::Enqueue` path depends on the separate `QueueEnqueuer` engine cut tracked in the migration master plan. + +## Known Gaps / Parity Notes + +- **`bridge` adapter — not ported.** It was engine-internal plumbing for + `TriggerAction::Enqueue`; that whole path is superseded by the engine's + `QueueEnqueuer` cut (MOT-3829), which is out of scope for this worker. +- **No grouped-fifo.** The engine's `GroupedFifoWorker` partitions `fifo` + delivery by `job.group_id` when `concurrency > 1`, running each group + serially but different groups concurrently. This worker's `fifo` mode is + strictly serial regardless of `concurrency` — there is no group + partitioning. +- **`resolve_dlq_name` bare-topic behavior ported verbatim, bug and all.** + DLQ operations (`redrive`, `dlq_count`, `topic_stats`, `dlq_peek`) + address a bare topic name and, on the builtin adapter, aggregate across + every subscriber's internal queue on that topic — matching the engine's + behavior (and its naming quirk) exactly, rather than fixing it here. +- **`dlq_topics` filters to `dlq_count > 0`.** The engine's equivalent also + iterates every known topic but returns all of them regardless of DLQ + depth; this worker only returns topics that currently have dead-lettered + messages. Documented divergence, not a bug to reconcile. +- **Fifo retry via `nack` can be overtaken by newer arrivals.** A failed + fifo message is re-queued via `nack` rather than blocking the poller + in-place (the engine's `FifoWorker` blocks); a message enqueued after + the failure can be delivered before the retried one catches up. diff --git a/queue/iii.worker.yaml b/queue/iii.worker.yaml index 05a9429b6..ec2f9ac49 100644 --- a/queue/iii.worker.yaml +++ b/queue/iii.worker.yaml @@ -5,6 +5,31 @@ deploy: binary manifest: Cargo.toml bin: queue description: Durable function queues - registers the `durable:subscriber` trigger type and the queue/DLQ service functions. +# `config` seeds the `configuration` worker's `queue` entry on first boot; +# runtime edits after that go through the configuration worker, not this +# file. `adapter.name` selects the transport. See README.md for the full +# config key reference per adapter. Alternatives to the default below: +# +# adapter: # redis: pub/sub only, no DLQ +# name: redis +# config: +# redis_url: redis://localhost:6379 +# +# adapter: # rabbitmq: full retry/DLQ/priority/fifo +# name: rabbitmq +# config: +# amqp_url: amqp://localhost:5672 +# max_attempts: 3 +# prefetch_count: 10 +# queue_mode: standard # or fifo +# priority_field: priority +# +# adapter: # builtin: file-backed, survives restarts +# name: builtin +# config: +# store_method: file_based +# file_path: ./data/queue +# save_interval_ms: 5000 config: adapter: name: builtin diff --git a/queue/skills/SKILL.md b/queue/skills/SKILL.md index 41151e283..5a210a704 100644 --- a/queue/skills/SKILL.md +++ b/queue/skills/SKILL.md @@ -29,13 +29,18 @@ the same `durable:subscriber` trigger type. ## Boundaries - Not fire-and-forget broadcast. Use `pubsub` when missed events are acceptable. -- Redis and RabbitMQ transports are follow-up work; this worker currently ships - the builtin in-process transport with `in_memory` and `file_based` storage. +- Three transports ship today: `builtin` (in-process, full DLQ/retry/fifo), + `redis` (pub/sub only, no DLQ), and `rabbitmq` (full: retry/DLQ/priority/fifo). + The `bridge` adapter is not ported — it was engine-internal and is superseded + by the engine's `QueueEnqueuer` cut (MOT-3829). - Engine `TriggerAction.Enqueue` / named function queues require the separate `QueueEnqueuer` engine cut. Until that lands, use `iii::durable::publish` and `durable:subscriber` triggers for this standalone worker. - File-backed mode survives worker restarts. In-memory mode loses pending jobs - on restart or transport hot-swap. + on restart or transport hot-swap. An unreachable `redis`/`rabbitmq` target at + boot fails the boot (no fallback to `builtin`). +- `fifo` mode is strictly serial — there is no grouped-fifo partitioning by + `group_id` like the engine's `GroupedFifoWorker`. - This worker carries work items, not key/value or stream state. Use `state` or `stream` for those. @@ -100,6 +105,29 @@ adapter: save_interval_ms: 5000 ``` +For pub/sub-only delivery against a shared Redis (no DLQ, no durability): + +```yaml +adapter: + name: redis + config: + redis_url: redis://localhost:6379 +``` + +For full retry/DLQ/priority/fifo delivery against RabbitMQ: + +```yaml +adapter: + name: rabbitmq + config: + amqp_url: amqp://localhost:5672 + max_attempts: 3 + prefetch_count: 10 + queue_mode: standard + priority_field: priority +``` + Changing the adapter config hot-swaps the transport and restarts every -consumer. File-backed jobs survive by construction; in-memory jobs are lost, -matching the builtin adapter's in-process durability profile. +consumer — the new adapter is built first, so a bad config leaves the +previous one serving. File-backed jobs survive by construction; in-memory +jobs are lost, matching the builtin adapter's in-process durability profile. diff --git a/queue/src/boot.rs b/queue/src/boot.rs index b104ba7a9..a8c960690 100644 --- a/queue/src/boot.rs +++ b/queue/src/boot.rs @@ -368,6 +368,16 @@ mod tests { ); } + // NOTE: this test never actually runs under plain `cargo test`, even with + // `--no-default-features`. The dev-dependency self-reference + // (`iii-queue = { path = ".", features = ["test-adapters"] }` above) + // doesn't set `default-features = false`, so Cargo's feature unification + // pulls the default `rabbitmq` feature back in for the whole test build + // regardless of what the outer `cargo test` invocation requested. The + // `#[cfg(not(feature = "rabbitmq"))]` guard is therefore always false in + // practice; this only compiles/runs if something changes the dev-dep to + // disable default features. Same gap as the engine's equivalent test + // (`engine/Cargo.toml:168-171`) — left unfixed here to stay 1:1 with it. #[cfg(not(feature = "rabbitmq"))] #[tokio::test] async fn build_adapter_rabbitmq_without_feature_mentions_the_feature() { From f03c30b96e5119e845acb392511cddbd9c89a6cd Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Tue, 7 Jul 2026 13:36:11 -0300 Subject: [PATCH 27/28] chore(queue): drop dead SwappableStore and fix rabbitmq maxRetries doc claim --- queue/README.md | 4 +-- queue/src/adapter.rs | 7 ++-- queue/src/store.rs | 79 +------------------------------------------- 3 files changed, 6 insertions(+), 84 deletions(-) diff --git a/queue/README.md b/queue/README.md index c327646b2..eb92534f7 100644 --- a/queue/README.md +++ b/queue/README.md @@ -123,7 +123,7 @@ adapter: | Field | Default | Description | |---|---|---| | `adapter.config.amqp_url` | `amqp://localhost:5672` | AMQP connection string. | -| `adapter.config.max_attempts` | `3` | Delivery attempts before a message moves to DLQ (adapter-level default; per-subscriber `queue_config.maxRetries` overrides it). | +| `adapter.config.max_attempts` | `3` | Delivery attempts before a message moves to DLQ. The retry budget is stamped on each message at publish time from this adapter-level value, so per-subscriber `queue_config.maxRetries` does not override it on this transport (it applies to the builtin adapter). | | `adapter.config.prefetch_count` | `10` | Consumer prefetch (QoS), used in `standard` queue mode. | | `adapter.config.queue_mode` | `standard` | `standard` or `fifo`. Unrecognized values fall back to `standard`. | | `adapter.config.priority_field` | none | JSON field read from each published message's payload to set the AMQP message priority. Only applies to subscribers whose `queue_config.maxPriority` declares the queue as a priority queue (`x-max-priority`). | @@ -146,7 +146,7 @@ The `durable:subscriber` trigger's `queue_config` accepts the full builtin | Field | Default | Description | |---|---|---| | `type` | `concurrent` | `fifo` (strictly serial) or `concurrent`. | -| `maxRetries` | `3` | Failed deliveries before DLQ, per subscriber. | +| `maxRetries` | `3` | Failed deliveries before DLQ, per subscriber (builtin adapter; RabbitMQ uses adapter-level `max_attempts`). | | `concurrency` | `10` (builtin) | In-flight invocations for `concurrent` mode. `0` pauses consumption. Ignored by `fifo` (always serial — this worker has no grouped-fifo partitioning; see Known Gaps). | | `visibilityTimeout` | — | Accepted for parity. | | `delaySeconds` | — | Accepted for parity. | diff --git a/queue/src/adapter.rs b/queue/src/adapter.rs index 9d2ae507c..355d5992b 100644 --- a/queue/src/adapter.rs +++ b/queue/src/adapter.rs @@ -222,10 +222,9 @@ pub trait QueueAdapter: Send + Sync + 'static { } /// Hot-swappable [`QueueAdapter`]: holds the currently active adapter behind -/// an `RwLock` and forwards every trait method to it. Same pattern as -/// [`crate::store::SwappableStore`] — a configuration change replaces the -/// inner adapter (e.g. builtin -> file-backed builtin) without the trigger -/// handler or service functions needing to know a swap happened. +/// an `RwLock` and forwards every trait method to it. A configuration change +/// replaces the inner adapter (e.g. builtin -> file-backed builtin) without +/// the trigger handler or service functions needing to know a swap happened. #[derive(Clone)] pub struct SwappableAdapter { inner: Arc>>, diff --git a/queue/src/store.rs b/queue/src/store.rs index 8eb6baa4d..a7096e853 100644 --- a/queue/src/store.rs +++ b/queue/src/store.rs @@ -15,7 +15,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::Value; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::Mutex; use uuid::Uuid; const STORE_FILE_NAME: &str = "queue_store.json"; @@ -53,83 +53,6 @@ pub trait QueueStore: Send + Sync + 'static { async fn discard_dlq_message(&self, topic: &str, job_id: &str) -> bool; } -#[derive(Clone)] -pub struct SwappableStore { - inner: Arc>>, -} - -impl SwappableStore { - pub fn new(store: Arc) -> Self { - Self { - inner: Arc::new(RwLock::new(store)), - } - } - - pub async fn current(&self) -> Arc { - self.inner.read().await.clone() - } - - pub async fn replace(&self, store: Arc) { - *self.inner.write().await = store; - } -} - -#[async_trait] -impl QueueStore for SwappableStore { - async fn enqueue(&self, topic: &str, payload: Value) -> anyhow::Result { - self.current().await.enqueue(topic, payload).await - } - - async fn dequeue(&self, topic: &str) -> Option { - self.current().await.dequeue(topic).await - } - - async fn ack(&self, topic: &str, job_id: &str) { - self.current().await.ack(topic, job_id).await; - } - - async fn nack(&self, topic: &str, job: Job, max_retries: u32, backoff_ms: u64) { - self.current() - .await - .nack(topic, job, max_retries, backoff_ms) - .await; - } - - async fn list_topics(&self) -> Vec { - self.current().await.list_topics().await - } - - async fn topic_stats(&self, topic: &str) -> TopicStats { - self.current().await.topic_stats(topic).await - } - - async fn dlq_topics(&self) -> Vec<(String, u64)> { - self.current().await.dlq_topics().await - } - - async fn dlq_messages(&self, topic: &str, limit: u64) -> Vec { - self.current().await.dlq_messages(topic, limit).await - } - - async fn redrive_dlq(&self, topic: &str) -> u64 { - self.current().await.redrive_dlq(topic).await - } - - async fn redrive_dlq_message(&self, topic: &str, job_id: &str) -> bool { - self.current() - .await - .redrive_dlq_message(topic, job_id) - .await - } - - async fn discard_dlq_message(&self, topic: &str, job_id: &str) -> bool { - self.current() - .await - .discard_dlq_message(topic, job_id) - .await - } -} - #[derive(Debug, Clone, Default, Serialize, Deserialize)] struct StoreData { queues: HashMap>, From c43e9581e93efb9e0122342a968833d413fefb65 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Tue, 7 Jul 2026 14:51:15 -0300 Subject: [PATCH 28/28] docs(queue): note MOT-3904 upstream tracking for the rabbitmq bare-topic DLQ quirk --- queue/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/queue/README.md b/queue/README.md index eb92534f7..5379affb5 100644 --- a/queue/README.md +++ b/queue/README.md @@ -176,7 +176,7 @@ On boot, this worker queries `engine::workers::list` and refuses to start if | Transports | builtin (memory/file), redis (pub/sub, no DLQ), rabbitmq (full: retry/DLQ/priority/fifo) | same as builtin | | bridge adapter | engine-internal (`TriggerAction::Enqueue`) | N/A — engine `QueueEnqueuer` cut (MOT-3829) | | Enqueue failure when worker offline | n/a, in-process | invocation fails explicitly once the engine remote-enqueue cut lands | -| Latency benchmark | in-process baseline required | tracked in `queue/benches/enqueue_latency.md`; final budget is a pre-deprecation gate | +| Latency benchmark | in-process baseline required | final budget is a pre-deprecation gate (tracked in the migration master plan) | The full engine `TriggerAction::Enqueue` path depends on the separate `QueueEnqueuer` engine cut tracked in the migration master plan. @@ -196,6 +196,10 @@ The full engine `TriggerAction::Enqueue` path depends on the separate address a bare topic name and, on the builtin adapter, aggregate across every subscriber's internal queue on that topic — matching the engine's behavior (and its naming quirk) exactly, rather than fixing it here. + On the rabbitmq transport this makes DLQ browse/redrive against a + subscriber topic 404 (retries write to the per-function DLQ), and the + AMQP channel closure stops every consumer on the worker. Upstream engine + fix tracked as MOT-3904; this worker follows once the engine lands it. - **`dlq_topics` filters to `dlq_count > 0`.** The engine's equivalent also iterates every known topic but returns all of them regardless of DLQ depth; this worker only returns topics that currently have dead-lettered