diff --git a/.release/desktop-candidate.json b/.release/desktop-candidate.json index 3957e635acc..b8516ee3463 100644 --- a/.release/desktop-candidate.json +++ b/.release/desktop-candidate.json @@ -1,10 +1,10 @@ { "schema": 2, - "version": "0.5.10", - "base_sha": "f35930104bcbdb1332ff13735214ecb9fce1fc7b", - "previous_tag": "desktop-v0.5.9", - "previous_base_sha": "f8f2ef0440e7a074223ec04dc3b32d817b8b9d9b", - "previous_merge_sha": "538e5e113fc33571f939c87b925567fd4e277109", - "tag": "desktop-v0.5.10", - "commit_count": 18 + "version": "0.5.11", + "base_sha": "4749bc7be3cdb78c2db4ce4864775ba7ab60b4cc", + "previous_tag": "desktop-v0.5.10", + "previous_base_sha": "f35930104bcbdb1332ff13735214ecb9fce1fc7b", + "previous_merge_sha": "4b3570671eb2786594267758af18784ac6e82972", + "tag": "desktop-v0.5.11", + "commit_count": 16 } diff --git a/AGENTS.md b/AGENTS.md index c0875305f66..f3793f5a29c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -172,8 +172,8 @@ place. | `.github/workflows/upstream-sync-merge.yml` | new | The deterministic (01:30) sync stage — the one that preserves the merge parent. Plain git, no AI. Optional `SYNC_PUSH_TOKEN` secret: a branch pushed with `GITHUB_TOKEN` does not start new workflow runs, so set a PAT if CI stops firing on sync PRs | | `.github/workflows/upstream-sync-ci-status.yml` | new | Labels an open sync PR `sync-ci-green`/`sync-ci-red` once checks settle, and re-requests the Copilot review that gh-aw's `reviewers:` fails to attach. Deliberately does not merge | | `migrations/0027_wallet_binding_fts.sql`, `0028_wallet_binding_fts_kind_move.sql` | new, and **kept after the feature was removed** | Search exclusions for the withdrawn NIP-SW wallet binding. They have already run on live databases and sqlx checksums applied migrations, so deleting them breaks startup validation. What they leave behind — a `search_tsv` expression excluding a kind nobody publishes — is inert, and unwinding it would rewrite a generated column across the whole events table for nothing. **Never edit or delete an applied migration**; add a follow-on | -| `migrations/0029_channels_id_lookup_index.sql`, `0030_long_reaction_payloads.sql` | upstream's `0027_channels_id_lookup_index.sql` and `0028_long_reaction_payloads.sql`, **renumbered**; contents byte-identical | The fork holds 0027 and 0028, so upstream's own new migrations have to arrive above them. Two syncs running, so treat this as the standing cost of the fork's migration block rather than a special case. See [Upstream migrations arrive renumbered](#upstream-migrations-arrive-renumbered) | -| `crates/buzz-db/src/migration.rs` | `migrations.len()` assertion is 30, not upstream's 28; upstream's channel-index assertion reads `migrations[28].version == 29`, its long-reaction assertion reads `migrations[29].version == 30`, and the highest applied version is `Some(30)` | Counts embedded migrations, so it moves whenever *either* side adds one. `0027` landed without bumping it and left the test failing on `main`; fixed in PR #9. Beyond the count, every upstream assertion that indexes `migrations[…]` past 25 or names a version above 26 has to be shifted by the fork's two — see the section below for why the test suite will *not* catch it if you forget | +| `migrations/0029_channels_id_lookup_index.sql`, `0030_long_reaction_payloads.sql`, `0031_community_deletion.sql`, `0032_community_deletion_recovery.sql` | upstream's `0027_channels_id_lookup_index.sql`, `0028_long_reaction_payloads.sql`, `0029_community_deletion.sql` and `0030_community_deletion_recovery.sql`, **renumbered**; contents byte-identical | The fork holds 0027 and 0028, so upstream's own new migrations have to arrive above them. Four syncs running, so treat this as the standing cost of the fork's migration block rather than a special case. See [Upstream migrations arrive renumbered](#upstream-migrations-arrive-renumbered) | +| `crates/buzz-db/src/migration.rs` | `migrations.len()` assertion is 32, not upstream's 30; upstream's channel-index assertion reads `migrations[28].version == 29`, long-reaction `migrations[29].version == 30`, deletion `migrations[30].version == 31`, deletion-recovery `migrations[31].version == 32`; and `deletion_surface_parity_between_migration_0029_and_schema_sql` looks up `version == 31` | Counts embedded migrations, so it moves whenever *either* side adds one. `0027` landed without bumping it and left the test failing on `main`; fixed in PR #9. Beyond the count, every upstream assertion that indexes `migrations[…]` past 25 or names a version above 26 has to be shifted by the fork's two — see the section below for why the test suite will *not* catch it if you forget. **The highest-applied-version assertion is no longer a fork patch**: upstream's 2026-08-13 range replaced the hardcoded `Some(30)` with a `latest_version` derived from `MIGRATOR`, so it now tracks the renumber on its own — take upstream's version if it ever conflicts again | | `.github/workflows/macos-canary.yml` | new; `push` trigger on `main` with desktop path filters | Unsigned macOS canary; upstream only has a *signed* one, which a fork cannot run. Builds automatically when `desktop/**`, `crates/**` or the root `Cargo.*` change, so the newest artifact always matches `main` — it was dispatch-only, and the sole artifact went 13 commits stale. Free: the repo is public, so GitHub-hosted macOS runners are unbilled. Stages the artifact and the usage notes under the product name read from `tauri.conf.json`, not a hardcoded one, so the brand rename below cannot publish a build under the old name. Sets `signingIdentity: "-"` in its inline config and runs **without** `--no-sign`, which would silently discard it; asserts the bundle signature of the `.app` inside the mounted DMG. Its **sidecar list must track upstream's non-Windows lanes**: `tauri.conf.json`'s `externalBin` is shared, and `scripts/bundle-sidecars.sh` exits 1 on a missing binary, so a sidecar upstream adds breaks this workflow without ever conflicting — `buzz-backend-kubernetes` (#4289) did exactly that in the 2026-08-03 sync | | `Dockerfile` | `buzz-paymaster` added to the cargo build, the strip step, and both `COPY` stages | The sponsor ships in the relay's image so there is one publish pipeline and one immutable `:sha-<7>` tag for `deploy-aws.yml` to pin. Four one-line additions, each inside an existing parallel list, so a conflict resolves as *keep ours, take upstream's*. It is **not** the `ENTRYPOINT` — `infra/aws/paymaster.tf` overrides `entryPoint` | | `.github/aw/actions-lock.json` | new | gh-aw action SHA pins | @@ -245,12 +245,16 @@ database**, so the side with *applied history* keeps it — the fork. Same shape collision, opposite resolution, because "already deployed" points at different parties in the two cases. -It has now happened twice running — `0027_channels_id_lookup_index.sql` (upstream -#4647) in the 2026-08-05 sync, then `0028_long_reaction_payloads.sql` (upstream #3833) -in the 2026-08-06 sync, renumbered to `0029` and `0030`. Expect it on any sync that -touches `migrations/`, and note that the *second* collision is the more dangerous -shape: upstream's 0028 landed on the fork's 0028, so the two files sorted adjacent and -the tree looked plausible. **A new file under `migrations/` is the tripwire — check the +It has now happened on three separate syncs, covering four migrations — +`0027_channels_id_lookup_index.sql` (upstream #4647) in the 2026-08-05 sync, then +`0028_long_reaction_payloads.sql` (upstream #3833) in the 2026-08-06 sync, then +`0029_community_deletion.sql` **and** `0030_community_deletion_recovery.sql` +(upstream #4425) together in the 2026-08-13 sync; renumbered to `0029`, `0030`, +`0031` and `0032`. Expect it on any sync that touches `migrations/`, and note that +the *second* collision is the more dangerous shape: upstream's 0028 landed on the +fork's 0028, so the two files sorted adjacent and the tree looked plausible. The +2026-08-13 sync was that shape twice over — both of upstream's new files landed on +fork-held integers. **A new file under `migrations/` is the tripwire — check the version integer before reading anything else in the diff.** The first one is worth keeping in full because it shows exactly how the failure hides @@ -297,10 +301,28 @@ merge cleanly into a tree where both are wrong: |------|----------|--------------------------------------| | 2026-08-05 | 0027 → 0029 | `migrations[26].version == 27` → `migrations[28].version == 29`; `applied_versions(…).last() == Some(27)` → `Some(29)` | | 2026-08-06 | 0028 → 0030 | `migrations[27].version == 28` → `migrations[29].version == 30`; `migrations.len()` 28 → 30; `applied_versions(…).last()` → `Some(30)` | +| 2026-08-13 | 0029 → 0031 **and** 0030 → 0032 | `migrations[28].version == 29` → `migrations[30].version == 31`; `migrations[29].version == 30` → `migrations[31].version == 32`; `migrations.len()` 30 → 32; **and a version-literal lookup**, `find(\|m\| m.version == 29)` → `31`, in `deletion_surface_parity_between_migration_0029_and_schema_sql` | Only `migrations.len()` arrives as a *conflict*; every indexed assertion arrives as clean context, which is why the diff will not point you at them. Grep the test module -for the old integer instead. A cheap independent check that the rename actually took: +for the old integer instead. + +**Grepping for the index is not enough — there is a third shape.** The 2026-08-13 +sync added `deletion_surface_parity_between_migration_0029_and_schema_sql`, which +resolves its migration with `MIGRATOR.iter().find(|m| m.version == 29)` rather than +by index. Fixing the count and both indexed assertions left it silently reading the +*fork's* 0029 (the channels index), where it found zero deletion tables and failed. +So sweep for all three: `migrations.len()`, `migrations[N]`, and `version == N`. +Unlike the indexed assertions this one does fail loudly, which is the only reason it +was caught — do not rely on that holding for the next one. + +That sync also renumbered **two** migrations at once, and upstream's second file +(`0030_community_deletion_recovery.sql`) landed exactly on the fork's own `0030`. +Renumber the whole upstream block in one pass and keep its internal order; the +recovery migration alters tables the deletion migration creates, so a reordering +that happens to dodge the collision would still fail at runtime. + +A cheap independent check that the rename actually took: ```bash ls migrations/*.sql | sed 's|.*/||' | cut -d_ -f1 | sort | uniq -d diff --git a/CHANGELOG.md b/CHANGELOG.md index 63783b36b8b..aa19e933d16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## v0.5.11 + +### Desktop and shared changes + +- perf(desktop): persist channel snapshot hash ([#5684](https://github.com/block/buzz/pull/5684)) ([`c86443c5997c96c42829ce200e73e6e6efe52d96`](https://github.com/block/buzz/commit/c86443c5997c96c42829ce200e73e6e6efe52d96)) +- fix(agent): raise output limit and allow 3 recoveries ([#5475](https://github.com/block/buzz/pull/5475)) ([`72d56e7bd3a94fa3ee20b5a50bd1b868a9329d9c`](https://github.com/block/buzz/commit/72d56e7bd3a94fa3ee20b5a50bd1b868a9329d9c)) +- fix(desktop): defer foreground resume work ([#5696](https://github.com/block/buzz/pull/5696)) ([`59f613c404958d8ac99525b4aaaf26843257de31`](https://github.com/block/buzz/commit/59f613c404958d8ac99525b4aaaf26843257de31)) +- perf(desktop): coalesce thread-activity localStorage writes ([#5693](https://github.com/block/buzz/pull/5693)) ([`c6c6e7eca70d6b526c43af925e596e8616b19fb8`](https://github.com/block/buzz/commit/c6c6e7eca70d6b526c43af925e596e8616b19fb8)) +- Batch observer-store publications per relay envelope ([#5680](https://github.com/block/buzz/pull/5680)) ([`c3b0ccf383fe4ee936abbe6b9c9134b5728cc2b5`](https://github.com/block/buzz/commit/c3b0ccf383fe4ee936abbe6b9c9134b5728cc2b5)) +- feat(buzz-acp): idle re-sleep for woken lazy pools ([#5682](https://github.com/block/buzz/pull/5682)) ([`dc2dbfe0f570abb818d3f3da8a71ea235555ed27`](https://github.com/block/buzz/commit/dc2dbfe0f570abb818d3f3da8a71ea235555ed27)) +- fix(desktop): preserve agent mention separator after send ([#5623](https://github.com/block/buzz/pull/5623)) ([`a8e5c89e23b85ee93306f2c3c11d8fe6300cd360`](https://github.com/block/buzz/commit/a8e5c89e23b85ee93306f2c3c11d8fe6300cd360)) +- fix(link-previews): proxy sent preview media ([#5627](https://github.com/block/buzz/pull/5627)) ([`884ed8a5d35dfba3892fc40437f39e08856dec7d`](https://github.com/block/buzz/commit/884ed8a5d35dfba3892fc40437f39e08856dec7d)) +- feat(deletion): add durable whole-community deletion ([#4425](https://github.com/block/buzz/pull/4425)) ([`8a2c9af2dbe0cf315e77f43a4560d3572da5e554`](https://github.com/block/buzz/commit/8a2c9af2dbe0cf315e77f43a4560d3572da5e554)) +- fix(desktop): preserve live channel timelines ([#5662](https://github.com/block/buzz/pull/5662)) ([`63d14a0e95c8d5ae19f3f80123027729ec209bb2`](https://github.com/block/buzz/commit/63d14a0e95c8d5ae19f3f80123027729ec209bb2)) +- Refine channel settings and profile panels ([#5574](https://github.com/block/buzz/pull/5574)) ([`63f961c7e4818a1d29f1185002c123e486bd4a19`](https://github.com/block/buzz/commit/63f961c7e4818a1d29f1185002c123e486bd4a19)) +- fix(deps): bump webbrowser to 1.2.4 for RUSTSEC-2026-0257 ([#5659](https://github.com/block/buzz/pull/5659)) ([`c966b862fe8b9018c68c384b1680ca0173d0128c`](https://github.com/block/buzz/commit/c966b862fe8b9018c68c384b1680ca0173d0128c)) +- fix(desktop): launch Databricks OAuth from passive model discovery ([#5607](https://github.com/block/buzz/pull/5607)) ([`1ff98fa685fdb7133dbc18437d23dcdeeb42ce6e`](https://github.com/block/buzz/commit/1ff98fa685fdb7133dbc18437d23dcdeeb42ce6e)) + +### Other repository changes + +- feat(acp): report standard adapter usage ([#4950](https://github.com/block/buzz/pull/4950)) ([`4749bc7be3cdb78c2db4ce4864775ba7ab60b4cc`](https://github.com/block/buzz/commit/4749bc7be3cdb78c2db4ce4864775ba7ab60b4cc)) +- fix(mobile): settle hydrated threads on latest reply ([#4702](https://github.com/block/buzz/pull/4702)) ([`7634fe74563ea7f3c86fb6017a0ad647a9934477`](https://github.com/block/buzz/commit/7634fe74563ea7f3c86fb6017a0ad647a9934477)) +- feat(acp): deliver channel description in prompt [Context] ([#4552](https://github.com/block/buzz/pull/4552)) ([`6e0631f6b5d2139e4e080bf94e27ecee8a3d4d74`](https://github.com/block/buzz/commit/6e0631f6b5d2139e4e080bf94e27ecee8a3d4d74)) + +[Compare desktop-v0.5.10...desktop-v0.5.11](https://github.com/block/buzz/compare/desktop-v0.5.10...desktop-v0.5.11) + ## v0.5.10 ### Desktop and shared changes diff --git a/Cargo.lock b/Cargo.lock index a512a8e2107..302f50f294a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -904,6 +904,7 @@ dependencies = [ "buzz-auth", "buzz-core", "buzz-db", + "buzz-deletion", "buzz-media", "buzz-pubsub", "buzz-search", @@ -1112,6 +1113,28 @@ dependencies = [ "uuid 1.23.1", ] +[[package]] +name = "buzz-deletion" +version = "0.1.0" +dependencies = [ + "anyhow", + "buzz-core", + "buzz-db", + "buzz-media", + "chrono", + "clap", + "deadpool-redis", + "hex", + "redis", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "uuid 1.23.1", +] + [[package]] name = "buzz-dev-mcp" version = "0.1.0" @@ -1306,6 +1329,7 @@ dependencies = [ "buzz-core", "buzz-datastore-tracing", "buzz-db", + "buzz-deletion", "buzz-media", "buzz-pubsub", "buzz-relay-mesh", @@ -1465,6 +1489,7 @@ version = "0.1.0" dependencies = [ "buzz-core", "buzz-db", + "buzz-deletion", "chrono", "cron", "dashmap", @@ -6511,6 +6536,17 @@ dependencies = [ "objc2-encode", ] +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags 2.13.0", + "objc2", + "objc2-foundation", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -11508,15 +11544,15 @@ dependencies = [ [[package]] name = "webbrowser" -version = "1.2.1" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc95580916af1e68ff6a7be07446fc5db73ebf71cf092de939bbf5f7e189f72" +checksum = "62c35be770821a214dbc362fc26908c853e776c0004294d0b10b8a6bad582f94" dependencies = [ - "core-foundation 0.10.1", "jni 0.22.4", "log", "ndk-context", "objc2", + "objc2-app-kit", "objc2-foundation", "url", "web-sys", diff --git a/Cargo.toml b/Cargo.toml index 7705a27f22a..ed6fe20945d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "crates/buzz-test-client", "crates/buzz-ws-client", "crates/buzz-admin", + "crates/buzz-deletion", "crates/buzz-workflow", "crates/buzz-media", "crates/buzz-paymaster", @@ -136,6 +137,7 @@ schemars = { version = "1", default-features = false } buzz-core = { path = "crates/buzz-core" } buzz-conformance = { path = "crates/buzz-conformance" } buzz-db = { path = "crates/buzz-db" } +buzz-deletion = { path = "crates/buzz-deletion" } buzz-auth = { path = "crates/buzz-auth" } buzz-pubsub = { path = "crates/buzz-pubsub" } buzz-search = { path = "crates/buzz-search" } diff --git a/crates/buzz-acp/src/acp.rs b/crates/buzz-acp/src/acp.rs index 8460372abab..f04b8eeec0d 100644 --- a/crates/buzz-acp/src/acp.rs +++ b/crates/buzz-acp/src/acp.rs @@ -14,7 +14,9 @@ use tokio::process::{Child, ChildStdin, ChildStdout}; use tokio_util::codec::{FramedRead, LinesCodec, LinesCodecError}; use crate::observer::{ObserverContext, ObserverHandle}; -use crate::usage::{TurnUsage, UsageTracker}; +use crate::usage::{ + PromptResponseUsage, StandardAdapterKind, StandardUsageTracker, TurnUsage, UsageTracker, +}; /// Maximum allowed size of a single NDJSON line from the agent's stdout. /// Lines exceeding this limit are rejected to prevent OOM from rogue agents. @@ -206,11 +208,12 @@ pub struct AcpClient { /// outside of a goose-native turn — the read loop's steer arm is /// disabled in that case. steer_rx: Option>, - /// Usage tracker — accumulates cumulative token counts from - /// `_goose/unstable/session/update` notifications and computes per-turn - /// deltas. Both goose and buzz-agent emit this notification; goose gates - /// on client capability advertisement, buzz-agent emits unconditionally. + /// Usage tracker for goose/buzz-agent's cumulative notification format. goose_usage: UsageTracker, + /// Per-turn prompt-response usage and Claude's optional cumulative cost. + standard_usage: StandardUsageTracker, + /// Known adapter identity for prompt-response usage mapping. + standard_adapter: Option, } /// Recursively merge `overlay` into `base`, with `overlay` winning on scalar/shape @@ -523,6 +526,14 @@ impl AcpClient { // console-subsystem child process spawned from a GUI/non-console parent. configure_no_window(&mut cmd); + let standard_adapter = + match crate::config::normalize_agent_command_identity(command).as_str() { + "claude-agent-acp" | "claude-code-acp" | "claude-code" | "claudecode" => { + Some(StandardAdapterKind::Claude) + } + "codex" | "codex-acp" => Some(StandardAdapterKind::Codex), + _ => None, + }; let mut child = cmd.spawn()?; let stdin = child @@ -550,6 +561,8 @@ impl AcpClient { steering_supported: false, steer_rx: None, goose_usage: UsageTracker::default(), + standard_usage: StandardUsageTracker::default(), + standard_adapter, }) } @@ -776,6 +789,7 @@ impl AcpClient { // prompt so that any setup notifications recorded earlier are not // misattributed to this turn. self.goose_usage.begin_turn(session_id); + self.standard_usage.begin_turn(session_id); self.last_prompt_id = Some(self.next_id); let id = self.next_id; @@ -821,7 +835,7 @@ impl AcpClient { self.current_hard_deadline = None; } } - self.parse_stop_reason(&result?) + self.parse_prompt_response(session_id, &result?) } /// Send a `session/cancel` **notification** (no `id` field, no response expected). @@ -867,18 +881,13 @@ impl AcpClient { self.steering_supported } - /// Consume and return the per-turn usage record computed from the most - /// recent `_goose/unstable/session/update` notification. - /// - /// Returns `None` if no usage update arrived since the last call (i.e. - /// the harness did not emit one for this turn, or this is not a goose - /// agent). Must be called at most once per turn; subsequent calls return - /// `None` until the next `usage_update` notification is recorded. - /// - /// Intended for consumption by `publish_agent_turn_metric` in `pool.rs` to - /// publish a kind 44200 NIP-AM event. + /// Consume per-turn usage for NIP-AM publishing. Goose/buzz-agent is an + /// exclusive cumulative path; standard ACP prompt usage is used only when + /// goose emitted nothing for this turn. pub fn take_turn_usage(&mut self) -> Option { - self.goose_usage.take() + let goose_usage = self.goose_usage.take(); + let standard_usage = self.standard_usage.take(); + goose_usage.or(standard_usage) } /// Notify the usage tracker that buzz-acp just spawned a new session. @@ -889,6 +898,7 @@ impl AcpClient { /// never when attaching to a pre-existing session. pub(crate) fn notify_session_spawned(&mut self, session_id: &str) { self.goose_usage.seed_zero_baseline(session_id); + self.standard_usage.seed_zero_baseline(session_id); } /// Install a per-turn steer request channel for goose-native @@ -1048,7 +1058,7 @@ impl AcpClient { remaining, ) .await?; - self.parse_stop_reason(&result) + self.parse_prompt_response(session_id, &result) } /// Serialize `value` as a single NDJSON line and flush to the agent's stdin. @@ -1831,6 +1841,10 @@ impl AcpClient { } false } + "usage_update" => { + self.handle_standard_usage_update(msg); + false + } "keepalive" => false, other => { tracing::debug!(target: "acp::update", "session/update: {other}"); @@ -1839,6 +1853,30 @@ impl AcpClient { } } + /// Record the standard ACP cumulative cost notification when emitted by + /// Claude. Unlike Goose's payload, `used`/`size` are context occupancy and + /// are intentionally not mapped to token accounting. + fn handle_standard_usage_update(&mut self, msg: &serde_json::Value) { + if self.standard_adapter != Some(StandardAdapterKind::Claude) { + return; + } + let session_id = match msg + .pointer("/params/sessionId") + .and_then(serde_json::Value::as_str) + { + Some(session_id) => session_id, + None => return, + }; + let cost = match msg + .pointer("/params/update/cost/amount") + .and_then(serde_json::Value::as_f64) + { + Some(cost) => cost, + None => return, + }; + self.standard_usage.record_cost(session_id, cost); + } + /// Parse a `_goose/unstable/session/update` notification and record the /// usage snapshot in the per-session tracker. /// @@ -1970,6 +2008,28 @@ impl AcpClient { Ok(()) } + /// Parse a completed prompt response and retain its optional per-turn usage. + fn parse_prompt_response( + &mut self, + session_id: &str, + result: &serde_json::Value, + ) -> Result { + let stop_reason = self.parse_stop_reason(result)?; + if let Some(adapter) = self.standard_adapter { + match serde_json::from_value::(result["usage"].clone()) { + Ok(usage) => self + .standard_usage + .record_prompt_usage(session_id, usage, adapter), + Err(_) if result.get("usage").is_some() => tracing::debug!( + target: "acp::usage", + "session/prompt response contained malformed standard usage" + ), + Err(_) => {} + } + } + Ok(stop_reason) + } + /// Parse `stopReason` from a `session/prompt` result value. fn parse_stop_reason(&self, result: &serde_json::Value) -> Result { let raw = result["stopReason"].as_str().ok_or_else(|| { @@ -2899,6 +2959,30 @@ mod tests { .expect("failed to spawn test script") } + #[cfg(unix)] + async fn spawn_named_script(name: &str, script: &str) -> (AcpClient, std::path::PathBuf) { + use std::os::unix::fs::PermissionsExt; + + let dir = std::env::temp_dir().join(format!( + "buzz-acp-{name}-{}-{}", + std::process::id(), + uuid::Uuid::new_v4() + )); + std::fs::create_dir_all(&dir).expect("create temp adapter dir"); + let path = dir.join(name); + std::fs::write(&path, format!("#!/usr/bin/env bash\n{script}\n")) + .expect("write fake adapter"); + let mut permissions = std::fs::metadata(&path) + .expect("adapter metadata") + .permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&path, permissions).expect("chmod fake adapter"); + let client = AcpClient::spawn(path.to_str().expect("utf8 path"), &[], &[], false) + .await + .expect("spawn named fake adapter"); + (client, dir) + } + /// Spawn a probe script whose file name carries a runtime identity (e.g. /// `hermes-acp`) and return the value of `var` as the child observed it. /// `` means the child did not receive the var. @@ -4275,6 +4359,254 @@ mod tests { } } + // ── Standard ACP prompt-response usage ───────────────────────────────── + + fn prompt_response_usage( + input: u64, + output: u64, + total: u64, + cached_read: Option, + cached_write: Option, + ) -> serde_json::Value { + let mut usage = serde_json::json!({ + "inputTokens": input, + "outputTokens": output, + "totalTokens": total, + }); + if let Some(cached_read) = cached_read { + usage["cachedReadTokens"] = serde_json::json!(cached_read); + } + if let Some(cached_write) = cached_write { + usage["cachedWriteTokens"] = serde_json::json!(cached_write); + } + serde_json::json!({"stopReason": "end_turn", "usage": usage}) + } + + fn standard_cost_update(session_id: &str, cost: f64) -> serde_json::Value { + serde_json::json!({ + "jsonrpc": "2.0", + "method": "session/update", + "params": { + "sessionId": session_id, + "update": { + "sessionUpdate": "usage_update", + "cost": {"amount": cost, "currency": "USD"} + } + } + }) + } + + #[tokio::test] + async fn claude_prompt_response_usage_merges_with_cumulative_cost() { + let mut client = spawn_inert_client().await; + client.standard_adapter = Some(StandardAdapterKind::Claude); + client.notify_session_spawned("claude-session"); + client.standard_usage.begin_turn("claude-session"); + client.handle_session_update(&standard_cost_update("claude-session", 0.042)); + assert_eq!( + client + .parse_prompt_response( + "claude-session", + &prompt_response_usage(100, 20, 175, Some(30), Some(25)), + ) + .unwrap(), + StopReason::EndTurn + ); + + let usage = client.take_turn_usage().expect("prompt usage"); + assert!(usage.delta_reliable, "response tokens need no baseline"); + assert_eq!(usage.turn_input_tokens, Some(155)); + assert_eq!(usage.turn_output_tokens, Some(20)); + assert_eq!( + usage.turn_total_tokens, None, + "Claude total is adapter-derived" + ); + assert_eq!(usage.turn_cache_read_tokens, Some(30)); + assert_eq!(usage.turn_cache_write_tokens, Some(25)); + assert_eq!(usage.turn_cost_usd, Some(0.042)); + assert_eq!(usage.cumulative_cost_usd, Some(0.042)); + assert_eq!(usage.cumulative_input_tokens, None); + assert_eq!(usage.cumulative_output_tokens, None); + } + + #[tokio::test] + async fn codex_prompt_response_usage_preserves_provider_total_without_cost() { + let mut client = spawn_inert_client().await; + client.standard_adapter = Some(StandardAdapterKind::Codex); + client.standard_usage.begin_turn("codex-session"); + client.handle_session_update(&standard_cost_update("codex-session", 0.042)); + client + .parse_prompt_response( + "codex-session", + &prompt_response_usage(90, 10, 140, Some(40), None), + ) + .unwrap(); + + let usage = client.take_turn_usage().expect("prompt usage"); + assert!(usage.delta_reliable); + assert_eq!(usage.turn_input_tokens, Some(130)); + assert_eq!(usage.turn_output_tokens, Some(10)); + assert_eq!(usage.turn_total_tokens, Some(140)); + assert_eq!(usage.turn_cache_read_tokens, Some(40)); + assert_eq!(usage.turn_cache_write_tokens, None); + assert_eq!( + usage.cumulative_cost_usd, None, + "Codex cost update is ignored" + ); + assert_eq!(usage.cumulative_input_tokens, None); + assert_eq!(usage.cumulative_output_tokens, None); + } + + #[tokio::test] + async fn standard_prompt_input_overflow_fails_closed() { + let mut client = spawn_inert_client().await; + client.standard_adapter = Some(StandardAdapterKind::Claude); + client.standard_usage.begin_turn("overflow-session"); + client + .parse_prompt_response( + "overflow-session", + &prompt_response_usage(u64::MAX, 10, u64::MAX, Some(1), None), + ) + .unwrap(); + + assert!( + client.take_turn_usage().is_none(), + "overflow without another valid signal must not emit all-null usage" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn claude_named_adapter_wire_lifecycle_records_prompt_and_cost() { + let script = r#" + read -r REQ + ID=$(printf '%s' "$REQ" | sed -E 's/.*"id":([0-9]+).*/\1/') + echo '{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"wire-session","update":{"sessionUpdate":"usage_update","cost":{"amount":0.5,"currency":"USD"}}}}' + echo '{"jsonrpc":"2.0","id":'"$ID"',"result":{"stopReason":"end_turn","usage":{"inputTokens":7,"outputTokens":3,"totalTokens":10,"cachedReadTokens":2}}}' + sleep 1 + "#; + let (mut client, dir) = spawn_named_script("claude-code", script).await; + assert_eq!(client.standard_adapter, Some(StandardAdapterKind::Claude)); + client.notify_session_spawned("wire-session"); + + let stop = client + .session_prompt_with_idle_timeout( + "wire-session", + "hello", + std::time::Duration::from_secs(2), + std::time::Duration::from_secs(5), + ) + .await + .expect("wire prompt"); + assert_eq!(stop, StopReason::EndTurn); + + let usage = client.take_turn_usage().expect("wire usage"); + assert_eq!(usage.turn_seq, 1); + assert_eq!(usage.turn_input_tokens, Some(9)); + assert_eq!(usage.turn_output_tokens, Some(3)); + assert_eq!(usage.turn_cost_usd, Some(0.5)); + assert_eq!(usage.cumulative_cost_usd, Some(0.5)); + drop(client); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn claude_cost_only_record_survives_missing_prompt_usage() { + let mut client = spawn_inert_client().await; + client.standard_adapter = Some(StandardAdapterKind::Claude); + client.notify_session_spawned("cost-only-session"); + client.standard_usage.begin_turn("cost-only-session"); + client.handle_session_update(&standard_cost_update("cost-only-session", 0.125)); + + let usage = client.take_turn_usage().expect("cost-only usage"); + assert_eq!(usage.turn_seq, 1); + assert!(usage.delta_reliable); + assert_eq!(usage.turn_input_tokens, None); + assert_eq!(usage.turn_cost_usd, Some(0.125)); + assert_eq!(usage.cumulative_cost_usd, Some(0.125)); + } + + #[tokio::test] + async fn attached_claude_session_does_not_invent_first_cost_delta() { + let mut client = spawn_inert_client().await; + client.standard_adapter = Some(StandardAdapterKind::Claude); + client.standard_usage.begin_turn("attached-session"); + client.handle_session_update(&standard_cost_update("attached-session", 1.25)); + client + .parse_prompt_response( + "attached-session", + &prompt_response_usage(10, 2, 12, None, None), + ) + .unwrap(); + + let usage = client.take_turn_usage().expect("attached usage"); + assert_eq!(usage.turn_cost_usd, None); + assert_eq!(usage.cumulative_cost_usd, Some(1.25)); + } + + #[tokio::test] + async fn standard_usage_two_prompts_preserve_both_monotonic_sequences() { + let mut client = spawn_inert_client().await; + client.standard_adapter = Some(StandardAdapterKind::Claude); + client.notify_session_spawned("two-prompt-session"); + + client.standard_usage.begin_turn("two-prompt-session"); + client.handle_session_update(&standard_cost_update("two-prompt-session", 0.1)); + client + .parse_prompt_response( + "two-prompt-session", + &prompt_response_usage(10, 2, 12, None, None), + ) + .unwrap(); + let initial = client.take_turn_usage().expect("initial prompt usage"); + + client.standard_usage.begin_turn("two-prompt-session"); + client.handle_session_update(&standard_cost_update("two-prompt-session", 0.25)); + client + .parse_prompt_response( + "two-prompt-session", + &prompt_response_usage(20, 3, 23, None, None), + ) + .unwrap(); + let user = client.take_turn_usage().expect("user prompt usage"); + + assert_eq!((initial.turn_seq, user.turn_seq), (1, 2)); + assert_eq!( + (initial.turn_input_tokens, user.turn_input_tokens), + (Some(10), Some(20)) + ); + assert_eq!( + (initial.turn_cost_usd, user.turn_cost_usd), + (Some(0.1), Some(0.15)) + ); + } + + #[tokio::test] + async fn goose_usage_stays_exclusive_and_drains_standard_usage() { + let mut client = spawn_inert_client().await; + client.standard_adapter = Some(StandardAdapterKind::Claude); + client.goose_usage.begin_turn("goose-session"); + client.standard_usage.begin_turn("goose-session"); + client.handle_goose_usage_update(&goose_usage_update_msg("goose-session", 1000, 200, None)); + client + .parse_prompt_response( + "goose-session", + &prompt_response_usage(100, 20, 120, None, None), + ) + .unwrap(); + + let usage = client.take_turn_usage().expect("goose usage"); + assert_eq!(usage.cumulative_input_tokens, Some(1000)); + assert_eq!( + usage.turn_input_tokens, None, + "goose first delta remains exclusive" + ); + assert!( + client.take_turn_usage().is_none(), + "standard usage was drained" + ); + } + // ── Goose usage notification integration ────────────────────────────── /// Build a `_goose/unstable/session/update` JSON-RPC notification. diff --git a/crates/buzz-acp/src/config.rs b/crates/buzz-acp/src/config.rs index 35aaec188db..f9e7bf1ed8a 100644 --- a/crates/buzz-acp/src/config.rs +++ b/crates/buzz-acp/src/config.rs @@ -482,6 +482,13 @@ pub struct CliArgs { /// Connect and subscribe before starting the ACP/LLM subprocess pool. #[arg(long, env = "BUZZ_ACP_LAZY_POOL", default_value_t = false)] pub lazy_pool: bool, + + /// Tear the woken pool back down to the lazy empty-slot state after this + /// many seconds with no dispatched turn in flight and an empty queue, + /// releasing worker subprocesses until the next accepted event re-wakes. + /// Requires `--lazy-pool`; ignored otherwise. 0 disables idle re-sleep. + #[arg(long, env = "BUZZ_ACP_IDLE_POOL_SLEEP", default_value_t = 0)] + pub idle_pool_sleep: u64, } /// Merged NIP-01 subscription filter for a single channel. @@ -559,6 +566,10 @@ pub struct Config { pub exit_after_inactivity_secs: u64, /// Whether ACP/LLM subprocess initialization is deferred until accepted work arrives. pub lazy_pool: bool, + /// Seconds with no dispatched turn in flight and an empty queue before a + /// woken lazy pool is torn back down to the empty-slot state. 0 = disabled. + /// Only meaningful when `lazy_pool` is true. + pub idle_pool_sleep_secs: u64, /// Agent owner pubkey (hex). Used for `--respond-to=owner-only` gate. /// Replaces the old REST-based owner lookup. pub agent_owner: Option, @@ -1107,6 +1118,7 @@ impl Config { relay_observer: args.relay_observer, exit_after_inactivity_secs: args.exit_after_inactivity, lazy_pool: args.lazy_pool, + idle_pool_sleep_secs: args.idle_pool_sleep, agent_owner: args.agent_owner.map(|s| s.trim().to_ascii_lowercase()), no_base_prompt: args.no_base_prompt, base_prompt_content, @@ -1478,6 +1490,7 @@ mod tests { relay_observer: false, exit_after_inactivity_secs: 0, lazy_pool: false, + idle_pool_sleep_secs: 0, agent_owner: None, no_base_prompt: false, base_prompt_content: None, @@ -2198,6 +2211,22 @@ channels = "ALL" assert!(!CliArgs::parse_from(["buzz-acp", "--private-key", &key]).lazy_pool); } + #[test] + fn idle_pool_sleep_defaults_disabled_and_accepts_cli_value() { + let key = "0".repeat(64); + let default = CliArgs::parse_from(["buzz-acp", "--private-key", &key]); + assert_eq!(default.idle_pool_sleep, 0); + + let configured = CliArgs::parse_from([ + "buzz-acp", + "--private-key", + &key, + "--idle-pool-sleep", + "300", + ]); + assert_eq!(configured.idle_pool_sleep, 300); + } + #[test] fn lazy_pool_cli_flag_enables_deferred_startup() { let key = "0".repeat(64); diff --git a/crates/buzz-acp/src/lib.rs b/crates/buzz-acp/src/lib.rs index fa348eeb3cf..27b9000b7bb 100644 --- a/crates/buzz-acp/src/lib.rs +++ b/crates/buzz-acp/src/lib.rs @@ -1468,6 +1468,33 @@ fn inactivity_expired( !bound.is_zero() && !turn_in_flight && now.duration_since(last_activity) >= bound } +/// Whether a woken lazy pool may be torn back down to the empty-slot state. +/// +/// True only when the pool is ready, the idle bound has elapsed with no +/// dispatched turn or heartbeat in flight and no in-flight prompt tasks, no +/// work is queued, and no wake/respawn task is running. The queue and task +/// gates make teardown race-safe with enqueue/wake: an event that landed in +/// the queue (or a wake/respawn already in flight) blocks this decision, so a +/// queued batch is never stranded — the caller's next loop iteration will +/// dispatch or wake it instead. +#[allow(clippy::too_many_arguments)] +fn idle_pool_sleep_due( + pool_ready: bool, + last_activity: tokio::time::Instant, + now: tokio::time::Instant, + bound: Duration, + turn_in_flight: bool, + prompt_tasks_in_flight: bool, + work_queued: bool, + wake_or_respawn_in_flight: bool, +) -> bool { + pool_ready + && !work_queued + && !prompt_tasks_in_flight + && !wake_or_respawn_in_flight + && inactivity_expired(last_activity, now, bound, turn_in_flight) +} + #[cfg(test)] mod inactivity_tests { use super::*; @@ -1512,6 +1539,179 @@ mod inactivity_tests { } } +#[cfg(test)] +mod idle_pool_sleep_tests { + use super::*; + + // The all-clear baseline: pool ready, bound elapsed, nothing busy or + // queued. Every negative case below flips exactly one gate off this. + fn ready_after_bound() -> (tokio::time::Instant, tokio::time::Instant, Duration) { + let started = tokio::time::Instant::now(); + ( + started, + started + Duration::from_secs(61), + Duration::from_secs(60), + ) + } + + #[test] + fn sleeps_when_ready_idle_and_quiet() { + let (last, now, bound) = ready_after_bound(); + assert!(idle_pool_sleep_due( + true, last, now, bound, false, false, false, false + )); + } + + #[test] + fn zero_bound_never_sleeps() { + let (last, now, _) = ready_after_bound(); + assert!(!idle_pool_sleep_due( + true, + last, + now, + Duration::ZERO, + false, + false, + false, + false + )); + } + + #[test] + fn not_ready_never_sleeps() { + // A still-sleeping (or waking) pool must not "re-sleep". + let (last, now, bound) = ready_after_bound(); + assert!(!idle_pool_sleep_due( + false, last, now, bound, false, false, false, false + )); + } + + #[test] + fn active_turn_defers_sleep() { + let (last, now, bound) = ready_after_bound(); + assert!(!idle_pool_sleep_due( + true, last, now, bound, true, false, false, false + )); + } + + #[test] + fn in_flight_prompt_task_defers_sleep() { + let (last, now, bound) = ready_after_bound(); + assert!(!idle_pool_sleep_due( + true, last, now, bound, false, true, false, false + )); + } + + #[test] + fn queued_work_at_boundary_defers_sleep() { + // Enqueue-at-teardown protection: a batch sitting in the queue blocks + // teardown so it is never stranded — the loop dispatches it instead. + let (last, now, bound) = ready_after_bound(); + assert!(!idle_pool_sleep_due( + true, last, now, bound, false, false, true, false + )); + } + + #[test] + fn wake_or_respawn_in_flight_defers_sleep() { + let (last, now, bound) = ready_after_bound(); + assert!(!idle_pool_sleep_due( + true, last, now, bound, false, false, false, true + )); + } + + #[test] + fn recent_activity_defers_sleep() { + // Activity 50s ago under a 60s bound: not yet idle. + let started = tokio::time::Instant::now(); + let recent = started + Duration::from_secs(50); + let now = started + Duration::from_secs(59); + assert!(!idle_pool_sleep_due( + true, + recent, + now, + Duration::from_secs(60), + false, + false, + false, + false + )); + } + + fn slot(respawn_in_flight: bool) -> SlotCircuit { + SlotCircuit { + crash_times: Vec::new(), + open_until: None, + respawn_in_flight, + } + } + + // The call-site signal for the `wake_or_respawn_in_flight` gate is + // `any_respawn_in_flight(&crash_history)`, NOT `!respawn_tasks.is_empty()`. + // Regression for the PR #5682 review blocker: completed respawn tasks are + // never joined from the `respawn_tasks` JoinSet (their payloads arrive + // out-of-band via `respawn_rx`), so `!is_empty()` stays true forever after + // the first refill/crash recovery and the pool could never re-sleep. The + // authoritative signal clears per-slot when the payload is received. + #[test] + fn respawn_in_flight_signal_gates_then_clears_for_sleep() { + let (last, now, bound) = ready_after_bound(); + + // A respawn in flight for any slot defers sleep. + let busy = [slot(false), slot(true), slot(false)]; + assert!(any_respawn_in_flight(&busy)); + assert!(!idle_pool_sleep_due( + true, + last, + now, + bound, + false, + false, + false, + any_respawn_in_flight(&busy), + )); + + // Once the respawn completes (payload received → flag cleared), the + // signal goes false and the otherwise-quiet pool becomes sleep-eligible + // — even though a naive `!JoinSet.is_empty()` would still be stuck true. + let quiet = [slot(false), slot(false), slot(false)]; + assert!(!any_respawn_in_flight(&quiet)); + assert!(idle_pool_sleep_due( + true, + last, + now, + bound, + false, + false, + false, + any_respawn_in_flight(&quiet), + )); + } + + // The reaper (`respawn_tasks.join_next().now_or_never()` loop) must drain + // completed handles so the JoinSet does not grow without bound and so + // `!respawn_tasks.is_empty()` cannot become a permanent busy bit if anyone + // ever reintroduces it as the gate signal. + #[tokio::test] + async fn completed_respawn_tasks_are_reaped_from_the_joinset() { + let mut respawn_tasks: tokio::task::JoinSet<()> = tokio::task::JoinSet::new(); + respawn_tasks.spawn(async {}); + respawn_tasks.spawn(async {}); + // Let both tasks run to completion. + tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(10)).await; + + // The reaper drains finished handles non-blockingly. + while respawn_tasks.join_next().now_or_never().flatten().is_some() {} + + assert!( + respawn_tasks.is_empty(), + "completed respawn tasks must be reaped so the set does not wedge \ + the idle-sleep gate or grow unbounded" + ); + } +} + pub fn run() -> Result<()> { config::propagate_legacy_env_vars(); tokio_main() @@ -1900,6 +2100,27 @@ async fn tokio_main() -> Result<()> { )) }; + // Idle pool re-sleep: tear a woken lazy pool back down to the empty-slot + // state after `idle_pool_sleep_bound` of quiet, releasing worker + // subprocesses. The next accepted event re-wakes it through the same lazy + // path. Only meaningful under `lazy_pool`; the tick arm additionally gates + // on `pool_ready`, so a still-sleeping pool never re-sleeps. Reuses the + // `last_activity` clock the dispatch path already maintains. + let idle_pool_sleep_bound = if config.lazy_pool { + Duration::from_secs(config.idle_pool_sleep_secs) + } else { + Duration::ZERO + }; + let mut idle_pool_sleep_reaper = if idle_pool_sleep_bound.is_zero() { + None + } else { + let interval = idle_pool_sleep_bound.min(Duration::from_secs(30)); + Some(tokio::time::interval_at( + tokio::time::Instant::now() + interval, + interval, + )) + }; + // Runs at the TOP of every loop iteration via Instant check — cannot be // starved by the biased select. Slot refill spawns background tasks so // spawn_and_init never blocks the main loop. @@ -2107,6 +2328,17 @@ async fn tokio_main() -> Result<()> { } } } + // Reap completed respawn handles from the JoinSet. Payloads are + // delivered out-of-band through `respawn_rx` (drained above), so the + // JoinSet is never joined by the normal flow — Tokio retains finished + // tasks until `join_next`, so without this the set grows on every + // refill/crash recovery and `!respawn_tasks.is_empty()` would stay true + // forever. Non-blocking (`now_or_never`), same pattern as + // `drain_ready_join_results` for `pool.join_set`. The authoritative + // in-flight signal is `any_respawn_in_flight(&crash_history)` (each + // slot's `respawn_in_flight` is cleared when its payload is received), + // not JoinSet occupancy. + while respawn_tasks.join_next().now_or_never().flatten().is_some() {} // Flush requeued events that were waiting for a live agent. Without // this, batches requeued during crash recovery sit idle until the // next relay event arrives — which can be minutes on quiet channels. @@ -2599,6 +2831,56 @@ async fn tokio_main() -> Result<()> { } None } + _ = async { + match idle_pool_sleep_reaper.as_mut() { + Some(timer) => timer.tick().await, + None => std::future::pending().await, + } + } => { + let _ = result_rx; // end split borrow before touching pool + // A wake in flight (pool not yet ready) is covered by the + // pool_ready gate; respawn tasks and in-flight prompt tasks + // are the remaining "busy" signals. Never sleep mid-work: + // `has_undispatched_work()` (not `has_flushable_work()`) + // keeps `work_queued` true for a retry-throttled batch too, + // so a failed turn awaiting backoff is never stranded — the + // next iteration dispatches or re-wakes it. + if idle_pool_sleep_due( + pool_ready, + last_activity, + tokio::time::Instant::now(), + idle_pool_sleep_bound, + queue.has_in_flight() || heartbeat_in_flight, + !pool.join_set.is_empty(), + queue.has_undispatched_work(), + !wake_tasks.is_empty() + || any_respawn_in_flight(&crash_history), + ) { + tracing::info!( + idle_pool_sleep_seconds = config.idle_pool_sleep_secs, + "idle pool sleep bound reached — tearing pool back to lazy state" + ); + shutdown_agent_pool(&mut pool).await; + // Return to the exact pre-wake lazy state: empty slots, + // Listening lifecycle. The top-of-loop wake path re-wakes + // on the next accepted event. No second lifecycle. + pool = AgentPool::from_slots( + (0..config.agents).map(|_| None).collect(), + ); + pool_ready = false; + pool_lifecycle = PoolLifecycle::listening(); + last_activity = tokio::time::Instant::now(); + emit_runtime_lifecycle( + observer.as_ref(), + &runtime_start_nonce, + &pubkey_hex, + &config.relay_url, + "listening", + None, + ); + } + None + } _ = async { match heartbeat.as_mut() { Some(hb) => hb.tick().await, @@ -5031,6 +5313,7 @@ mod author_gate_tests { relay::ChannelInfo { name: "dm".into(), channel_type: "dm".into(), + description: None, }, ), ( @@ -5038,6 +5321,7 @@ mod author_gate_tests { relay::ChannelInfo { name: "stream".into(), channel_type: "stream".into(), + description: None, }, ), ]); @@ -5054,6 +5338,7 @@ mod author_gate_tests { relay::ChannelInfo { name: "unknown".into(), channel_type: "unknown".into(), + description: None, }, )]); assert!( @@ -6249,6 +6534,7 @@ mod build_mcp_servers_tests { relay_observer: false, exit_after_inactivity_secs: 0, lazy_pool: false, + idle_pool_sleep_secs: 0, agent_owner: None, no_base_prompt: false, base_prompt_content: None, @@ -6471,6 +6757,7 @@ mod error_outcome_emission_tests { relay_observer: false, exit_after_inactivity_secs: 0, lazy_pool: false, + idle_pool_sleep_secs: 0, agent_owner: None, no_base_prompt: false, base_prompt_content: None, diff --git a/crates/buzz-acp/src/pool.rs b/crates/buzz-acp/src/pool.rs index 33bd5507fb3..e38fa9b83e4 100644 --- a/crates/buzz-acp/src/pool.rs +++ b/crates/buzz-acp/src/pool.rs @@ -524,6 +524,7 @@ impl ChannelInfoResolver { PromptChannelInfo { name: info.name, channel_type: info.channel_type, + description: info.description, }, )) }) @@ -1849,6 +1850,16 @@ pub async fn run_prompt_task( if !agent.has_system_prompt_support() { agent.state.mark_channel_delivery_success(*cid, true, []); } + let usage = agent.acp.take_turn_usage(); + publish_agent_turn_metric( + &ctx, + usage, + Some(*cid), + &session_id, + &format!("{turn_id}:initial"), + Some(acp_stop_to_core(&stop_reason)), + ) + .await; } Err(AcpError::AgentExited) => { agent.state.invalidate_all(); @@ -1873,7 +1884,17 @@ pub async fn run_prompt_task( .cancel_with_cleanup(&session_id, ctx.idle_timeout) .await { - Ok(_) => { + Ok(stop_reason) => { + let usage = agent.acp.take_turn_usage(); + publish_agent_turn_metric( + &ctx, + usage, + Some(*cid), + &session_id, + &format!("{turn_id}:initial"), + Some(acp_stop_to_core(&stop_reason)), + ) + .await; agent.state.invalidate(&source); } Err(AcpError::AgentExited) => { @@ -2577,17 +2598,25 @@ pub(crate) async fn fetch_channel_info( let ev = events.first()?; let tags = ev.get("tags")?.as_array()?; let mut name = None; + let mut description = None; for tag in tags { if let Some(arr) = tag.as_array() { - if arr.first().and_then(|v| v.as_str()) == Some("name") { - name = arr.get(1).and_then(|v| v.as_str()); + match arr.first().and_then(|v| v.as_str()) { + Some("name") => name = arr.get(1).and_then(|v| v.as_str()), + Some("about") => description = arr.get(1).and_then(|v| v.as_str()), + _ => {} } } } let channel_type = crate::relay::channel_type_from_tags(tags); + let description = description + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(str::to_string); Some(PromptChannelInfo { name: name.unwrap_or(UNKNOWN_CHANNEL_NAME).to_string(), channel_type, + description, }) } Ok(Err(e)) => { @@ -5803,6 +5832,7 @@ done"# crate::relay::ChannelInfo { name: "test-dm".into(), channel_type: "dm".into(), + description: None, }, )]), RestClient { @@ -5964,6 +5994,7 @@ printf '%s\n' '{{"jsonrpc":"2.0","id":0,"result":{{"stopReason":"end_turn"}}}}'" crate::relay::ChannelInfo { name: "test-dm".into(), channel_type: "dm".into(), + description: None, }, )]), RestClient { @@ -7855,6 +7886,38 @@ printf '%s\n' '{{"jsonrpc":"2.0","id":0,"result":{{"stopReason":"end_turn"}}}}'" server.abort(); } + /// A channel's `about` tag is parsed through the lazy-fetch path and + /// delivered as the resolved description. + #[tokio::test] + async fn test_channel_resolver_delivers_description() { + let id = Uuid::new_v4(); + let response = channel_metadata_response( + id, + &[ + ["name", "team-chat"], + ["t", "stream"], + ["about", "Engineering discussions"], + ], + ); + let (resolver, _requests, server) = counting_resolver(response).await; + + let info = resolver.resolve(id).await.expect("should resolve"); + assert_eq!(info.description.as_deref(), Some("Engineering discussions")); + server.abort(); + } + + /// A metadata event with no `about` tag yields no description. + #[tokio::test] + async fn test_channel_resolver_absent_description_when_no_about_tag() { + let id = Uuid::new_v4(); + let response = channel_metadata_response(id, &[["name", "buzz-dev"], ["t", "stream"]]); + let (resolver, _requests, server) = counting_resolver(response).await; + + let info = resolver.resolve(id).await.expect("should resolve"); + assert_eq!(info.description, None); + server.abort(); + } + /// A DM carries no useful name, so it gets the bare agent title (and no /// canvas section). #[tokio::test] diff --git a/crates/buzz-acp/src/queue.rs b/crates/buzz-acp/src/queue.rs index 3bf19622429..dabee13afd5 100644 --- a/crates/buzz-acp/src/queue.rs +++ b/crates/buzz-acp/src/queue.rs @@ -590,6 +590,39 @@ impl EventQueue { .any(|id| !self.in_flight_channels.contains(id)) } + /// Returns `true` if any undispatched work remains for a channel that is + /// NOT currently in-flight — *including* work held back only by a + /// `retry_after` backoff throttle. + /// + /// This is deliberately broader than [`has_flushable_work`](Self::has_flushable_work): + /// that method excludes `retry_after`-throttled channels because they are + /// not flushable *right now*, but the events are still queued and MUST be + /// delivered once the backoff deadline passes. Idle-pool-sleep teardown + /// must gate on this, not on flushability — a failed turn requeued with a + /// future backoff deadline is real queued work, and sleeping on it (while + /// the maintenance timer is disabled and lazy re-wake is itself gated by + /// flushability) would strand the batch until unrelated traffic arrives. + /// + /// Covers the three tables where undispatched, non-in-flight work can + /// live: non-empty `queues` (throttled or not), pending `cancelled_batches`, + /// and `withheld_native_steer` events. Read-only (no in-flight expiry) — + /// in-flight liveness is gated separately by [`has_in_flight`](Self::has_in_flight). + pub fn has_undispatched_work(&self) -> bool { + let has_queued = self + .queues + .iter() + .any(|(id, q)| !q.is_empty() && !self.in_flight_channels.contains(id)); + let has_cancelled = self + .cancelled_batches + .keys() + .any(|id| !self.in_flight_channels.contains(id)); + let has_withheld = self + .withheld_native_steer + .iter() + .any(|(id, v)| !v.is_empty() && !self.in_flight_channels.contains(id)); + has_queued || has_cancelled || has_withheld + } + /// Number of channels with pending events. pub fn pending_channels(&self) -> usize { self.queues.len() @@ -1003,6 +1036,8 @@ pub struct ContextMessage { pub struct PromptChannelInfo { pub name: String, pub channel_type: String, + /// Channel description from the kind-39000 `about` tag, if present. + pub description: Option, } /// Minimal profile fields needed to label users in ACP prompts. @@ -1231,6 +1266,48 @@ fn resolve_reply_anchor( ) } +/// Maximum length (in characters) of a channel description rendered into `[Context]`. +/// +/// Limits prompt bloat from unusually long descriptions; a raw embedded newline +/// in a description must not be able to spoof another `[Context]` field, so +/// multiline text is collapsed to single-space-joined lines before truncation. +const MAX_DESCRIPTION_LEN: usize = 500; + +/// Append a `Description: …` line to a `[Context]` block when non-empty. +/// +/// Collapses internal newlines (any `\r\n`, `\r`, or `\n`) to a single space +/// so a multi-line description cannot inject a fake `[Context]` field line. +/// Truncates at [`MAX_DESCRIPTION_LEN`] characters with a `…` marker. +fn append_channel_description(s: &mut String, channel_info: Option<&PromptChannelInfo>) { + let desc = match channel_info.and_then(|ci| ci.description.as_deref()) { + Some(d) if !d.is_empty() => d, + _ => return, + }; + // Collapse newlines to spaces so the description can never spoof another field. + let collapsed: String = desc + .split(['\n', '\r']) + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect::>() + .join(" "); + if collapsed.is_empty() { + return; + } + // Truncate at a character boundary (not byte boundary) to avoid splitting + // multi-byte sequences. + let truncated = if collapsed.chars().count() > MAX_DESCRIPTION_LEN { + let end = collapsed + .char_indices() + .nth(MAX_DESCRIPTION_LEN) + .map(|(i, _)| i) + .unwrap_or(collapsed.len()); + format!("{}…", &collapsed[..end]) + } else { + collapsed + }; + s.push_str(&format!("\nDescription: {truncated}")); +} + /// Format a `[Context]` hints section based on event scope. /// /// `reply_anchor` is the pre-resolved `--reply-to` target for this turn (see @@ -1301,9 +1378,10 @@ fn format_context_hints( let mut s = format!( "[Context]\n\ Scope: thread\n\ - Channel: {channel_display}\n\ - Thread root: {root}" + Channel: {channel_display}" ); + append_channel_description(&mut s, channel_info); + s.push_str(&format!("\nThread root: {root}")); if let Some(ref parent) = thread_tags.parent_event_id { if parent != root { s.push_str(&format!("\nParent: {parent}")); @@ -1318,8 +1396,11 @@ fn format_context_hints( let mut s = format!( "[Context]\n\ Scope: channel\n\ - Channel: {channel_display}\n\ - Hint: Use `buzz messages get --channel ` for recent messages if needed." + Channel: {channel_display}" + ); + append_channel_description(&mut s, channel_info); + s.push_str( + "\nHint: Use `buzz messages get --channel ` for recent messages if needed.", ); if let Some(event_id) = reply_anchor { append_new_thread_reply_instruction(&mut s, event_id); @@ -2202,6 +2283,85 @@ mod tests { assert_eq!(batch2.events[1].event.content, "msg2"); } + // ── Retry-throttled work must block idle-pool-sleep teardown ──────────── + // + // Regression for the PR #5682 review blocker: a failed turn requeued with a + // future backoff deadline is real queued work. `has_flushable_work()` + // returns false for it (throttled → not flushable *now*), so gating + // idle-pool-sleep on flushability would tear down the pool while the batch + // sits waiting — and because lazy re-wake is itself gated on flushability + // and the maintenance timer is disabled while sleeping, the batch would be + // stranded until unrelated traffic arrived. `has_undispatched_work()` must + // see the throttled batch so the sleep gate keeps the pool alive. + #[test] + fn test_retry_throttled_batch_is_undispatched_but_not_flushable() { + let mut queue = EventQueue::new(DedupMode::Queue); + let ch = Uuid::new_v4(); + queue.push(make_queued(ch, "msg1")); + queue.push(make_queued(ch, "msg2")); + + // Drive a real failure → requeue-with-backoff → mark_complete cycle. + let batch = queue.flush_next().unwrap(); + assert_eq!(batch.events.len(), 2); + assert!( + queue.requeue(batch).is_none(), + "batch requeued, not dead-lettered" + ); + queue.mark_complete(ch); + + // The batch is back in the queue, no longer in-flight, and throttled by + // a future `retry_after`. BASE_RETRY_DELAY guarantees the deadline is in + // the future, so this is not timing-fragile. + assert!( + queue + .retry_after + .get(&ch) + .is_some_and(|&t| t > Instant::now()), + "requeue must have set a future backoff deadline" + ); + assert!(!queue.has_in_flight(), "turn completed, nothing in-flight"); + + // The bug: throttled work is invisible to flushability... + assert!( + !queue.has_flushable_work(), + "throttled batch must NOT be flushable yet" + ); + // ...but it IS undispatched work the sleep gate must protect. + assert!( + queue.has_undispatched_work(), + "retry-throttled batch MUST count as undispatched work" + ); + } + + #[test] + fn test_has_undispatched_work_false_when_truly_empty_or_in_flight() { + let mut queue = EventQueue::new(DedupMode::Queue); + let ch = Uuid::new_v4(); + + // Empty queue: no undispatched work. + assert!(!queue.has_undispatched_work()); + + // Dispatched batch (in-flight): the events left the queue, and an + // in-flight turn is gated separately (has_in_flight), so this must be + // false — otherwise the pool could never sleep after any turn. + queue.push(make_queued(ch, "msg1")); + assert!( + queue.has_undispatched_work(), + "queued-but-not-flushed is undispatched" + ); + let batch = queue.flush_next().unwrap(); + assert!(queue.has_in_flight()); + assert!( + !queue.has_undispatched_work(), + "in-flight work is not undispatched — it is gated by has_in_flight" + ); + + // Completed cleanly (no requeue): fully drained, nothing left. + queue.mark_complete(batch.channel_id); + assert!(!queue.has_undispatched_work()); + assert!(!queue.has_in_flight()); + } + #[test] fn test_requeue_interleaves_with_other_channels() { let mut queue = EventQueue::new(DedupMode::Queue); @@ -3087,6 +3247,7 @@ mod tests { let ci = PromptChannelInfo { name: "engineering".into(), channel_type: "stream".into(), + description: None, }; let prompt = format_prompt( @@ -3118,6 +3279,7 @@ mod tests { let ci = PromptChannelInfo { name: "DM".into(), channel_type: "dm".into(), + description: None, }; let prompt = format_prompt( @@ -3230,6 +3392,7 @@ mod tests { let ci = PromptChannelInfo { name: "DM".into(), channel_type: "dm".into(), + description: None, }; let ctx = ConversationContext::Dm { messages: vec![ContextMessage { @@ -3488,6 +3651,7 @@ mod tests { let ci = PromptChannelInfo { name: "DM".into(), channel_type: "dm".into(), + description: None, }; // Thread context fetched (as the fetch path does for DM replies). let ctx = ConversationContext::Thread { @@ -3587,6 +3751,7 @@ mod tests { let ci = PromptChannelInfo { name: "DM".into(), channel_type: "dm".into(), + description: None, }; let trigger_only_prompt = format_prompt( @@ -3635,6 +3800,7 @@ mod tests { let ci = PromptChannelInfo { name: "DM".into(), channel_type: "dm".into(), + description: None, }; // No context fetched — hints only. @@ -4130,6 +4296,7 @@ mod tests { let ci = PromptChannelInfo { name: "DM".into(), channel_type: "dm".into(), + description: None, }; let prompt = format_prompt( @@ -4193,6 +4360,7 @@ mod tests { let ci = PromptChannelInfo { name: "DM".into(), channel_type: "dm".into(), + description: None, }; let prompt = format_prompt( @@ -4961,4 +5129,254 @@ mod tests { "second extend must not move deadline backward (monotonic)" ); } + + // ── channel description delivery ───────────────────────────────────────── + + #[test] + fn test_append_channel_description_adds_description_line() { + let ci = PromptChannelInfo { + name: "team".into(), + channel_type: "stream".into(), + description: Some("Engineering discussions".into()), + }; + let mut s = "[Context]\nScope: channel\nChannel: team (#abc)".to_string(); + append_channel_description(&mut s, Some(&ci)); + assert!( + s.contains("\nDescription: Engineering discussions"), + "description must be appended; got: {s}" + ); + } + + #[test] + fn test_append_channel_description_absent_when_none() { + let ci = PromptChannelInfo { + name: "team".into(), + channel_type: "stream".into(), + description: None, + }; + let mut s = "[Context]\nScope: channel".to_string(); + append_channel_description(&mut s, Some(&ci)); + assert!( + !s.contains("Description:"), + "no description must be appended when None; got: {s}" + ); + } + + #[test] + fn test_append_channel_description_absent_when_channel_info_none() { + let mut s = "[Context]\nScope: channel".to_string(); + append_channel_description(&mut s, None); + assert!( + !s.contains("Description:"), + "no description must be appended when channel_info is None; got: {s}" + ); + } + + #[test] + fn test_append_channel_description_collapses_newlines_spoof_prevention() { + // A multiline description must not be able to inject a fake [Context] field. + let ci = PromptChannelInfo { + name: "team".into(), + channel_type: "stream".into(), + description: Some("Line one\nScope: injected\nLine two".into()), + }; + let mut s = "[Context]\nScope: channel".to_string(); + append_channel_description(&mut s, Some(&ci)); + // The whole description is on a single Description line — no injected field. + let desc_line = s.lines().find(|l| l.starts_with("Description:")).unwrap(); + assert_eq!( + desc_line, "Description: Line one Scope: injected Line two", + "multiline description must collapse to one line, never a fake field" + ); + assert_eq!( + s.lines().filter(|l| l.starts_with("Description:")).count(), + 1, + "exactly one Description line is rendered" + ); + } + + #[test] + fn test_append_channel_description_truncates_at_cap() { + let long_desc = "x".repeat(600); + let ci = PromptChannelInfo { + name: "team".into(), + channel_type: "stream".into(), + description: Some(long_desc), + }; + let mut s = "[Context]\nScope: channel".to_string(); + append_channel_description(&mut s, Some(&ci)); + let desc_line = s.lines().find(|l| l.starts_with("Description:")).unwrap(); + assert!( + desc_line.ends_with('…'), + "truncated description must end with '…'; got: {desc_line}" + ); + // Value = first MAX_DESCRIPTION_LEN chars + the "…" marker. + let value = desc_line.strip_prefix("Description: ").unwrap(); + assert_eq!( + value.chars().count(), + MAX_DESCRIPTION_LEN + 1, + "truncated value is exactly the cap plus the ellipsis marker" + ); + } + + #[test] + fn test_append_channel_description_multibyte_truncation_is_char_safe() { + // Truncation must land on a char boundary, never split a multi-byte code point. + let long_desc = "é".repeat(600); + let ci = PromptChannelInfo { + name: "team".into(), + channel_type: "stream".into(), + description: Some(long_desc), + }; + let mut s = "[Context]\nScope: channel".to_string(); + append_channel_description(&mut s, Some(&ci)); + let desc_line = s.lines().find(|l| l.starts_with("Description:")).unwrap(); + let value = desc_line.strip_prefix("Description: ").unwrap(); + assert_eq!(value.chars().count(), MAX_DESCRIPTION_LEN + 1); + } + + #[test] + fn test_append_channel_description_whitespace_only_is_absent() { + let ci = PromptChannelInfo { + name: "team".into(), + channel_type: "stream".into(), + description: Some("\n \r\n \n".into()), + }; + let mut s = "[Context]\nScope: channel".to_string(); + append_channel_description(&mut s, Some(&ci)); + assert!( + !s.contains("Description:"), + "a whitespace-only description collapses to empty and is not rendered; got: {s}" + ); + } + + fn description_batch(ch: Uuid, event: Event) -> FlushBatch { + FlushBatch { + channel_id: ch, + events: vec![BatchEvent { + event, + prompt_tag: "test".into(), + received_at: Instant::now(), + }], + cancelled_events: vec![], + cancel_reason: None, + } + } + + #[test] + fn test_format_prompt_includes_description_in_context_for_channel_turn() { + let ch = Uuid::new_v4(); + let batch = description_batch(ch, make_event("what should we build?")); + let ci = PromptChannelInfo { + name: "engineering".into(), + channel_type: "stream".into(), + description: Some("Engineering discussions and planning.".into()), + }; + let prompt = format_prompt( + &batch, + &FormatPromptArgs { + channel_info: Some(&ci), + has_system_prompt_support: true, + ..Default::default() + }, + ) + .join("\n\n"); + assert!( + prompt.contains("Scope: channel"), + "channel-scope turn expected; got: {prompt}" + ); + assert!( + prompt.contains("Description: Engineering discussions and planning."), + "description must appear in [Context] for channel turns; got: {prompt}" + ); + } + + #[test] + fn test_format_prompt_includes_description_in_context_for_thread_turn() { + let ch = Uuid::new_v4(); + let event = make_event_with_tags( + "reply in thread", + vec![vec![ + "e".into(), + "root123".into(), + "".into(), + "reply".into(), + ]], + ); + let batch = description_batch(ch, event); + let ci = PromptChannelInfo { + name: "engineering".into(), + channel_type: "stream".into(), + description: Some("Engineering discussions and planning.".into()), + }; + let prompt = format_prompt( + &batch, + &FormatPromptArgs { + channel_info: Some(&ci), + has_system_prompt_support: true, + ..Default::default() + }, + ) + .join("\n\n"); + assert!( + prompt.contains("Scope: thread"), + "thread-scope turn expected; got: {prompt}" + ); + assert!( + prompt.contains("Description: Engineering discussions and planning."), + "description must appear in [Context] for thread turns; got: {prompt}" + ); + } + + #[test] + fn test_format_prompt_excludes_description_for_dm_turn() { + let ch = Uuid::new_v4(); + let batch = description_batch(ch, make_event("hey")); + let ci = PromptChannelInfo { + name: "DM".into(), + channel_type: "dm".into(), + description: Some("This should not appear.".into()), + }; + let prompt = format_prompt( + &batch, + &FormatPromptArgs { + channel_info: Some(&ci), + has_system_prompt_support: true, + ..Default::default() + }, + ) + .join("\n\n"); + assert!( + prompt.contains("Scope: dm"), + "dm-scope turn expected; got: {prompt}" + ); + assert!( + !prompt.contains("Description:"), + "DM turn must not include a Description field; got: {prompt}" + ); + } + + #[test] + fn test_format_prompt_no_description_when_channel_metadata_unresolved() { + let ch = Uuid::new_v4(); + let batch = description_batch(ch, make_event("what should we build?")); + // channel_info None models unresolved metadata: no name, no description. + let prompt = format_prompt( + &batch, + &FormatPromptArgs { + channel_info: None, + has_system_prompt_support: true, + ..Default::default() + }, + ) + .join("\n\n"); + assert!( + prompt.contains("Scope: channel"), + "channel-scope turn expected; got: {prompt}" + ); + assert!( + !prompt.contains("Description:"), + "unresolved metadata must not render a Description field; got: {prompt}" + ); + } } diff --git a/crates/buzz-acp/src/relay.rs b/crates/buzz-acp/src/relay.rs index 2cbb82411fd..17a818867dd 100644 --- a/crates/buzz-acp/src/relay.rs +++ b/crates/buzz-acp/src/relay.rs @@ -136,6 +136,8 @@ use crate::config::ChannelFilter; pub struct ChannelInfo { pub name: String, pub channel_type: String, + /// Channel description from the kind-39000 `about` tag, if present. + pub description: Option, } pub(crate) fn channel_type_from_tags(tags: &[serde_json::Value]) -> String { @@ -175,7 +177,7 @@ pub(crate) fn merge_discovered_channels( channel_uuids: Vec, meta_events: &serde_json::Value, ) -> HashMap { - let mut meta_map: HashMap = HashMap::new(); + let mut meta_map: HashMap)> = HashMap::new(); let mut archived: std::collections::HashSet = std::collections::HashSet::new(); if let Some(arr) = meta_events.as_array() { for ev in arr { @@ -186,11 +188,13 @@ pub(crate) fn merge_discovered_channels( let mut d_val = None; let mut name = None; let mut is_archived = false; + let mut description = None; for tag in tags { if let Some(arr) = tag.as_array() { match arr.first().and_then(|v| v.as_str()) { Some("d") => d_val = arr.get(1).and_then(|v| v.as_str()), Some("name") => name = arr.get(1).and_then(|v| v.as_str()), + Some("about") => description = arr.get(1).and_then(|v| v.as_str()), Some("archived") => { is_archived = arr.get(1).and_then(|v| v.as_str()) == Some("true") } @@ -206,7 +210,11 @@ pub(crate) fn merge_discovered_channels( } let ch_name = name.unwrap_or("unknown").to_string(); let ch_type = channel_type_from_tags(tags); - meta_map.insert(uuid, (ch_name, ch_type)); + let ch_desc = description + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(str::to_string); + meta_map.insert(uuid, (ch_name, ch_type, ch_desc)); } } } @@ -217,10 +225,17 @@ pub(crate) fn merge_discovered_channels( if archived.contains(&uuid) { continue; } - let (name, channel_type) = meta_map + let (name, channel_type, description) = meta_map .remove(&uuid) - .unwrap_or_else(|| ("unknown".to_string(), "unknown".to_string())); - map.insert(uuid, ChannelInfo { name, channel_type }); + .unwrap_or_else(|| ("unknown".to_string(), "unknown".to_string(), None)); + map.insert( + uuid, + ChannelInfo { + name, + channel_type, + description, + }, + ); } map } @@ -4163,6 +4178,46 @@ mod tests { assert!(map.contains_key(&ch), "archived=false is treated as live"); } + #[test] + fn merge_discovered_channels_parses_about_as_description() { + let ch = Uuid::new_v4(); + let meta = serde_json::json!([meta_event( + ch, + "team", + &["t", "stream", "about", "Engineering discussions"] + )]); + + let map = merge_discovered_channels(vec![ch], &meta); + + assert_eq!( + map[&ch].description.as_deref(), + Some("Engineering discussions") + ); + } + + #[test] + fn merge_discovered_channels_blank_about_is_none() { + let ch = Uuid::new_v4(); + let meta = serde_json::json!([meta_event(ch, "team", &["about", " "])]); + + let map = merge_discovered_channels(vec![ch], &meta); + + assert_eq!( + map[&ch].description, None, + "a whitespace-only about tag is trimmed away to None" + ); + } + + #[test] + fn merge_discovered_channels_missing_about_is_none() { + let ch = Uuid::new_v4(); + let meta = serde_json::json!([meta_event(ch, "team", &["t", "stream"])]); + + let map = merge_discovered_channels(vec![ch], &meta); + + assert_eq!(map[&ch].description, None); + } + #[test] fn parse_ok_accepted() { let text = r#"["OK","abc123",true,""]"#; diff --git a/crates/buzz-acp/src/usage.rs b/crates/buzz-acp/src/usage.rs index 1eb3eba3b17..2197b99ef5f 100644 --- a/crates/buzz-acp/src/usage.rs +++ b/crates/buzz-acp/src/usage.rs @@ -244,6 +244,158 @@ pub struct TurnUsage { pub pricing_identity: Option, } +/// Per-turn usage carried by a standard ACP `session/prompt` response. +/// Adapter input excludes cache reads and writes, so NIP-AM input must add +/// those subsets with checked arithmetic. +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PromptResponseUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub total_tokens: u64, + pub cached_read_tokens: Option, + pub cached_write_tokens: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StandardAdapterKind { + Claude, + Codex, +} + +#[derive(Debug, Default)] +struct StandardSessionState { + published_seq: u64, + last_cost: Option, + cost_poisoned: bool, +} + +#[derive(Debug, Default)] +pub(crate) struct StandardUsageTracker { + sessions: HashMap, + in_flight_session: Option, + pending_cost: Option<(String, f64)>, + pending_prompt: Option<(String, PromptResponseUsage, StandardAdapterKind)>, +} + +impl StandardUsageTracker { + pub(crate) fn seed_zero_baseline(&mut self, session_id: &str) { + self.sessions + .entry(session_id.to_string()) + .or_insert_with(|| StandardSessionState { + published_seq: 0, + last_cost: Some(0.0), + cost_poisoned: false, + }); + } + + pub(crate) fn begin_turn(&mut self, session_id: &str) { + self.in_flight_session = Some(session_id.to_string()); + self.pending_cost = None; + self.pending_prompt = None; + } + + /// Claude's `usage_update.cost.amount` is a raw session-cumulative total. + pub(crate) fn record_cost(&mut self, session_id: &str, cost: f64) { + if cost.is_finite() && cost >= 0.0 && self.in_flight_session.as_deref() == Some(session_id) + { + self.pending_cost = Some((session_id.to_string(), cost)); + } + } + + pub(crate) fn record_prompt_usage( + &mut self, + session_id: &str, + usage: PromptResponseUsage, + adapter: StandardAdapterKind, + ) { + if self.in_flight_session.as_deref() == Some(session_id) { + self.pending_prompt = Some((session_id.to_string(), usage, adapter)); + } + } + + pub(crate) fn take(&mut self) -> Option { + self.in_flight_session = None; + let prompt = self.pending_prompt.take(); + let cost = self.pending_cost.take(); + let session_id = prompt + .as_ref() + .map(|(session_id, _, _)| session_id.clone()) + .or_else(|| cost.as_ref().map(|(session_id, _)| session_id.clone()))?; + + let (inclusive_input, output_tokens, total_tokens, cache_read, cache_write) = match prompt { + Some((_, usage, adapter)) => { + let inclusive_input = usage + .input_tokens + .checked_add(usage.cached_read_tokens.unwrap_or(0)) + .and_then(|input| input.checked_add(usage.cached_write_tokens.unwrap_or(0))); + let total_tokens = + (adapter == StandardAdapterKind::Codex).then_some(usage.total_tokens); + ( + inclusive_input, + inclusive_input.map(|_| usage.output_tokens), + inclusive_input.and(total_tokens), + inclusive_input.and(usage.cached_read_tokens), + inclusive_input.and(usage.cached_write_tokens), + ) + } + None => (None, None, None, None, None), + }; + + let state = self.sessions.entry(session_id.clone()).or_default(); + let cumulative_cost = cost.map(|(_, cost)| cost); + let turn_cost = match (state.cost_poisoned, state.last_cost, cumulative_cost) { + (false, Some(previous), Some(current)) if current >= previous => { + let delta = current - previous; + delta.is_finite().then_some(delta) + } + _ => None, + }; + if let Some(current) = cumulative_cost { + // A decrease means the cumulative series restarted or is corrupt. + // Poison the baseline rather than deriving a later delta across the + // discontinuity. The raw cumulative value still remains observable. + if state.last_cost.is_some_and(|previous| current < previous) { + state.cost_poisoned = true; + state.last_cost = None; + } else if !state.cost_poisoned { + state.last_cost = Some(current); + } + } + + // Input overflow invalidates the standard prompt counters. Emit only if + // another valid signal (normally Claude cost) remains; NIP-AM forbids an + // otherwise all-null usage record. + if inclusive_input.is_none() && cumulative_cost.is_none() { + return None; + } + + state.published_seq += 1; + Some(TurnUsage { + session_id, + turn_seq: state.published_seq, + // Standard prompt counters are per-turn already. A cost-only record + // is reliable only when a seeded/previous cumulative baseline made + // the cost delta provable. + delta_reliable: inclusive_input.is_some() || turn_cost.is_some(), + turn_input_tokens: inclusive_input, + turn_output_tokens: output_tokens, + turn_total_tokens: total_tokens, + turn_cost_usd: turn_cost, + turn_cache_read_tokens: cache_read, + turn_cache_write_tokens: cache_write, + cumulative_input_tokens: None, + cumulative_output_tokens: None, + cumulative_total_tokens: None, + cumulative_cost_usd: cumulative_cost, + cumulative_cache_read_tokens: None, + cumulative_cache_write_tokens: None, + model: None, + pricing_identity: None, + }) + } +} + /// Tracks per-session cumulative usage state across turns. /// /// Cheap to construct. Usage lifecycle per turn: diff --git a/crates/buzz-admin/Cargo.toml b/crates/buzz-admin/Cargo.toml index 7a69e146bb9..00c2804cbcf 100644 --- a/crates/buzz-admin/Cargo.toml +++ b/crates/buzz-admin/Cargo.toml @@ -13,6 +13,7 @@ path = "src/main.rs" [dependencies] buzz-db = { workspace = true } +buzz-deletion = { workspace = true } buzz-core = { workspace = true } buzz-auth = { workspace = true } buzz-pubsub = { workspace = true } diff --git a/crates/buzz-admin/src/deletions.rs b/crates/buzz-admin/src/deletions.rs new file mode 100644 index 00000000000..64cb8bd732a --- /dev/null +++ b/crates/buzz-admin/src/deletions.rs @@ -0,0 +1,19 @@ +//! Thin `buzz-admin deletions` adapter. + +pub use buzz_deletion::Command as DeletionsCommand; + +/// Delegate to the shared durable deletion engine. +pub async fn run(command: DeletionsCommand) -> anyhow::Result { + buzz_deletion::run(command).await +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + #[test] + fn continuous_worker_command_is_not_exposed() { + let command = crate::Cli::try_parse_from(["buzz-admin", "deletions", "worker"]); + assert!(command.is_err()); + } +} diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index bb30ddfae4f..580d5865913 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -20,6 +20,8 @@ //! newest timestamp and collide on the bumped second. run.sh serialization is //! the guard against parallel adds (e.g. `xargs -P`). +mod deletions; + use std::sync::Arc; use anyhow::Result; @@ -81,6 +83,11 @@ enum Command { #[command(subcommand)] command: ProductFeedbackCommand, }, + /// Durable CLI-only whole-community deletion control plane. + Deletions { + #[command(subcommand)] + command: deletions::DeletionsCommand, + }, /// Emit kind:39000/39002 events for channels missing them. /// /// Channels created via direct SQL (seed scripts, pre-migration data) won't @@ -148,6 +155,7 @@ async fn run(cli: Cli) -> Result { Command::ProductFeedback { command: ProductFeedbackCommand::List { limit }, } => cmd_list_product_feedback(limit).await, + Command::Deletions { command } => deletions::run(command).await, Command::ReconcileChannels { relay_key } => { reconcile_channels(relay_key).await?; Ok(0) diff --git a/crates/buzz-agent/README.md b/crates/buzz-agent/README.md index 5d942777d5e..0bc03db7813 100644 --- a/crates/buzz-agent/README.md +++ b/crates/buzz-agent/README.md @@ -153,7 +153,8 @@ Everything is environment variables. No flags, no config files. (We are a subpro | `BUZZ_AGENT_SYSTEM_PROMPT` | built-in | Inline system prompt. | | `BUZZ_AGENT_SYSTEM_PROMPT_FILE` | — | File path. Mutually exclusive with the above. | | `BUZZ_AGENT_MAX_ROUNDS` | `0` | Tool-loop iteration cap. 0 = unlimited. | -| `BUZZ_AGENT_MAX_OUTPUT_TOKENS` | `32768` | Per LLM call. Headroom for large tool-call inputs (e.g. file writes via heredoc); Sonnet 4 / Opus 4 cap at 64K. | +| `BUZZ_AGENT_MAX_OUTPUT_TOKENS` | `65536` | Desired per-call ceiling. Set this at or below the served model's output limit for each agent deployment. Proactive handoff is independently based on 90% of `BUZZ_AGENT_MAX_CONTEXT_TOKENS`. | +| `BUZZ_AGENT_MAX_TOKEN_RECOVERIES` | `3` | Retries after a successful response is truncated at the output-token limit. `0` disables recovery; the finite value and `BUZZ_AGENT_MAX_ROUNDS` prevent infinite retries. | | `BUZZ_AGENT_MAX_CONTEXT_TOKENS` | `200000` | Provider context window used by the handoff gate. | | `BUZZ_AGENT_MAX_HANDOFFS` | `10` | Max context handoffs per session before falling back to truncation. | | `BUZZ_AGENT_LLM_TIMEOUT_SECS` | `240` | Max seconds with no response bytes before abandoning an LLM call (per-read inactivity, not wall-clock). | diff --git a/crates/buzz-agent/src/agent.rs b/crates/buzz-agent/src/agent.rs index 0805fddb12d..9258ce449f3 100644 --- a/crates/buzz-agent/src/agent.rs +++ b/crates/buzz-agent/src/agent.rs @@ -31,12 +31,7 @@ const UNSUPPORTED_IMAGE_TOOL_MESSAGE: &str = "The current model does not support /// its output-token limit. This is a user message rather than a synthetic tool /// result because truncation can happen without a tool call (and an unpaired /// tool result is invalid on every provider wire format). -const MAX_TOKENS_RECOVERY_MESSAGE: &str = "Your previous response exceeded the model's output token limit and was truncated. Any incomplete tool call was not run. Continue the task, breaking the work or tool call into smaller steps and keeping the response concise."; - -/// A provider can repeatedly spend its entire output allowance without making -/// progress, while `max_rounds` is unbounded by default. Keep the in-turn rescue -/// finite so a persistently truncating model eventually surfaces `max_tokens`. -const MAX_TOKENS_RECOVERIES_PER_RUN: u32 = 2; +const MAX_TOKENS_RECOVERY_MESSAGE: &str = "Your previous response reached the model's output token limit and was truncated. Any incomplete tool calls were discarded and were not run. Stop prolonged internal reasoning now. Use the available tools immediately: write a script or artifact to a file and run it in small, verifiable steps instead of emitting the entire solution inline. Continue the task concisely from the preserved text."; /// Remove image blocks that the provider has explicitly rejected while keeping /// their surrounding tool result (and therefore the tool-call/result pairing) @@ -667,9 +662,10 @@ impl RunCtx<'_> { tool_calls: Vec::new(), reasoning_details: response.reasoning_details, }); - if max_tokens_recoveries >= MAX_TOKENS_RECOVERIES_PER_RUN { + if max_tokens_recoveries >= self.cfg.max_token_recoveries { tracing::warn!( recoveries = max_tokens_recoveries, + max_recoveries = self.cfg.max_token_recoveries, "provider repeatedly hit output token limit; recovery budget exhausted" ); return Ok(StopReason::MaxTokens); @@ -677,8 +673,7 @@ impl RunCtx<'_> { max_tokens_recoveries = max_tokens_recoveries.saturating_add(1); tracing::warn!( recovery = max_tokens_recoveries, - max_recoveries = MAX_TOKENS_RECOVERIES_PER_RUN, - discarded_tool_calls = response.tool_calls.len(), + max_recoveries = self.cfg.max_token_recoveries, "provider hit output token limit; asking model to continue in smaller steps" ); self.history diff --git a/crates/buzz-agent/src/config.rs b/crates/buzz-agent/src/config.rs index 501d3123d79..b29bf3d2fb8 100644 --- a/crates/buzz-agent/src/config.rs +++ b/crates/buzz-agent/src/config.rs @@ -781,6 +781,10 @@ pub struct Config { pub system_prompt: String, pub max_rounds: u32, pub max_output_tokens: u32, + /// Maximum number of retries after a provider returns a successful but + /// output-truncated response. Zero disables truncation recovery. This is + /// independent of `max_rounds`, which still bounds all successful calls. + pub max_token_recoveries: u32, pub llm_timeout: Duration, pub tool_timeout: Duration, pub mcp_init_timeout: Duration, @@ -931,7 +935,8 @@ impl Config { anthropic_api_version: env_or("ANTHROPIC_API_VERSION", "2023-06-01"), openai_api, max_rounds: parse_env("BUZZ_AGENT_MAX_ROUNDS", 0)?, - max_output_tokens: parse_env("BUZZ_AGENT_MAX_OUTPUT_TOKENS", 32_768)?, + max_output_tokens: parse_env("BUZZ_AGENT_MAX_OUTPUT_TOKENS", 65_536)?, + max_token_recoveries: parse_env("BUZZ_AGENT_MAX_TOKEN_RECOVERIES", 3u32)?, llm_timeout: Duration::from_secs(parse_env("BUZZ_AGENT_LLM_TIMEOUT_SECS", 240)?), tool_timeout: Duration::from_secs(parse_env("BUZZ_AGENT_TOOL_TIMEOUT_SECS", 660)?), mcp_init_timeout: Duration::from_secs(parse_env( @@ -983,6 +988,7 @@ impl Config { openai_api: OpenAiApi::Chat, max_rounds: 0, max_output_tokens: 1, + max_token_recoveries: 0, llm_timeout: Duration::from_secs(30), tool_timeout: Duration::from_secs(30), mcp_init_timeout: Duration::from_secs(30), diff --git a/crates/buzz-agent/src/handoff.rs b/crates/buzz-agent/src/handoff.rs index 4748678059d..869fbe06664 100644 --- a/crates/buzz-agent/src/handoff.rs +++ b/crates/buzz-agent/src/handoff.rs @@ -59,8 +59,7 @@ impl RunCtx<'_> { } if *handoff_attempts >= self.cfg.max_handoffs { let projected = self.projected_handoff_input_tokens(); - let threshold = - token_threshold(self.cfg.max_context_tokens, self.cfg.max_output_tokens); + let threshold = token_threshold(self.cfg.max_context_tokens); tracing::warn!( session_id = self.session_id, reason = "preflight", @@ -246,7 +245,7 @@ impl RunCtx<'_> { match *self.last_request_input_tokens { Some(_) => { self.projected_handoff_input_tokens() - >= token_threshold(self.cfg.max_context_tokens, self.cfg.max_output_tokens) + >= token_threshold(self.cfg.max_context_tokens) } None => { let bytes: usize = self @@ -257,7 +256,6 @@ impl RunCtx<'_> { bytes > byte_fallback_threshold( self.cfg.max_context_tokens, - self.cfg.max_output_tokens, self.cfg.max_history_bytes, ) } @@ -495,28 +493,20 @@ fn estimate_tokens_from_bytes(bytes: usize) -> u64 { (bytes as u64).div_ceil(CONSERVATIVE_BYTES_PER_TOKEN) } -/// Input-token count at which to hand off. Caps at the configured fraction of -/// the window and also leaves room for `max_output_tokens`, so input + output -/// can't together exceed the window. Free function so the policy math is unit -/// testable without constructing a [`RunCtx`]. -fn token_threshold(max_context_tokens: u64, max_output_tokens: u32) -> u64 { +/// Input-token count at which to hand off. Uses 90% of the configured context +/// window, independent of the request's output allowance. Free function so the +/// policy math is unit testable without constructing a [`RunCtx`]. +fn token_threshold(max_context_tokens: u64) -> u64 { // Integer math: handoff threshold is 90%, i.e. window * 9 / 10. - let fractional = max_context_tokens / 10 * 9; - let output_reserved = max_context_tokens.saturating_sub(u64::from(max_output_tokens)); - fractional.min(output_reserved) + max_context_tokens / 10 * 9 } /// Conservative byte cap used only before any usage is known. Maps the token /// threshold to bytes at the conservative bytes/token ratio (so the cap is /// small and the handoff fires early), clamped to the configured byte budget /// so it can only ever be more conservative than the old byte-only behavior. -fn byte_fallback_threshold( - max_context_tokens: u64, - max_output_tokens: u32, - max_history_bytes: usize, -) -> usize { - let derived = token_threshold(max_context_tokens, max_output_tokens) - .saturating_mul(CONSERVATIVE_BYTES_PER_TOKEN); +fn byte_fallback_threshold(max_context_tokens: u64, max_history_bytes: usize) -> usize { + let derived = token_threshold(max_context_tokens).saturating_mul(CONSERVATIVE_BYTES_PER_TOKEN); let byte_cap = max_history_bytes / 10 * 9; usize::try_from(derived).unwrap_or(usize::MAX).min(byte_cap) } @@ -605,36 +595,23 @@ mod tests { } #[test] - fn token_threshold_uses_fraction_when_output_is_small() { - // 200k window, 1k output. fractional = 0.9*200000 = 180000; - // output_reserved = 200000-1000 = 199000; min = 180000. - assert_eq!(token_threshold(200_000, 1_000), 180_000); - } - - #[test] - fn token_threshold_reserves_output_headroom() { - // Large output relative to window: the output-reserve term dominates, - // keeping input+output within the window. - // 100k window, 40k output: fractional=90k, reserved=60k -> 60k. - assert_eq!(token_threshold(100_000, 40_000), 60_000); - } - - #[test] - fn token_threshold_saturates_when_output_exceeds_window() { - // Degenerate (config validation forbids this, but math must not panic): - // reserved saturates to 0, so threshold is 0 -> always hand off. - assert_eq!(token_threshold(1000, 5000), 0); + fn token_threshold_is_independent_of_output_allowance() { + // Handoff always begins at 90% of the input context budget, including + // when the request's output allowance grows or exceeds the window. + assert_eq!(token_threshold(200_000), 180_000); + assert_eq!(token_threshold(100_000), 90_000); + assert_eq!(token_threshold(1_000), 900); } #[test] fn byte_fallback_is_conservative_and_capped() { // Derived = token_threshold * 1 (1 byte/token upper bound). For - // 200k/1k: 180000 bytes, well under a 16 MiB byte budget, so derived - // wins (early handoff). - let t = byte_fallback_threshold(200_000, 1_000, 16 * 1024 * 1024); + // 200k window: 180000 bytes, well under a 16 MiB byte budget, so the + // derived threshold wins (early handoff). + let t = byte_fallback_threshold(200_000, 16 * 1024 * 1024); assert_eq!(t, 180_000); // With a tiny byte budget the cap wins -> never exceeds it (window*90%). - let capped = byte_fallback_threshold(200_000, 1_000, 8192); + let capped = byte_fallback_threshold(200_000, 8192); assert_eq!(capped, 8192 / 10 * 9); } diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index 1f3cac3df49..a963de1e7c1 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -1135,7 +1135,7 @@ fn map_stop(s: Option<&str>) -> ProviderStop { match s { Some("end_turn" | "stop") => ProviderStop::EndTurn, Some("tool_use" | "tool_calls") => ProviderStop::ToolUse, - Some("max_tokens" | "length") => ProviderStop::MaxTokens, + Some("max_tokens" | "length" | "model_context_window_exceeded") => ProviderStop::MaxTokens, Some("refusal" | "content_filter") => ProviderStop::Refusal, _ => ProviderStop::Other, } @@ -2589,6 +2589,7 @@ mod tests { system_prompt: "system".into(), max_rounds: 10, max_output_tokens: 1024, + max_token_recoveries: 3, llm_timeout: Duration::from_secs(10), tool_timeout: Duration::from_secs(10), mcp_init_timeout: Duration::from_secs(10), @@ -3104,6 +3105,17 @@ mod tests { assert!(r.tool_calls.is_empty()); } + #[test] + fn anthropic_context_window_exhaustion_is_truncation() { + let v = serde_json::json!({ + "stop_reason": "model_context_window_exceeded", + "content": [{"type": "text", "text": "partial text"}], + }); + let r = parse_anthropic(v).unwrap(); + assert_eq!(r.stop, ProviderStop::MaxTokens); + assert_eq!(r.text, "partial text"); + } + #[test] fn truncated_anthropic_tool_use_is_discarded_not_rejected() { let v = serde_json::json!({ diff --git a/crates/buzz-agent/src/types.rs b/crates/buzz-agent/src/types.rs index a814a27cf7c..10ac65b46ef 100644 --- a/crates/buzz-agent/src/types.rs +++ b/crates/buzz-agent/src/types.rs @@ -687,11 +687,11 @@ mod tests { // Regression: a single ~3.1M-base64-byte `view_image` result on an // otherwise-empty history must NOT exceed the default pre-usage // handoff cap. The gate's byte-fallback threshold with the shipped - // defaults (max_context_tokens=200_000, max_output_tokens=32_768) is - // min(200_000*9/10, 200_000-32_768) = 167_232 "bytes". Before the fix + // defaults (max_context_tokens=200_000) is 200_000*9/10 = 180_000 + // "bytes". Before the fix // this item counted ~3.1M and tripped instantly. let item = image_item(3_118_884); - const DEFAULT_PRE_USAGE_THRESHOLD: usize = 167_232; + const DEFAULT_PRE_USAGE_THRESHOLD: usize = 180_000; assert!( item.context_pressure_bytes() <= DEFAULT_PRE_USAGE_THRESHOLD, "one image charged {} bytes of context pressure, over the {} threshold", diff --git a/crates/buzz-agent/tests/regressions.rs b/crates/buzz-agent/tests/regressions.rs index a972f9b2429..6a4f347f6bb 100644 --- a/crates/buzz-agent/tests/regressions.rs +++ b/crates/buzz-agent/tests/regressions.rs @@ -2314,6 +2314,27 @@ fn reply_guard_rejects_unparseable_toggle() { ); } +#[test] +fn max_token_recoveries_rejects_unparseable_value() { + let out = std::process::Command::new(env!("CARGO_BIN_EXE_buzz-agent")) + .env("BUZZ_AGENT_PROVIDER", "openai") + .env("OPENAI_COMPAT_API_KEY", "test") + .env("OPENAI_COMPAT_MODEL", "fake-model") + .env("BUZZ_AGENT_MAX_TOKEN_RECOVERIES", "unbounded") + .stdin(Stdio::null()) + .output() + .expect("run buzz-agent"); + assert!( + !out.status.success(), + "invalid recovery budget was accepted" + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("BUZZ_AGENT_MAX_TOKEN_RECOVERIES"), + "expected offending key in config error: {stderr}" + ); +} + /// A prompt large enough that the recovery ladder's halving stays above /// `HANDOFF_MIN_PROMPT_BUDGET_BYTES` (4 KiB) for all three rungs. /// @@ -2494,9 +2515,11 @@ async fn max_tokens_recovers_in_turn_without_running_partial_tool_call() { ); assert!( serialized.contains("output token limit") - && serialized.contains("smaller steps") - && serialized.contains("tool call"), - "retry lacks actionable truncation feedback: {retry}" + && serialized.contains("Stop prolonged internal reasoning") + && serialized.contains("Use the available tools immediately") + && serialized.contains("write a script or artifact") + && serialized.contains("small, verifiable steps"), + "retry lacks the tool-first truncation directive: {retry}" ); assert!( !serialized.contains("partial-call") && !serialized.contains("tool_call_id"), @@ -2542,7 +2565,14 @@ async fn repeated_max_tokens_is_bounded() { .map(|_| openai_max_tokens("still truncated", json!([]))) .collect(); let llm = spawn_capturing_llm(responses).await; - let mut h = Harness::spawn_with_env(&llm.url, &[("BUZZ_AGENT_MAX_ROUNDS", "0")]).await; + let mut h = Harness::spawn_with_env( + &llm.url, + &[ + ("BUZZ_AGENT_MAX_ROUNDS", "0"), + ("BUZZ_AGENT_MAX_TOKEN_RECOVERIES", "2"), + ], + ) + .await; let sid = init_session(&mut h, json!([])).await; let prompt_id = h .send( @@ -2561,6 +2591,93 @@ async fn repeated_max_tokens_is_bounded() { h.shutdown().await; } +/// The default value is the exact number of retries: three recoveries produce +/// four truncating requests, then surface `max_tokens` without another call. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn default_max_tokens_recovery_budget_is_exact() { + let responses = (0..5) + .map(|_| openai_max_tokens("truncated", json!([]))) + .collect(); + let llm = spawn_capturing_llm(responses).await; + let mut h = Harness::spawn(&llm.url).await; + let sid = init_session(&mut h, json!([])).await; + let prompt_id = h + .send( + "session/prompt", + json!({"sessionId": sid, "prompt": [{"type":"text","text":"go"}]}), + ) + .await; + let reply = h.recv_until(|v| v["id"] == json!(prompt_id)).await; + assert_eq!(reply["result"]["stopReason"], "max_tokens", "{reply}"); + let requests = llm.captured.lock().await; + assert_eq!(requests.len(), 4); + assert_eq!( + requests[0]["max_completion_tokens"], 65_536, + "default output ceiling must be carried on the actual request path" + ); + drop(requests); + h.shutdown().await; +} + +/// Zero means disabled, not unlimited: the first truncated response is terminal +/// and no recovery directive is sent. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn zero_max_token_recoveries_disables_retry() { + let llm = spawn_capturing_llm(vec![ + openai_max_tokens("truncated", json!([])), + openai_text("must not be requested"), + ]) + .await; + let mut h = + Harness::spawn_with_env(&llm.url, &[("BUZZ_AGENT_MAX_TOKEN_RECOVERIES", "0")]).await; + let sid = init_session(&mut h, json!([])).await; + let prompt_id = h + .send( + "session/prompt", + json!({"sessionId": sid, "prompt": [{"type":"text","text":"go"}]}), + ) + .await; + let reply = h.recv_until(|v| v["id"] == json!(prompt_id)).await; + assert_eq!(reply["result"]["stopReason"], "max_tokens", "{reply}"); + assert_eq!(llm.captured.lock().await.len(), 1); + h.shutdown().await; +} + +/// A successful recovery may proceed directly to a real tool call. This pins +/// that only tool calls from the truncated response are discarded. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn max_tokens_recovery_can_proceed_to_tool_call() { + let llm = spawn_capturing_llm(vec![ + openai_max_tokens( + "partial", + json!([{"id":"discard-me","type":"function","function":{"name":"dev__shell","arguments":"{\"command\":\"false\"}"}}]), + ), + openai_tool_call("kept-call", "fake__tool_0", json!({})), + openai_text("done"), + ]) + .await; + let mut h = Harness::spawn(&llm.url).await; + let sid = init_session_with_fake_mcp(&mut h, &[("FAKE_MCP_TOOL_COUNT", "1")]).await; + let prompt_id = h + .send( + "session/prompt", + json!({"sessionId": sid, "prompt": [{"type":"text","text":"go"}]}), + ) + .await; + let reply = h.recv_until(|v| v["id"] == json!(prompt_id)).await; + assert_eq!(reply["result"]["stopReason"], "end_turn", "{reply}"); + let requests = llm.captured.lock().await; + assert_eq!(requests.len(), 3); + let wire = requests[1].to_string(); + assert!( + !wire.contains("discard-me"), + "discarded call leaked: {wire}" + ); + assert!(requests[2].to_string().contains("kept-call")); + drop(requests); + h.shutdown().await; +} + /// A successful recovery must actually send the recovered completion, even when /// `max_rounds` is finite. `round` is incremented BEFORE the completion that /// gets rejected, so a naive `continue` after recovery re-enters the loop with diff --git a/crates/buzz-db/src/channel.rs b/crates/buzz-db/src/channel.rs index fecb6b0ac98..3ca9b3d901c 100644 --- a/crates/buzz-db/src/channel.rs +++ b/crates/buzz-db/src/channel.rs @@ -1506,6 +1506,7 @@ pub async fn reap_expired_ephemeral_channels(pool: &PgPool) -> Result Option { + match self { + Self::Submitted => Some(Self::Inventoried), + Self::Inventoried => Some(Self::Approved), + Self::Approved => Some(Self::Fenced), + Self::Fenced => Some(Self::Drained), + Self::Drained => Some(Self::BindingsRemoved), + Self::BindingsRemoved => Some(Self::PostgresPurged), + Self::PostgresPurged => Some(Self::CachePurged), + Self::CachePurged => Some(Self::LogicallyVerified), + Self::LogicallyVerified => Some(Self::RetentionPending), + Self::RetentionPending | Self::Aborted => None, + } + } + + /// Whether execution may claim this stage. + pub const fn runnable(self) -> bool { + matches!( + self, + Self::Approved + | Self::Fenced + | Self::Drained + | Self::BindingsRemoved + | Self::PostgresPurged + | Self::CachePurged + | Self::LogicallyVerified + ) + } +} + +impl fmt::Display for DeletionStage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let value = match self { + Self::Submitted => "submitted", + Self::Inventoried => "inventoried", + Self::Approved => "approved", + Self::Fenced => "fenced", + Self::Drained => "drained", + Self::BindingsRemoved => "bindings_removed", + Self::PostgresPurged => "postgres_purged", + Self::CachePurged => "cache_purged", + Self::LogicallyVerified => "logically_verified", + Self::RetentionPending => "retention_pending", + Self::Aborted => "aborted", + }; + f.write_str(value) + } +} + +impl FromStr for DeletionStage { + type Err = DbError; + + fn from_str(value: &str) -> std::result::Result { + match value { + "submitted" => Ok(Self::Submitted), + "inventoried" => Ok(Self::Inventoried), + "approved" => Ok(Self::Approved), + "fenced" => Ok(Self::Fenced), + "drained" => Ok(Self::Drained), + "bindings_removed" => Ok(Self::BindingsRemoved), + "postgres_purged" => Ok(Self::PostgresPurged), + "cache_purged" => Ok(Self::CachePurged), + "logically_verified" => Ok(Self::LogicallyVerified), + "retention_pending" => Ok(Self::RetentionPending), + "aborted" => Ok(Self::Aborted), + other => Err(DbError::DeletionSafety(format!( + "unknown community deletion stage: {other}" + ))), + } + } +} + +/// Durable community deletion request. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionRequest { + /// Request identifier. + pub id: Uuid, + /// Target community. + #[serde(serialize_with = "serialize_community_id")] + pub community_id: CommunityId, + /// Permanently reserved canonical host. + pub community_host: String, + /// Current lifecycle stage. + pub stage: DeletionStage, + /// Stage at which the current consecutive retry streak started. + pub retry_stage: Option, + /// Operator identity that submitted the request. + pub requested_by: String, + /// Optional request reason. + pub reason: Option, + /// Frozen catalog manifest. + pub schema_manifest: Option, + /// Frozen community-prefix storage manifest observed at submission. + pub storage_manifest: Option, + /// Destructive storage manifest frozen after the durable fence. + pub destructive_storage_manifest: Option, + /// Frozen inventory aggregate. + pub inventory_manifest: Option, + /// Hex SHA-256 of the frozen inventory. + pub inventory_digest: Option, + /// Durable community fence generation. + pub fence_generation: Option, + /// Current claim owner. + pub lease_owner: Option, + /// Monotonic claim generation. + pub lease_generation: i64, + /// Claim expiry. + pub lease_until: Option>, + /// Number of claims. + pub attempts: i32, + /// Number of consecutive failed execution attempts at `retry_stage`. + pub retry_count: i32, + /// Last bounded error. + pub last_error: Option, + /// Earliest time a transient failure may be claimed again. + pub next_attempt_at: DateTime, + /// Permanent fail-closed block reason. + pub blocked_reason: Option, + /// Submission time. + pub created_at: DateTime, + /// Last lifecycle update. + pub updated_at: DateTime, + /// Archive timestamp captured before quiescing changed serving state. + pub pre_quiesce_archived_at: Option>, + /// Whether the pre-quiesce archive value has been captured (including null). + pub quiescing_started_at: Option>, + /// Operator that terminally aborted the request. + pub aborted_by: Option, + /// Reason recorded for terminal abort. + pub abort_reason: Option, + /// Abort completion time. + pub aborted_at: Option>, + /// Terminal logical-deletion time. + pub completed_at: Option>, +} + +/// Frozen PostgreSQL catalog inventory. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SchemaManifest { + /// Sorted community-scoped table names. + pub scoped_tables: Vec, + /// Per-table row counts for the target. + pub row_counts: BTreeMap, + /// Sorted tables with the universal write-fence trigger. + pub fenced_tables: Vec, +} + +/// Frozen storage inventory supplied by the object-store adapter: slim +/// per-prefix summaries for the target community. The concrete key list never +/// lives on the request row — the destructive freeze persists it as chunked +/// `community_deletion_manifest_keys` rows that must hash to these digests. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StorageManifest { + /// Adapter schema version. + pub version: i32, + /// Per-prefix frozen summaries, strictly sorted by prefix. + pub prefixes: Vec, +} + +/// Frozen summary of one community-scoped key prefix. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PrefixManifest { + /// Exact community-scoped listing prefix. + pub prefix: String, + /// Objects under the prefix at enumeration time. + pub object_count: u64, + /// Total object bytes under the prefix at enumeration time. + pub total_bytes: u64, + /// Hex SHA-256 of the newline-terminated ascending key stream. + pub keys_digest: String, +} + +/// One frozen chunk of the destructive key list. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ManifestKeyChunk { + /// Position in the frozen chunk sequence. + pub chunk_no: i64, + /// The tenant prefix every key in this chunk lives under. + pub prefix: String, + /// Strictly ascending keys. + pub keys: Vec, +} + +/// One durable fleet-wide object-store taxonomy sweep record. +#[derive(Debug, Clone, Serialize)] +pub struct TaxonomySweep { + /// Sweep identity. + pub id: Uuid, + /// Listing start time. + pub started_at: DateTime, + /// Record time. + pub completed_at: DateTime, + /// Total objects listed. + pub listed_objects: i64, + /// Exact count of keys outside the known writer taxonomy. + pub unknown_object_count: i64, + /// Bounded sample of unknown keys. + pub unknown_key_sample: Vec, + /// Fleet object cap the sweep ran under. + pub object_cap: i64, +} + +type TaxonomySweepRow = ( + Uuid, + DateTime, + DateTime, + i64, + i64, + sqlx::types::Json>, + i64, +); + +/// Streaming SHA-256 over a strictly ascending key stream. +/// +/// The executor's prefix enumeration and the destructive freeze's chunk +/// validation both fold keys through this, so "the chunk rows are exactly +/// the frozen enumeration" reduces to digest equality. Each key is hashed +/// with a trailing newline so concatenation cannot alias two streams. +pub struct KeyStreamDigest { + hasher: Sha256, + last: Option, + count: u64, +} + +impl Default for KeyStreamDigest { + fn default() -> Self { + Self::new() + } +} + +impl KeyStreamDigest { + /// Start an empty stream. + pub fn new() -> Self { + Self { + hasher: Sha256::new(), + last: None, + count: 0, + } + } + + /// Fold the next key. Keys must arrive strictly ascending — S3 + /// `ListObjectsV2` order — so one out-of-order or duplicate key fails + /// closed instead of silently producing a different digest. + pub fn fold(&mut self, key: &str) -> Result<()> { + if self.last.as_deref().is_some_and(|last| last >= key) { + return Err(DbError::DeletionSafety(format!( + "storage key stream is not strictly ascending at {key}" + ))); + } + self.hasher.update(key.as_bytes()); + self.hasher.update(b"\n"); + self.last = Some(key.to_owned()); + self.count += 1; + Ok(()) + } + + /// Hex digest and key count of everything folded. + pub fn finish(self) -> (String, u64) { + (hex::encode(self.hasher.finalize()), self.count) + } +} + +/// Full frozen inventory approved at the destructive boundary. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FrozenInventory { + /// PostgreSQL catalog state. + pub schema: SchemaManifest, + /// Object-store state. + pub storage: StorageManifest, +} + +impl FrozenInventory { + /// Canonical JSON bytes and SHA-256 digest used to bind approval. + pub fn digest(&self) -> Result> { + Ok(Sha256::digest(serde_json::to_vec(self)?).to_vec()) + } +} + +/// One durable unit checkpoint. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionCheckpoint { + /// Stage containing the unit. + pub stage: String, + /// Stable unit key. + pub unit_key: String, + /// `started`, `completed`, or `failed`. + pub status: String, + /// Claim generation that last touched it. + pub lease_generation: i64, + /// Attempt count for this unit. + pub attempts: i32, + /// Structured bounded details. + pub detail: serde_json::Value, + /// Last failure. + pub error: Option, + /// Start time. + pub started_at: DateTime, + /// Completion time. + pub completed_at: Option>, +} + +/// Full inspect response. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionInspection { + /// Durable request. + pub request: DeletionRequest, + /// Explicit approval evidence, if present. + pub approval: Option, + /// Unit checkpoints. + pub checkpoints: Vec, +} + +/// Explicit approval evidence. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionApproval { + /// Hex frozen inventory digest. + pub inventory_digest: String, + /// Approving operator identity. + pub approved_by: String, + /// Optional approval note. + pub note: Option, + /// Approval timestamp. + pub approved_at: DateTime, +} + +/// Monotonic lease token required by every execution mutation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LeaseToken { + /// Request id. + pub request_id: Uuid, + /// Executor identity. + pub owner: String, + /// Monotonic lease generation. + pub generation: i64, + /// Target community. + pub community_id: CommunityId, + /// Community fence generation, once fenced. + pub fence_generation: Option, +} + +/// A claimed request with its durable token. +#[derive(Debug, Clone)] +pub struct ClaimedDeletion { + /// Request snapshot. + pub request: DeletionRequest, + /// Required token. + pub lease: LeaseToken, +} + +/// Short-lived durable lease for an external serving side effect. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServingWriteLease { + /// Lease row identifier. + pub id: Uuid, + /// Community protected by this lease. + pub community_id: CommunityId, + /// Operation category for diagnostics. + pub operation: String, + /// Process/executor identity. + pub owner: String, + /// Monotonic lease generation. + pub generation: i64, + /// Community fence generation observed when the lease was acquired. + pub fence_generation: i64, + /// Lease expiry. + pub lease_until: DateTime, +} + +/// Validate the minimum catalog contract used by serving-path fences. +pub const REQUIRED_SERVING_TABLES: &[&str] = &[ + "communities", + "community_serving_write_leases", + "community_deletion_requests", +]; + +/// Bounded-cardinality operational snapshot for the hot serving-lease table. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ServingLeaseStats { + /// Unexpired serving-write leases. + pub active: i64, + /// Expired rows awaiting cleanup. + pub expired: i64, + /// PostgreSQL's estimated dead tuples for the lease table. + pub dead_tuples: i64, +} + +/// PostgreSQL deletion adapter. Clone is cheap. +#[derive(Clone)] +pub struct DeletionStore { + pool: PgPool, +} + +impl DeletionStore { + /// Construct from the writer pool used by [`crate::Db`]. + pub(crate) fn new(pool: PgPool) -> Self { + Self { pool } + } + + /// Check deletion control-plane/schema connectivity. + /// + /// Probe the deployed catalog rather than SQLx's migration ledger. Buzz also + /// supports desired-state schema application through `pgschema`, which creates + /// the same deletion objects without creating `_sqlx_migrations`. + pub async fn ping(&self) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT to_regclass('community_deletion_requests') IS NOT NULL", + ) + .fetch_one(&self.pool) + .await + .unwrap_or(false) + } + + /// Persist a request. Only active non-tombstone communities may be submitted. + pub async fn submit( + &self, + community_host: &str, + requested_by: &str, + reason: Option<&str>, + ) -> Result { + let row = sqlx::query( + r#" + WITH target AS ( + SELECT id, host + FROM communities + WHERE lower(host) = lower($1) + AND deletion_state = 'active' + AND deleted_at IS NULL + ), inserted AS ( + INSERT INTO community_deletion_requests + (community_id, community_host, requested_by, reason) + SELECT id, host, $2, $3 FROM target + ON CONFLICT (community_id) WHERE stage <> 'aborted' DO NOTHING + RETURNING * + ) + SELECT * FROM inserted + UNION ALL + SELECT request.* + FROM community_deletion_requests request + JOIN target ON target.id = request.community_id + WHERE request.stage = 'submitted' + AND request.requested_by = $2 + AND NOT EXISTS (SELECT 1 FROM inserted) + LIMIT 1 + "#, + ) + .bind(community_host) + .bind(requested_by) + .bind(reason) + .fetch_optional(&self.pool) + .await?; + match row { + Some(row) => row_to_request(row), + None => Err(DbError::DeletionSafety(format!( + "community {community_host:?} is missing, already requested, fenced, or tombstoned" + ))), + } + } + + /// List requests newest first with a hard bound. + pub async fn list(&self, limit: i64) -> Result> { + let rows = sqlx::query( + "SELECT * FROM community_deletion_requests ORDER BY created_at DESC LIMIT $1", + ) + .bind(limit.clamp(1, 1000)) + .fetch_all(&self.pool) + .await?; + rows.into_iter().map(row_to_request).collect() + } + + /// Read one request. + pub async fn get(&self, request_id: Uuid) -> Result { + let row = sqlx::query("SELECT * FROM community_deletion_requests WHERE id = $1") + .bind(request_id) + .fetch_optional(&self.pool) + .await? + .ok_or_else(|| DbError::NotFound(format!("community deletion {request_id}")))?; + row_to_request(row) + } + + /// Inspect request, approval, checkpoints, and retention holds. + pub async fn inspect(&self, request_id: Uuid) -> Result { + let request = self.get(request_id).await?; + let approval_row = sqlx::query( + "SELECT inventory_digest, approved_by, note, approved_at \ + FROM community_deletion_approvals WHERE request_id = $1", + ) + .bind(request_id) + .fetch_optional(&self.pool) + .await?; + let approval = approval_row + .map(|row| { + Ok::(DeletionApproval { + inventory_digest: hex::encode(row.try_get::, _>("inventory_digest")?), + approved_by: row.try_get("approved_by")?, + note: row.try_get("note")?, + approved_at: row.try_get("approved_at")?, + }) + }) + .transpose()?; + let checkpoints = sqlx::query( + "SELECT stage, unit_key, status, lease_generation, attempts, detail, error, \ + started_at, completed_at \ + FROM community_deletion_checkpoints WHERE request_id = $1 ORDER BY sequence", + ) + .bind(request_id) + .fetch_all(&self.pool) + .await? + .into_iter() + .map(|row| { + Ok(DeletionCheckpoint { + stage: row.try_get("stage")?, + unit_key: row.try_get("unit_key")?, + status: row.try_get("status")?, + lease_generation: row.try_get("lease_generation")?, + attempts: row.try_get("attempts")?, + detail: row.try_get("detail")?, + error: row.try_get("error")?, + started_at: row.try_get("started_at")?, + completed_at: row.try_get("completed_at")?, + }) + }) + .collect::>>()?; + Ok(DeletionInspection { + request, + approval, + checkpoints, + }) + } + + /// Validate the deletion catalog contract required by relay serving. + pub async fn validate_serving_catalog(&self) -> Result<()> { + let runtime_columns = sqlx::query( + "SELECT attname, format_type(atttypid, atttypmod) AS type_name, attnotnull \ + FROM pg_attribute WHERE attrelid = 'communities'::regclass \ + AND attname IN ('deletion_state', 'deletion_fence_generation', 'deleted_at') \ + AND NOT attisdropped ORDER BY attname", + ) + .fetch_all(&self.pool) + .await?; + let column_contract = runtime_columns + .iter() + .map(|row| { + Ok::<_, DbError>(( + row.try_get::("attname")?, + row.try_get::("type_name")?, + row.try_get::("attnotnull")?, + )) + }) + .collect::>>()?; + let expected_columns = BTreeSet::from([ + ( + "deleted_at".to_string(), + "timestamp with time zone".to_string(), + false, + ), + ( + "deletion_fence_generation".to_string(), + "bigint".to_string(), + true, + ), + ("deletion_state".to_string(), "text".to_string(), true), + ]); + if column_contract != expected_columns { + return Err(DbError::DeletionSafety( + "community serving fence columns are missing or incompatible".to_string(), + )); + } + + let required_tables = REQUIRED_SERVING_TABLES + .iter() + .copied() + .map(str::to_owned) + .collect::>(); + let required_table_names = REQUIRED_SERVING_TABLES + .iter() + .map(ToString::to_string) + .collect::>(); + let live_tables: BTreeSet = sqlx::query_scalar( + "SELECT table_name FROM information_schema.tables \ + WHERE table_schema = 'public' AND table_name = ANY($1) \ + ORDER BY table_name", + ) + .bind(&required_table_names) + .fetch_all(&self.pool) + .await? + .into_iter() + .collect(); + if live_tables != required_tables { + return Err(DbError::DeletionSafety(format!( + "community serving fence tables missing: {}", + required_tables + .difference(&live_tables) + .cloned() + .collect::>() + .join(",") + ))); + } + + let required_fences = EXPECTED_SCOPED_TABLES + .iter() + .copied() + .map(str::to_owned) + .collect::>(); + let live_fences = self.live_fenced_tables().await?; + let missing_fences = required_fences + .difference(&live_fences) + .cloned() + .collect::>(); + if !missing_fences.is_empty() { + return Err(DbError::DeletionSafety(format!( + "community serving write fences missing: {}", + missing_fences.join(",") + ))); + } + + let required_objects_present: bool = sqlx::query_scalar( + "SELECT to_regprocedure('community_deletion_lock_key(uuid)') IS NOT NULL \ + AND to_regprocedure('community_write_allowed(uuid)') IS NOT NULL \ + AND (SELECT provolatile = 'v' FROM pg_proc \ + WHERE oid = 'community_write_allowed(uuid)'::regprocedure) \ + AND to_regprocedure('assert_community_write_allowed(uuid)') IS NOT NULL \ + AND to_regprocedure('enforce_community_write_fence()') IS NOT NULL \ + AND EXISTS (SELECT 1 FROM pg_trigger t \ + JOIN pg_class c ON c.oid = t.tgrelid \ + JOIN pg_proc p ON p.oid = t.tgfoid \ + WHERE c.relname = 'communities' \ + AND p.proname = 'enforce_community_tombstone' \ + AND NOT t.tgisinternal AND t.tgenabled = 'O')", + ) + .fetch_one(&self.pool) + .await?; + if !required_objects_present { + return Err(DbError::DeletionSafety( + "community serving fence functions or tombstone trigger are missing".to_string(), + )); + } + Ok(()) + } + + /// Validate the exact live scoped-table and write-fence catalog for destruction. + /// + /// Exact table and fence equality rejects unknown tenant data even + /// while unrelated SQLx migrations continue to advance. This pool-based + /// check is an early rejection only; destructive transactions revalidate + /// on their own connection under [`SCHEMA_DESTRUCTION_LOCK_KEY`]. + pub async fn validate_catalog(&self) -> Result<()> { + let mut conn = self.pool.acquire().await?; + validate_catalog_on(&mut conn).await + } + + /// Build and validate a live PostgreSQL schema inventory. + /// + /// Row counts are observational evidence captured at submission. They bind + /// operator approval to the target's visible PostgreSQL footprint, but the + /// executor still revalidates the structural catalog and proves zero rows + /// after purge rather than requiring these live counts to remain unchanged. + pub async fn inventory_schema(&self, community: CommunityId) -> Result { + self.validate_catalog().await?; + let live_tables = self.live_scoped_tables().await?; + let fenced_tables = self.live_fenced_tables().await?; + let mut row_counts = BTreeMap::new(); + for table in &live_tables { + let sql = format!("SELECT count(*)::BIGINT FROM {table} WHERE community_id = $1"); + let count: i64 = sqlx::query_scalar(AssertSqlSafe(sql)) + .bind(community.as_uuid()) + .fetch_one(&self.pool) + .await?; + row_counts.insert(table.clone(), count); + } + Ok(SchemaManifest { + scoped_tables: live_tables.into_iter().collect(), + row_counts, + fenced_tables: fenced_tables.into_iter().collect(), + }) + } + + /// Freeze inventory and move submitted → inventoried atomically. + pub async fn freeze_inventory( + &self, + request_id: Uuid, + inventory: &FrozenInventory, + ) -> Result { + validate_storage_manifest(&inventory.storage)?; + let digest = inventory.digest()?; + let schema = serde_json::to_value(&inventory.schema)?; + let storage = serde_json::to_value(&inventory.storage)?; + let frozen = serde_json::to_value(inventory)?; + let row = sqlx::query( + r#" + UPDATE community_deletion_requests + SET stage = 'inventoried', schema_manifest = $2, storage_manifest = $3, + inventory_manifest = $4, inventory_digest = $5, + inventory_frozen_at = now(), updated_at = now(), + last_error = NULL, last_error_at = NULL + WHERE id = $1 AND stage = 'submitted' AND blocked_at IS NULL + RETURNING * + "#, + ) + .bind(request_id) + .bind(schema) + .bind(storage) + .bind(frozen) + .bind(digest) + .fetch_optional(&self.pool) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {request_id} is not an unblocked submitted request" + )) + })?; + row_to_request(row) + } + + /// Approve the exact frozen inventory and move inventoried → approved. + pub async fn approve( + &self, + request_id: Uuid, + approved_by: &str, + note: Option<&str>, + ) -> Result { + let mut tx = self.pool.begin().await?; + let (community_id, digest, inventory_manifest): (Uuid, Vec, serde_json::Value) = + sqlx::query_as( + "SELECT community_id, inventory_digest, inventory_manifest \ + FROM community_deletion_requests \ + WHERE id = $1 AND stage = 'inventoried' AND blocked_at IS NULL FOR UPDATE", + ) + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {request_id} is not an unblocked inventoried request" + )) + })?; + let inventory: FrozenInventory = serde_json::from_value(inventory_manifest)?; + let recomputed_digest = inventory.digest()?; + if digest.as_slice() != recomputed_digest { + return Err(DbError::DeletionSafety(format!( + "deletion {request_id} frozen inventory digest does not match its manifest" + ))); + } + sqlx::query( + "INSERT INTO community_deletion_approvals \ + (request_id, community_id, inventory_digest, approved_by, note) \ + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(request_id) + .bind(community_id) + .bind(&digest) + .bind(approved_by) + .bind(note) + .execute(&mut *tx) + .await?; + let row = sqlx::query( + "UPDATE community_deletion_requests \ + SET stage = 'approved', updated_at = now(), next_attempt_at = now() \ + WHERE id = $1 AND stage = 'inventoried' AND blocked_at IS NULL \ + RETURNING *", + ) + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {request_id} changed before approval could be recorded" + )) + })?; + tx.commit().await?; + row_to_request(row) + } + + /// Claim a specific runnable request. Expired claims may be reclaimed. + pub async fn claim_specific( + &self, + request_id: Uuid, + owner: &str, + lease_duration: Duration, + ) -> Result> { + self.claim(Some(request_id), owner, lease_duration).await + } + + /// Claim the oldest runnable request. Expired claims may be reclaimed. + pub async fn claim_next( + &self, + owner: &str, + lease_duration: Duration, + ) -> Result> { + self.claim(None, owner, lease_duration).await + } + + async fn claim( + &self, + request_id: Option, + owner: &str, + lease_duration: Duration, + ) -> Result> { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + let candidate = sqlx::query( + r#"SELECT request.* FROM community_deletion_requests request + JOIN community_deletion_approvals approval ON approval.request_id = request.id + AND approval.community_id = request.community_id + AND approval.inventory_digest = request.inventory_digest + WHERE ($1::uuid IS NULL OR request.id = $1) + AND request.stage IN ('approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified') + AND request.blocked_at IS NULL AND request.next_attempt_at <= now() + AND (request.lease_until IS NULL OR request.lease_until < now()) + ORDER BY request.created_at, request.id + FOR UPDATE OF request SKIP LOCKED LIMIT 1"#, + ) + .bind(request_id) + .fetch_optional(&mut *tx) + .await?; + let Some(candidate_row) = candidate else { + tx.commit().await?; + return Ok(None); + }; + let candidate = row_to_request(candidate_row)?; + if let Err(error) = validate_catalog_on(&mut tx).await { + let message = bound_text(&error.to_string(), 4096); + sqlx::query( + r#"INSERT INTO community_deletion_checkpoints + (request_id, stage, unit_key, status, lease_generation, error) + VALUES ($1, $2, 'claim:catalog_validation', 'failed', $3, $4) + ON CONFLICT (request_id, stage, unit_key) DO UPDATE + SET status = 'failed', lease_generation = EXCLUDED.lease_generation, + attempts = community_deletion_checkpoints.attempts + 1, + error = EXCLUDED.error, completed_at = NULL"#, + ) + .bind(candidate.id) + .bind(candidate.stage.to_string()) + .bind(candidate.lease_generation.max(1)) + .bind(&message) + .execute(&mut *tx) + .await?; + sqlx::query( + "UPDATE community_deletion_requests SET blocked_at = now(), blocked_reason = $2, \ + last_error = $2, last_error_at = now(), lease_owner = NULL, lease_until = NULL, \ + updated_at = now() WHERE id = $1", + ) + .bind(candidate.id) + .bind(&message) + .execute(&mut *tx) + .await?; + tx.commit().await?; + return Ok(None); + } + let row = sqlx::query( + "UPDATE community_deletion_requests SET lease_owner = $2, \ + lease_generation = lease_generation + 1, \ + lease_until = now() + make_interval(secs => $3), attempts = attempts + 1, \ + updated_at = now() WHERE id = $1 RETURNING *", + ) + .bind(candidate.id) + .bind(owner) + .bind(lease_seconds) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + let request = row_to_request(row)?; + let lease = LeaseToken { + request_id: request.id, + owner: owner.to_owned(), + generation: request.lease_generation, + community_id: request.community_id, + fence_generation: request.fence_generation, + }; + Ok(Some(ClaimedDeletion { request, lease })) + } + + /// Verify that a deletion lease/fence token is still current for a stage. + pub async fn verify_execution_token( + &self, + token: &LeaseToken, + stage: DeletionStage, + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + if let Some(generation) = token.fence_generation { + verify_lease_and_fence(&mut tx, token, stage, generation).await?; + } else { + verify_lease(&mut tx, token, stage).await?; + } + tx.commit().await?; + Ok(()) + } + + /// Renew an owned claim and persist executor liveness. + pub async fn heartbeat( + &self, + token: &LeaseToken, + executor_mode: &str, + lease_duration: Duration, + draining: bool, + ) -> Result<()> { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + let affected = sqlx::query( + "UPDATE community_deletion_requests request \ + SET lease_until = now() + make_interval(secs => $4), updated_at = now() \ + WHERE request.id = $1 AND request.lease_owner = $2 \ + AND request.lease_generation = $3 AND request.lease_until >= now() \ + AND request.blocked_at IS NULL \ + AND request.stage IN ('approved', 'fenced', 'drained', 'bindings_removed', \ + 'postgres_purged', 'cache_purged', 'logically_verified') \ + AND EXISTS (SELECT 1 FROM community_deletion_approvals approval \ + WHERE approval.request_id = request.id \ + AND approval.community_id = request.community_id \ + AND approval.inventory_digest = request.inventory_digest)", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(lease_seconds) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(stale_lease_error(token)); + } + sqlx::query( + "INSERT INTO community_deletion_executor_heartbeats \ + (executor_id, mode, request_id, draining) VALUES ($1, $2, $3, $4) \ + ON CONFLICT (executor_id) DO UPDATE SET mode = EXCLUDED.mode, \ + request_id = EXCLUDED.request_id, heartbeat_at = now(), \ + draining = EXCLUDED.draining, stopped_at = NULL", + ) + .bind(&token.owner) + .bind(executor_mode) + .bind(token.request_id) + .bind(draining) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Mark an executor stopped and release its current claim if still owned. + pub async fn stop_executor(&self, token: Option<&LeaseToken>, executor_id: &str) -> Result<()> { + let mut tx = self.pool.begin().await?; + if let Some(token) = token { + sqlx::query( + "UPDATE community_deletion_requests \ + SET lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .execute(&mut *tx) + .await?; + } + sqlx::query( + "UPDATE community_deletion_executor_heartbeats \ + SET request_id = NULL, draining = true, heartbeat_at = now(), stopped_at = now() \ + WHERE executor_id = $1", + ) + .bind(executor_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Persist quiescing intent before waiting for active serving leases. + /// + /// This is the irreversible fail-closed point: a request intentionally has + /// no automatic unquiesce/unblock transition after operator approval. + /// + /// The transition takes the same exclusive advisory lock as serving lease + /// acquisition, so after commit no newer external effect can be admitted. + /// Already-acquired leases remain renewable, verifiable, and releasable so + /// admitted remote effects retain their exclusion proof until completion. + pub async fn begin_quiescing(&self, token: &LeaseToken) -> Result<()> { + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, DeletionStage::Approved).await?; + sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await?; + verify_lease(&mut tx, token, DeletionStage::Approved).await?; + let (generation, archived_at): (i64, Option>) = sqlx::query_as( + "SELECT deletion_fence_generation, archived_at FROM communities WHERE id = $1 FOR UPDATE", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + sqlx::query( + "UPDATE community_deletion_requests SET pre_quiesce_archived_at = $2, \ + quiescing_started_at = now(), updated_at = now() \ + WHERE id = $1 AND quiescing_started_at IS NULL", + ) + .bind(token.request_id) + .bind(archived_at) + .execute(&mut *tx) + .await?; + set_executor_gucs(&mut tx, token.community_id, generation).await?; + let affected = sqlx::query( + "UPDATE communities SET deletion_state = 'quiescing', \ + archived_at = COALESCE(archived_at, now()) \ + WHERE id = $1 AND deletion_state IN ('active', 'quiescing') \ + AND deleted_at IS NULL", + ) + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "community {} cannot enter quiescing", + token.community_id + ))); + } + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Approved, + "quiesce_serving_writes", + serde_json::json!({"community_state": "quiescing"}), + ) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Acquire the universal durable fence after all pre-quiesce serving leases drain. + pub async fn fence(&self, token: &LeaseToken) -> Result { + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, DeletionStage::Approved).await?; + sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await?; + verify_lease(&mut tx, token, DeletionStage::Approved).await?; + let active_serving_writes = sqlx::query( + "SELECT count(*)::BIGINT AS active_count, \ + COALESCE(array_agg(DISTINCT operation ORDER BY operation), ARRAY[]::TEXT[]) AS operations \ + FROM community_serving_write_leases \ + WHERE community_id = $1 AND lease_until >= now()", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + let active_count: i64 = active_serving_writes.try_get("active_count")?; + if active_count > 0 { + return Err(DbError::ServingWritesNotDrained { + community_id: *token.community_id.as_uuid(), + active_count, + operations: active_serving_writes.try_get("operations")?, + }); + } + let current_generation: i64 = sqlx::query_scalar( + "SELECT deletion_fence_generation FROM communities WHERE id = $1 FOR UPDATE", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + let generation = current_generation.checked_add(1).ok_or_else(|| { + DbError::DeletionSafety("community deletion fence generation overflow".to_string()) + })?; + set_executor_gucs(&mut tx, token.community_id, generation).await?; + let affected = sqlx::query( + "UPDATE communities SET deletion_state = 'fenced', \ + deletion_fence_generation = $2, archived_at = COALESCE(archived_at, now()) \ + WHERE id = $1 AND deletion_state = 'quiescing'", + ) + .bind(token.community_id.as_uuid()) + .bind(generation) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "community {} is no longer quiescing while fencing", + token.community_id + ))); + } + advance_request_tx( + &mut tx, + token, + DeletionStage::Approved, + DeletionStage::Fenced, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Approved, + "activate_fence", + serde_json::json!({"fence_generation": generation}), + ) + .await?; + tx.commit().await?; + Ok(generation) + } + + /// Freeze the exact post-fence storage binding manifest. + pub async fn freeze_destructive_storage_manifest( + &self, + token: &LeaseToken, + manifest: &StorageManifest, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + // Serialize the freeze boundary with chunk INSERTs. The database trigger + // takes the same request-row lock before admitting each new chunk. + sqlx::query("SELECT id FROM community_deletion_requests WHERE id = $1 FOR UPDATE") + .bind(token.request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion request {} disappeared before manifest freeze", + token.request_id + )) + })?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + validate_storage_manifest(manifest)?; + // The chunk rows are the concrete delete list; the freeze commits only + // if they hash to the manifest's frozen per-prefix digests. Loading the + // full chunk stream is a one-time freeze-boundary cost proportional to + // this community's bindings, never the fleet bucket. + let chunks: Vec<(i64, String, sqlx::types::Json>)> = sqlx::query_as( + "SELECT chunk_no, prefix, keys FROM community_deletion_manifest_keys \ + WHERE request_id = $1 ORDER BY chunk_no", + ) + .bind(token.request_id) + .fetch_all(&mut *tx) + .await?; + validate_manifest_key_chunks(manifest, &chunks)?; + let affected = sqlx::query( + "UPDATE community_deletion_requests \ + SET destructive_storage_manifest = COALESCE(destructive_storage_manifest, $4), \ + destructive_storage_frozen_at = COALESCE(destructive_storage_frozen_at, now()), \ + updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3 \ + AND stage = 'fenced' \ + AND (destructive_storage_manifest IS NULL \ + OR destructive_storage_manifest = $4) \ + RETURNING id", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(serde_json::to_value(manifest)?) + .fetch_optional(&mut *tx) + .await?; + if affected.is_none() { + return Err(DbError::DeletionSafety(format!( + "destructive storage manifest changed or deletion lease is stale for request {}", + token.request_id + ))); + } + tx.commit().await?; + Ok(()) + } + + /// Remove key chunks left by an interrupted destructive freeze. + /// + /// The chunk-table guard rejects this once the destructive manifest has + /// frozen, so a retried freeze can only rewrite chunks that were never + /// bound to a committed manifest. + pub async fn clear_manifest_key_chunks(&self, token: &LeaseToken) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + sqlx::query("DELETE FROM community_deletion_manifest_keys WHERE request_id = $1") + .bind(token.request_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Append one immutable chunk of the destructive key list. + pub async fn append_manifest_key_chunk( + &self, + token: &LeaseToken, + chunk_no: i64, + prefix: &str, + keys: &[String], + ) -> Result<()> { + if keys.is_empty() { + return Err(DbError::DeletionSafety( + "refusing to persist an empty manifest key chunk".to_string(), + )); + } + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + sqlx::query( + "INSERT INTO community_deletion_manifest_keys \ + (request_id, chunk_no, prefix, keys) VALUES ($1, $2, $3, $4)", + ) + .bind(token.request_id) + .bind(chunk_no) + .bind(prefix) + .bind(sqlx::types::Json(keys)) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Return the next frozen chunk not yet confirmed deleted, in chunk order. + pub async fn next_pending_manifest_chunk( + &self, + token: &LeaseToken, + ) -> Result> { + let row: Option<(i64, String, sqlx::types::Json>)> = sqlx::query_as( + "SELECT chunk_no, prefix, keys FROM community_deletion_manifest_keys \ + WHERE request_id = $1 AND deleted_at IS NULL ORDER BY chunk_no LIMIT 1", + ) + .bind(token.request_id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|(chunk_no, prefix, keys)| ManifestKeyChunk { + chunk_no, + prefix, + keys: keys.0, + })) + } + + /// Return `(total, deleted)` chunk counts for one request. + pub async fn manifest_chunk_progress(&self, request_id: Uuid) -> Result<(i64, i64)> { + sqlx::query_as( + "SELECT count(*), count(deleted_at) FROM community_deletion_manifest_keys \ + WHERE request_id = $1", + ) + .bind(request_id) + .fetch_one(&self.pool) + .await + .map_err(Into::into) + } + + /// Stamp one chunk's keys durably removed and checkpoint it atomically. + pub async fn mark_manifest_chunk_deleted( + &self, + token: &LeaseToken, + chunk_no: i64, + detail: serde_json::Value, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Drained, generation).await?; + let affected = sqlx::query( + "UPDATE community_deletion_manifest_keys SET deleted_at = now() \ + WHERE request_id = $1 AND chunk_no = $2 AND deleted_at IS NULL", + ) + .bind(token.request_id) + .bind(chunk_no) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "manifest key chunk {chunk_no} is missing or already stamped for request {}", + token.request_id + ))); + } + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Drained, + &format!("chunk:{chunk_no}"), + detail, + ) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Record one completed fleet-wide taxonomy sweep. + pub async fn record_taxonomy_sweep( + &self, + started_at: DateTime, + listed_objects: u64, + unknown_object_count: u64, + unknown_key_sample: &[String], + object_cap: u64, + ) -> Result { + let listed = i64::try_from(listed_objects) + .map_err(|_| DbError::DeletionSafety("sweep object count overflow".to_string()))?; + let unknown = i64::try_from(unknown_object_count) + .map_err(|_| DbError::DeletionSafety("sweep unknown count overflow".to_string()))?; + let cap = i64::try_from(object_cap) + .map_err(|_| DbError::DeletionSafety("sweep object cap overflow".to_string()))?; + // Completion is authoritative database time. Small positive sweeper + // skew is clamped at that boundary; materially future starts are rejected. + let row: Option<(Uuid, DateTime, DateTime)> = sqlx::query_as( + "INSERT INTO storage_taxonomy_sweeps \ + (started_at, completed_at, listed_objects, unknown_object_count, \ + unknown_key_sample, object_cap) \ + SELECT LEAST($1, db_now), db_now, $2, $3, $4, $5 \ + FROM (SELECT clock_timestamp() AS db_now) clock \ + WHERE $1 <= db_now + interval '5 minutes' \ + RETURNING id, started_at, completed_at", + ) + .bind(started_at) + .bind(listed) + .bind(unknown) + .bind(sqlx::types::Json(unknown_key_sample)) + .bind(cap) + .fetch_optional(&self.pool) + .await?; + let (id, started_at, completed_at) = row.ok_or_else(|| { + DbError::DeletionSafety( + "taxonomy sweep start time is more than five minutes in the future".to_string(), + ) + })?; + Ok(TaxonomySweep { + id, + started_at, + completed_at, + listed_objects: listed, + unknown_object_count: unknown, + unknown_key_sample: unknown_key_sample.to_vec(), + object_cap: cap, + }) + } + + /// Return the most recently completed taxonomy sweep, if any. + pub async fn latest_taxonomy_sweep(&self) -> Result> { + let row: Option = sqlx::query_as( + "SELECT id, started_at, completed_at, listed_objects, unknown_object_count, \ + unknown_key_sample, object_cap \ + FROM storage_taxonomy_sweeps ORDER BY completed_at DESC LIMIT 1", + ) + .fetch_optional(&self.pool) + .await?; + Ok(row.map( + |(id, started_at, completed_at, listed, unknown, sample, cap)| TaxonomySweep { + id, + started_at, + completed_at, + listed_objects: listed, + unknown_object_count: unknown, + unknown_key_sample: sample.0, + object_cap: cap, + }, + )) + } + + /// Return whether all pre-fence external side-effect leases have expired or released. + pub async fn serving_writes_drained(&self, community: CommunityId) -> Result { + sqlx::query_scalar( + "SELECT NOT EXISTS(SELECT 1 FROM community_serving_write_leases \ + WHERE community_id = $1 AND lease_until >= now())", + ) + .bind(community.as_uuid()) + .fetch_one(&self.pool) + .await + .map_err(Into::into) + } + + /// Verify fence ownership and record that serving writes drained. + pub async fn mark_drained(&self, token: &LeaseToken) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + let active_serving_writes: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases \ + WHERE community_id = $1 AND lease_until >= now())", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + if active_serving_writes { + return Err(DbError::DeletionSafety( + "serving writes have not drained".to_string(), + )); + } + advance_request_tx( + &mut tx, + token, + DeletionStage::Fenced, + DeletionStage::Drained, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Fenced, + "serving_writes_drained", + serde_json::json!({"fence_generation": generation}), + ) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Mark storage binding removal after adapter verification. + pub async fn mark_bindings_removed( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + self.advance_with_checkpoint( + token, + DeletionStage::Drained, + DeletionStage::BindingsRemoved, + "remove_storage_bindings", + detail, + ) + .await + } + + /// Purge every scoped PostgreSQL table, preserve the community tombstone, and + /// move bindings_removed → postgres_purged in one transaction. + pub async fn purge_postgres(&self, token: &LeaseToken) -> Result> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + // Revalidate the exact catalog inside the purge transaction under the + // shared schema/destruction lock. Migrations hold the exclusive + // counterpart for their entire run, so no migration can commit a new + // scoped table between this validation and the purge commit. + lock_schema_destruction_shared(&mut tx).await?; + validate_catalog_on(&mut tx).await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::BindingsRemoved, generation).await?; + set_executor_gucs(&mut tx, token.community_id, generation).await?; + // Migration 0011 fences hard deletion of NIP-RS rows against legacy + // writers. Whole-community deletion is an intentional hard-delete path, + // and the transaction is already bound to an approved, fenced tenant. + sqlx::query("SELECT set_config('buzz.nip_rs_hard_delete', 'on', true)") + .execute(&mut *tx) + .await?; + + // Preserve deployment-global operator evidence while severing tenant provenance. + for table in ["product_feedback", "rate_limit_violations"] { + let sql = format!("UPDATE {table} SET community_id = NULL WHERE community_id = $1"); + let affected = sqlx::query(AssertSqlSafe(sql)) + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await? + .rows_affected(); + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::BindingsRemoved, + &format!("clear_provenance:{table}"), + serde_json::json!({"rows": affected}), + ) + .await?; + } + + let mut deleted = BTreeMap::new(); + // The order is child-before-parent/FK-safe, not alphabetical. Cascades + // can make later units observe zero rows; each scoped WHERE stays idempotent. + for table in PURGE_SCOPED_TABLES { + let sql = format!("DELETE FROM {table} WHERE community_id = $1"); + let affected = sqlx::query(AssertSqlSafe(sql)) + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await? + .rows_affected(); + deleted.insert((*table).to_owned(), affected); + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::BindingsRemoved, + &format!("purge:{table}"), + serde_json::json!({"rows": affected}), + ) + .await?; + } + + let affected = sqlx::query( + "UPDATE communities SET deletion_state = 'tombstone', \ + deleted_at = COALESCE(deleted_at, now()), \ + archived_at = COALESCE(archived_at, now()), \ + signing_key = NULL, icon = NULL \ + WHERE id = $1 AND deletion_state = 'fenced' \ + AND deletion_fence_generation = $2", + ) + .bind(token.community_id.as_uuid()) + .bind(generation) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "community {} tombstone update affected {affected} rows", + token.community_id + ))); + } + advance_request_tx( + &mut tx, + token, + DeletionStage::BindingsRemoved, + DeletionStage::PostgresPurged, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::BindingsRemoved, + "postgres_tombstone_committed", + serde_json::to_value(&deleted)?, + ) + .await?; + tx.commit().await?; + Ok(deleted) + } + + /// Mark cache purge after Redis adapter verification. + pub async fn mark_cache_purged( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + self.advance_with_checkpoint( + token, + DeletionStage::PostgresPurged, + DeletionStage::CachePurged, + "purge_cache_namespace", + detail, + ) + .await + } + + /// Verify PostgreSQL logical absence without advancing the cross-store stage. + /// + /// The caller must verify object storage and Redis too, then call + /// [`Self::mark_logically_verified`]. Keeping the transition separate makes + /// a crash after any partial verification safely repeat the whole proof. + pub async fn verify_postgres_logically_deleted(&self, token: &LeaseToken) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + // The absence proof is only as strong as the surface it iterates: + // validate the live catalog under the shared schema/destruction lock + // so a scoped table committed after the purge fails this stage closed + // instead of silently escaping verification. + lock_schema_destruction_shared(&mut tx).await?; + validate_catalog_on(&mut tx).await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::CachePurged, generation).await?; + let tombstone: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 \ + AND deletion_state = 'tombstone' AND deleted_at IS NOT NULL \ + AND deletion_fence_generation = $2)", + ) + .bind(token.community_id.as_uuid()) + .bind(generation) + .fetch_one(&mut *tx) + .await?; + if !tombstone { + return Err(DbError::DeletionSafety(format!( + "community {} tombstone/fence verification failed", + token.community_id + ))); + } + for table in EXPECTED_SCOPED_TABLES { + let sql = + format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE community_id = $1 LIMIT 1)"); + let remains: bool = sqlx::query_scalar(AssertSqlSafe(sql)) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + if remains { + return Err(DbError::DeletionSafety(format!( + "logical verification found tenant rows in {table}" + ))); + } + } + tx.commit().await?; + Ok(()) + } + + /// Commit the cross-store logical verification checkpoint and drop the + /// frozen key chunks in the same transaction. + /// + /// The chunk rows are working data, not audit evidence — per-prefix + /// counts, digests, and checkpoint history stay on the request row, and + /// the raw key list of a deleted community should not be retained. + /// Blocked requests never reach this transition, so their chunks survive + /// for resumption or operator inspection. + pub async fn mark_logically_verified( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::CachePurged, generation).await?; + advance_request_tx( + &mut tx, + token, + DeletionStage::CachePurged, + DeletionStage::LogicallyVerified, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::CachePurged, + "verify_cross_store_absence", + detail, + ) + .await?; + sqlx::query("DELETE FROM community_deletion_manifest_keys WHERE request_id = $1") + .bind(token.request_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Finish logical deletion and enter the physical-expiry pending state. + pub async fn mark_retention_pending( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::LogicallyVerified, generation) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::LogicallyVerified, + "retention_physical_expiry_pending", + detail, + ) + .await?; + let affected = sqlx::query( + "UPDATE community_deletion_requests \ + SET stage = 'retention_pending', completed_at = now(), updated_at = now(), \ + lease_owner = NULL, lease_until = NULL, retry_count = 0, retry_stage = NULL, \ + last_error = NULL, last_error_at = NULL \ + WHERE id = $1 AND stage = 'logically_verified' \ + AND lease_owner = $2 AND lease_generation = $3 AND lease_until >= now() \ + AND fence_generation = $4", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(generation) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(stale_lease_error(token)); + } + tx.commit().await?; + Ok(()) + } + + /// Persist a retryable unit failure and release the claim. + /// + /// The eighth consecutive failure at the same stage becomes a durable + /// block. A successful stage transition clears the streak; an operator may + /// use [`Self::unblock`] after remediating an exhausted dependency failure. + pub async fn record_retry( + &self, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + error: &str, + retry_after: Duration, + ) -> Result<()> { + let bounded = bound_text(error, 4096); + let retry_seconds = i64::try_from(retry_after.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, stage).await?; + let (retry_count, retry_stage): (i32, Option) = sqlx::query_as( + "SELECT retry_count, retry_stage FROM community_deletion_requests WHERE id = $1 FOR UPDATE", + ) + .bind(token.request_id) + .fetch_one(&mut *tx) + .await?; + let stage_name = stage.to_string(); + let consecutive_retries = if retry_stage.as_deref() == Some(stage_name.as_str()) { + retry_count.saturating_add(1) + } else { + 1 + }; + let exhausted = consecutive_retries >= 8; + checkpoint_failed_tx(&mut tx, token, stage, unit_key, &bounded).await?; + sqlx::query( + "UPDATE community_deletion_requests \ + SET retry_count = $7, retry_stage = $8, last_error = $4, last_error_at = now(), \ + next_attempt_at = CASE WHEN $6 THEN next_attempt_at \ + ELSE now() + make_interval(secs => $5) END, \ + blocked_at = CASE WHEN $6 THEN now() ELSE blocked_at END, \ + blocked_reason = CASE WHEN $6 THEN $4 ELSE blocked_reason END, \ + lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(&bounded) + .bind(retry_seconds) + .bind(exhausted) + .bind(consecutive_retries) + .bind(stage_name) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Terminally abort an approved or fenced request before object deletion begins. + pub async fn abort( + &self, + request_id: Uuid, + aborted_by: &str, + reason: &str, + ) -> Result { + let aborted_by = aborted_by.trim(); + let reason = reason.trim(); + if aborted_by.is_empty() || reason.is_empty() { + return Err(DbError::DeletionSafety( + "abort requires non-empty operator identity and reason".to_string(), + )); + } + let mut tx = self.pool.begin().await?; + let community_id: CommunityId = sqlx::query_scalar::<_, Uuid>( + "SELECT community_id FROM community_deletion_requests WHERE id = $1", + ) + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + .map(CommunityId::from_uuid) + .ok_or_else(|| DbError::NotFound(format!("community deletion {request_id}")))?; + // Every lifecycle transition takes the community lock before any row lock. + // Inverting this order lets abort and the executor deadlock each other. + sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") + .bind(community_id.as_uuid()) + .execute(&mut *tx) + .await?; + let row = sqlx::query("SELECT * FROM community_deletion_requests WHERE id = $1 FOR UPDATE") + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| DbError::NotFound(format!("community deletion {request_id}")))?; + let request = row_to_request(row)?; + if request.community_id != community_id { + return Err(DbError::DeletionSafety(format!( + "deletion {request_id} changed community while abort waited for the community lock" + ))); + } + if !matches!( + request.stage, + DeletionStage::Approved | DeletionStage::Fenced + ) { + return Err(DbError::DeletionSafety(format!( + "deletion {request_id} at stage {} cannot be aborted", + request.stage + ))); + } + let active_writes: i64 = sqlx::query_scalar( + "SELECT count(*)::BIGINT FROM community_serving_write_leases \ + WHERE community_id = $1 AND lease_until >= now()", + ) + .bind(request.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + if active_writes > 0 { + return Err(DbError::DeletionSafety(format!( + "deletion {request_id} cannot abort while {active_writes} serving write lease(s) remain active" + ))); + } + let (old_generation, current_archived_at): (i64, Option>) = sqlx::query_as( + "SELECT deletion_fence_generation, archived_at FROM communities WHERE id = $1 FOR UPDATE", + ) + .bind(request.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + let new_generation = old_generation.checked_add(1).ok_or_else(|| { + DbError::DeletionSafety("community deletion fence generation overflow".to_string()) + })?; + let restored_archived_at = if request.quiescing_started_at.is_some() { + request.pre_quiesce_archived_at + } else { + current_archived_at + }; + set_executor_gucs(&mut tx, request.community_id, new_generation).await?; + let restored = sqlx::query( + "UPDATE communities SET deletion_state = 'active', deletion_fence_generation = $2, \ + archived_at = $3 WHERE id = $1 AND deletion_state IN ('active', 'quiescing', 'fenced') \ + AND deleted_at IS NULL", + ) + .bind(request.community_id.as_uuid()) + .bind(new_generation) + .bind(restored_archived_at) + .execute(&mut *tx) + .await? + .rows_affected(); + if restored != 1 { + return Err(DbError::DeletionSafety(format!( + "community {} cannot be restored during abort", + request.community_id + ))); + } + sqlx::query( + "INSERT INTO community_deletion_checkpoints \ + (request_id, stage, unit_key, status, lease_generation, detail, completed_at) \ + VALUES ($1, $2, $3, 'completed', $4, $5, now())", + ) + .bind(request.id) + .bind(request.stage.to_string()) + .bind(format!("operator_abort:{}", Uuid::new_v4())) + .bind(request.lease_generation.max(1)) + .bind(serde_json::json!({ + "aborted_by": bound_text(aborted_by, 512), + "reason": bound_text(reason, 4096), + "old_fence_generation": old_generation, + "new_fence_generation": new_generation, + })) + .execute(&mut *tx) + .await?; + let row = sqlx::query( + "UPDATE community_deletion_requests SET stage = 'aborted', aborted_by = $2, \ + abort_reason = $3, aborted_at = now(), completed_at = now(), fence_generation = $4, \ + lease_owner = NULL, lease_until = NULL, lease_generation = lease_generation + 1, \ + blocked_at = NULL, blocked_reason = NULL, retry_count = 0, retry_stage = NULL, \ + next_attempt_at = now(), updated_at = now() WHERE id = $1 RETURNING *", + ) + .bind(request.id) + .bind(bound_text(aborted_by, 512)) + .bind(bound_text(reason, 4096)) + .bind(new_generation) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + row_to_request(row) + } + + /// operator checkpoint and only makes runnable stages immediately claimable. + /// Clear a fail-closed block after an operator has remediated its cause. + /// + /// Recovery preserves the immutable target, approval, inventory, stage, + /// fence generation, and prior failure checkpoint. It appends an auditable + pub async fn unblock( + &self, + request_id: Uuid, + unblocked_by: &str, + reason: &str, + ) -> Result { + let unblocked_by = unblocked_by.trim(); + let reason = reason.trim(); + if unblocked_by.is_empty() || reason.is_empty() { + return Err(DbError::InvalidData( + "unblock requires non-empty operator identity and remediation reason".to_string(), + )); + } + + let mut tx = self.pool.begin().await?; + let request_row = sqlx::query( + "SELECT * FROM community_deletion_requests \ + WHERE id = $1 AND blocked_at IS NOT NULL FOR UPDATE", + ) + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {request_id} is missing or is not blocked" + )) + })?; + let request = row_to_request(request_row)?; + if matches!( + request.stage, + DeletionStage::RetentionPending | DeletionStage::Aborted + ) { + return Err(DbError::DeletionSafety(format!( + "blocked deletion {request_id} at terminal stage {} cannot resume", + request.stage + ))); + } + if request.lease_owner.is_some() + && request + .lease_until + .is_some_and(|lease_until| lease_until >= Utc::now()) + { + return Err(DbError::DeletionSafety(format!( + "blocked deletion {request_id} still has a live executor lease" + ))); + } + + let prior_block = request.blocked_reason.clone(); + sqlx::query( + "INSERT INTO community_deletion_checkpoints \ + (request_id, stage, unit_key, status, lease_generation, detail, completed_at) \ + VALUES ($1, $2, $3, 'completed', $4, $5, now())", + ) + .bind(request.id) + .bind(request.stage.to_string()) + .bind(format!("operator_unblock:{}", Uuid::new_v4())) + .bind(request.lease_generation.max(1)) + .bind(serde_json::json!({ + "unblocked_by": bound_text(unblocked_by, 512), + "reason": bound_text(reason, 4096), + "previous_block": prior_block, + })) + .execute(&mut *tx) + .await?; + + let row = sqlx::query( + "UPDATE community_deletion_requests \ + SET blocked_at = NULL, blocked_reason = NULL, retry_count = 0, retry_stage = NULL, \ + last_error = NULL, last_error_at = NULL, next_attempt_at = now(), \ + lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND blocked_at IS NOT NULL RETURNING *", + ) + .bind(request_id) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + row_to_request(row) + } + + /// Persist a fail-closed setup failure before an identifiable request is claimed. + pub async fn block_preclaim_setup( + &self, + request_id: Uuid, + unit_key: &str, + error: &str, + ) -> Result { + let bounded = bound_text(error, 4096); + let mut tx = self.pool.begin().await?; + let request = + sqlx::query("SELECT * FROM community_deletion_requests WHERE id = $1 FOR UPDATE") + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| DbError::NotFound(format!("community deletion {request_id}")))?; + let request = row_to_request(request)?; + if matches!( + request.stage, + DeletionStage::RetentionPending | DeletionStage::Aborted + ) { + return Err(DbError::DeletionSafety(format!( + "deletion {request_id} at terminal stage {} cannot record a setup failure", + request.stage + ))); + } + if request.lease_owner.is_some() + && request + .lease_until + .is_some_and(|lease_until| lease_until >= Utc::now()) + { + return Err(DbError::DeletionSafety(format!( + "deletion {request_id} is leased by another executor" + ))); + } + sqlx::query( + r#" + INSERT INTO community_deletion_checkpoints + (request_id, stage, unit_key, status, lease_generation, detail, error) + VALUES ($1, $2, $3, 'failed', $4, $5, $6) + ON CONFLICT (request_id, stage, unit_key) DO UPDATE + SET status = 'failed', lease_generation = EXCLUDED.lease_generation, + attempts = community_deletion_checkpoints.attempts + 1, + detail = EXCLUDED.detail, error = EXCLUDED.error, completed_at = NULL + "#, + ) + .bind(request.id) + .bind(request.stage.to_string()) + .bind(unit_key) + .bind(request.lease_generation.max(1)) + .bind(serde_json::json!({"error": &bounded})) + .bind(&bounded) + .execute(&mut *tx) + .await?; + let row = sqlx::query( + "UPDATE community_deletion_requests \ + SET blocked_at = now(), blocked_reason = $2, last_error = $2, \ + last_error_at = now(), updated_at = now() \ + WHERE id = $1 RETURNING *", + ) + .bind(request_id) + .bind(&bounded) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + row_to_request(row) + } + + /// Persist a fail-closed permanent block and release the claim. + pub async fn block( + &self, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + error: &str, + ) -> Result<()> { + let bounded = bound_text(error, 4096); + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, stage).await?; + checkpoint_failed_tx(&mut tx, token, stage, unit_key, &bounded).await?; + sqlx::query( + "UPDATE community_deletion_requests \ + SET blocked_at = now(), blocked_reason = $4, last_error = $4, \ + last_error_at = now(), lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(&bounded) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Take the shared community deletion lock inside an existing transaction. + pub async fn guard_transaction( + &self, + tx: &mut Transaction<'_, Postgres>, + community: CommunityId, + ) -> Result<()> { + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(community.as_uuid()) + .execute(&mut **tx) + .await?; + let state: Option = sqlx::query_scalar( + "SELECT deletion_state FROM communities WHERE id = $1 AND deleted_at IS NULL", + ) + .bind(community.as_uuid()) + .fetch_optional(&mut **tx) + .await?; + match state.as_deref() { + Some("active") => Ok(()), + Some(other) => Err(DbError::AccessDenied(format!( + "community {community} is write-fenced ({other})" + ))), + None => Err(DbError::AccessDenied(format!( + "community {community} is missing or tombstoned" + ))), + } + } + + /// Take the shared community deletion lock inside an existing transaction + /// and authorize a final mutation under an already-admitted serving lease. + /// + /// The lease is checked in the same transaction as the mutation. During + /// quiescing, only this exact unexpired lease and fence generation may + /// finish; active communities continue to accept the admitted write too. + pub async fn guard_transaction_with_serving_lease( + &self, + tx: &mut Transaction<'_, Postgres>, + lease: &ServingWriteLease, + ) -> Result<()> { + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(lease.community_id.as_uuid()) + .execute(&mut **tx) + .await?; + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases lease \ + JOIN communities community ON community.id = lease.community_id \ + WHERE lease.id = $1 AND lease.community_id = $2 AND lease.owner = $3 \ + AND lease.generation = $4 AND lease.fence_generation = $5 \ + AND lease.lease_until >= now() AND community.deleted_at IS NULL \ + AND community.deletion_state IN ('active', 'quiescing') \ + AND community.deletion_fence_generation = lease.fence_generation)", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .fetch_one(&mut **tx) + .await?; + if !valid { + return Err(DbError::AccessDenied(format!( + "stale serving write lease {}", + lease.id + ))); + } + sqlx::query( + "SELECT set_config('buzz.serving_write_community', $1, true), \ + set_config('buzz.serving_write_lease_id', $2, true), \ + set_config('buzz.serving_write_owner', $3, true), \ + set_config('buzz.serving_write_generation', $4, true), \ + set_config('buzz.serving_write_fence_generation', $5, true)", + ) + .bind(lease.community_id.to_string()) + .bind(lease.id.to_string()) + .bind(&lease.owner) + .bind(lease.generation.to_string()) + .bind(lease.fence_generation.to_string()) + .execute(&mut **tx) + .await?; + Ok(()) + } + + /// Acquire a durable, expiring lease for an external serving side effect. + /// + /// The short transaction shares the same advisory lock as the destructive + /// fence. The fence therefore orders after all acquisitions that began + /// first, changes lifecycle state, then refuses every later acquisition. + pub async fn acquire_serving_write_lease( + &self, + community: CommunityId, + operation: &str, + owner: &str, + lease_duration: Duration, + ) -> Result { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + // The assertion owns both the shared ordering lock and the supported + // READ COMMITTED check. The lease table is trigger-excluded, so this + // explicit admission is its database-enforced write fence. + if let Err(error) = sqlx::query("SELECT assert_community_write_allowed($1)") + .bind(community.as_uuid()) + .execute(&mut *tx) + .await + { + if error.as_database_error().is_some_and(|database_error| { + database_error.code().as_deref() == Some("55000") + && database_error.message().starts_with("community write") + }) { + return Err(DbError::AccessDenied(format!( + "community {community} is write-fenced or missing" + ))); + } + return Err(error.into()); + } + let row = sqlx::query( + "INSERT INTO community_serving_write_leases \ + (community_id, operation, owner, fence_generation, lease_until) \ + SELECT id, $2, $3, deletion_fence_generation, \ + now() + make_interval(secs => $4) \ + FROM communities WHERE id = $1 AND deletion_state = 'active' \ + AND deleted_at IS NULL \ + RETURNING id, generation, fence_generation, lease_until", + ) + .bind(community.as_uuid()) + .bind(operation) + .bind(owner) + .bind(lease_seconds) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::AccessDenied(format!("community {community} is write-fenced or missing")) + })?; + let lease = ServingWriteLease { + id: row.try_get("id")?, + community_id: community, + operation: operation.to_owned(), + owner: owner.to_owned(), + generation: row.try_get("generation")?, + fence_generation: row.try_get("fence_generation")?, + lease_until: row.try_get("lease_until")?, + }; + tx.commit().await?; + Ok(lease) + } + + /// Renew an already-admitted external side-effect lease while the community + /// is active or quiescing. + /// + /// Quiescing rejects new acquisition, but the exact existing, unexpired + /// lease must remain renewable until its operation finishes. Otherwise the + /// heartbeat would abandon the exclusion proof while remote I/O may still + /// commit. Fence generation, owner, generation, expiry, and tombstone checks + /// continue to reject stale or post-fence renewal. + pub async fn renew_serving_write_lease( + &self, + lease: &mut ServingWriteLease, + lease_duration: Duration, + ) -> Result<()> { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(lease.community_id.as_uuid()) + .execute(&mut *tx) + .await?; + let lease_until: Option> = sqlx::query_scalar( + "UPDATE community_serving_write_leases lease \ + SET lease_until = now() + make_interval(secs => $6), heartbeat_at = now() \ + FROM communities community \ + WHERE lease.id = $1 AND lease.community_id = $2 AND lease.owner = $3 \ + AND lease.generation = $4 AND lease.fence_generation = $5 \ + AND lease.lease_until >= now() \ + AND community.id = lease.community_id \ + AND community.deletion_state IN ('active', 'quiescing') \ + AND community.deleted_at IS NULL \ + AND community.deletion_fence_generation = lease.fence_generation \ + RETURNING lease.lease_until", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .bind(lease_seconds) + .fetch_optional(&mut *tx) + .await?; + let lease_until = lease_until.ok_or_else(|| { + DbError::AccessDenied(format!("stale serving write lease {}", lease.id)) + })?; + tx.commit().await?; + lease.lease_until = lease_until; + Ok(()) + } + + /// Release a serving side-effect lease. A stale release is harmless. + pub async fn release_serving_write_lease(&self, lease: &ServingWriteLease) -> Result { + let deleted = sqlx::query( + "DELETE FROM community_serving_write_leases \ + WHERE id = $1 AND community_id = $2 AND owner = $3 AND generation = $4 \ + AND fence_generation = $5", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .execute(&self.pool) + .await? + .rows_affected(); + Ok(deleted == 1) + } + + /// Check that an external side-effect lease remains current for finalization. + /// + /// A lease admitted before quiescing may renew, complete, and release; new + /// work remains blocked, preserving an accurate drain without abandoning an + /// admitted remote effect. + pub async fn verify_serving_write_lease(&self, lease: &ServingWriteLease) -> Result<()> { + let mut tx = self.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(lease.community_id.as_uuid()) + .execute(&mut *tx) + .await?; + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases lease \ + JOIN communities community ON community.id = lease.community_id \ + WHERE lease.id = $1 AND lease.community_id = $2 AND lease.owner = $3 \ + AND lease.generation = $4 AND lease.fence_generation = $5 \ + AND lease.lease_until >= now() \ + AND community.deleted_at IS NULL \ + AND community.deletion_state IN ('active', 'quiescing') \ + AND community.deletion_fence_generation = lease.fence_generation)", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .fetch_one(&mut *tx) + .await?; + if valid { + tx.commit().await?; + Ok(()) + } else { + Err(DbError::AccessDenied(format!( + "stale serving write lease {}", + lease.id + ))) + } + } + + /// Delete expired serving leases in a bounded batch. + pub async fn reap_expired_serving_write_leases(&self, limit: i64) -> Result { + let affected = sqlx::query( + "WITH expired AS ( \ + SELECT id FROM community_serving_write_leases \ + WHERE lease_until < now() ORDER BY lease_until LIMIT $1 \ + FOR UPDATE SKIP LOCKED \ + ) DELETE FROM community_serving_write_leases lease \ + USING expired WHERE lease.id = expired.id", + ) + .bind(limit.clamp(1, 10_000)) + .execute(&self.pool) + .await? + .rows_affected(); + Ok(affected) + } + + /// Return serving-lease counts and dead-tuple estimate for observability. + pub async fn serving_lease_stats(&self) -> Result { + let row = sqlx::query( + "SELECT count(*) FILTER (WHERE lease_until >= now())::BIGINT AS active, \ + count(*) FILTER (WHERE lease_until < now())::BIGINT AS expired, \ + COALESCE((SELECT n_dead_tup::BIGINT FROM pg_stat_user_tables \ + WHERE relname = 'community_serving_write_leases'), 0) AS dead_tuples \ + FROM community_serving_write_leases", + ) + .fetch_one(&self.pool) + .await?; + Ok(ServingLeaseStats { + active: row.try_get("active")?, + expired: row.try_get("expired")?, + dead_tuples: row.try_get("dead_tuples")?, + }) + } + + /// Whether a community remains active and serving-write eligible. + pub async fn is_serving_active(&self, community: CommunityId) -> Result { + sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 \ + AND archived_at IS NULL AND deleted_at IS NULL AND deletion_state = 'active')", + ) + .bind(community.as_uuid()) + .fetch_one(&self.pool) + .await + .map_err(Into::into) + } + + async fn advance_with_checkpoint( + &self, + token: &LeaseToken, + from: DeletionStage, + to: DeletionStage, + unit_key: &str, + detail: serde_json::Value, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, from, generation).await?; + advance_request_tx(&mut tx, token, from, to, Some(generation)).await?; + checkpoint_completed_tx(&mut tx, token, from, unit_key, detail).await?; + tx.commit().await?; + Ok(()) + } + + async fn live_scoped_tables(&self) -> Result> { + let mut conn = self.pool.acquire().await?; + live_scoped_tables_on(&mut conn).await + } + + async fn live_fenced_tables(&self) -> Result> { + let mut conn = self.pool.acquire().await?; + live_fenced_tables_on(&mut conn).await + } +} + +/// Take the shared schema/destruction advisory lock for the current +/// transaction. +/// +/// Transaction-scoped so every abort path — including executor death — +/// releases it. Migrations hold the exclusive session counterpart for their +/// whole run (see [`crate::migration::run_migrations`]); shared holders do +/// not block each other, so concurrent deletion executors are unaffected. +async fn lock_schema_destruction_shared(conn: &mut PgConnection) -> Result<()> { + sqlx::query("SELECT pg_advisory_xact_lock_shared($1)") + .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + .execute(conn) + .await?; + Ok(()) +} + +/// Connection-bound form of [`DeletionStore::validate_catalog`]. +/// +/// Destructive transactions call this on their own transaction after taking +/// the shared schema/destruction lock, so the validated surface cannot change +/// before the transaction commits. +async fn validate_catalog_on(conn: &mut PgConnection) -> Result<()> { + let expected = EXPECTED_SCOPED_TABLES + .iter() + .copied() + .map(str::to_owned) + .collect::>(); + let live_tables = live_scoped_tables_on(conn).await?; + if live_tables != expected { + let missing = expected + .difference(&live_tables) + .cloned() + .collect::>(); + let unknown = live_tables + .difference(&expected) + .cloned() + .collect::>(); + return Err(DbError::DeletionSafety(format!( + "community deletion catalog drift (missing={}, unknown={})", + missing.join(","), + unknown.join(",") + ))); + } + + let fenced_tables = live_fenced_tables_on(conn).await?; + if fenced_tables != expected { + let missing = expected + .difference(&fenced_tables) + .cloned() + .collect::>(); + let unknown = fenced_tables + .difference(&expected) + .cloned() + .collect::>(); + return Err(DbError::DeletionSafety(format!( + "community deletion write-fence drift (missing={}, unknown={})", + missing.join(","), + unknown.join(",") + ))); + } + Ok(()) +} + +async fn live_scoped_tables_on(conn: &mut PgConnection) -> Result> { + let rows: Vec = sqlx::query_scalar( + r#" + SELECT c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attribute a ON a.attrelid = c.oid + WHERE n.nspname = 'public' + AND c.relkind IN ('r', 'p') + AND NOT c.relispartition + AND a.attname = 'community_id' + AND NOT a.attisdropped + AND NOT community_write_fence_excluded_table(c.relname) + ORDER BY c.relname + "#, + ) + .fetch_all(conn) + .await?; + Ok(rows.into_iter().collect()) +} + +async fn live_fenced_tables_on(conn: &mut PgConnection) -> Result> { + let rows: Vec = sqlx::query_scalar( + r#" + SELECT c.relname + FROM pg_trigger trigger + JOIN pg_class c ON c.oid = trigger.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_proc procedure ON procedure.oid = trigger.tgfoid + WHERE n.nspname = 'public' + AND NOT trigger.tgisinternal + AND NOT c.relispartition + AND procedure.proname = 'enforce_community_write_fence' + AND trigger.tgenabled = 'O' + AND (trigger.tgtype & 1) = 1 + AND (trigger.tgtype & 2) = 2 + AND (trigger.tgtype & 4) = 4 + AND (trigger.tgtype & 8) = 8 + AND (trigger.tgtype & 16) = 16 + ORDER BY c.relname + "#, + ) + .fetch_all(conn) + .await?; + Ok(rows.into_iter().collect()) +} + +/// Fail closed when a community-prefix inventory has an unsafe shape. +pub fn validate_storage_manifest(manifest: &StorageManifest) -> Result<()> { + if manifest.version != 4 { + return Err(DbError::DeletionSafety(format!( + "unsupported storage manifest version {}", + manifest.version + ))); + } + if manifest.prefixes.is_empty() { + return Err(DbError::DeletionSafety( + "storage manifest has no tenant prefixes".to_string(), + )); + } + if manifest + .prefixes + .windows(2) + .any(|pair| pair[0].prefix >= pair[1].prefix) + { + return Err(DbError::DeletionSafety( + "storage manifest prefixes are not strictly sorted".to_string(), + )); + } + for prefix in &manifest.prefixes { + // An empty prefix would enumerate — and delete — the whole bucket. + if prefix.prefix.is_empty() { + return Err(DbError::DeletionSafety( + "storage manifest contains an empty prefix".to_string(), + )); + } + if prefix.keys_digest.len() != 64 + || !prefix + .keys_digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(DbError::DeletionSafety(format!( + "storage manifest digest for {} is not lowercase hex sha-256", + prefix.prefix + ))); + } + } + Ok(()) +} + +/// Verify the persisted chunk stream is exactly the frozen enumeration: +/// contiguous chunk numbers, chunks grouped by manifest prefix order, every +/// key under its chunk's prefix, and per-prefix digest/count equality. +fn validate_manifest_key_chunks( + manifest: &StorageManifest, + chunks: &[(i64, String, sqlx::types::Json>)], +) -> Result<()> { + let close = |summary: &PrefixManifest, digest: KeyStreamDigest| -> Result<()> { + let (hex_digest, count) = digest.finish(); + if hex_digest != summary.keys_digest || count != summary.object_count { + return Err(DbError::DeletionSafety(format!( + "frozen key chunks do not match the destructive manifest for prefix {}", + summary.prefix + ))); + } + Ok(()) + }; + let mut remaining = manifest.prefixes.iter(); + let mut current = remaining.next(); + let mut digest = KeyStreamDigest::new(); + for (index, (chunk_no, chunk_prefix, keys)) in chunks.iter().enumerate() { + if *chunk_no != i64::try_from(index).unwrap_or(i64::MAX) { + return Err(DbError::DeletionSafety( + "frozen key chunk sequence has gaps".to_string(), + )); + } + loop { + match current { + Some(summary) if summary.prefix == *chunk_prefix => break, + Some(summary) => { + close(summary, std::mem::take(&mut digest))?; + current = remaining.next(); + } + None => { + return Err(DbError::DeletionSafety(format!( + "frozen key chunk prefix {chunk_prefix} is not in the destructive manifest" + ))); + } + } + } + if keys.0.is_empty() { + return Err(DbError::DeletionSafety( + "frozen key chunk is empty".to_string(), + )); + } + for key in &keys.0 { + if !key.starts_with(chunk_prefix.as_str()) { + return Err(DbError::DeletionSafety(format!( + "frozen key {key} is outside its chunk prefix {chunk_prefix}" + ))); + } + digest.fold(key)?; + } + } + if let Some(summary) = current { + close(summary, digest)?; + } + for summary in remaining { + close(summary, KeyStreamDigest::new())?; + } + Ok(()) +} + +async fn verify_lease( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, +) -> Result<()> { + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests request \ + JOIN community_deletion_approvals approval ON approval.request_id = request.id \ + AND approval.community_id = request.community_id \ + AND approval.inventory_digest = request.inventory_digest \ + WHERE request.id = $1 AND request.community_id = $5 AND request.stage = $2 \ + AND request.lease_owner = $3 AND request.lease_generation = $4 \ + AND request.lease_until >= now() AND request.blocked_at IS NULL)", + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(&token.owner) + .bind(token.generation) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut **tx) + .await?; + if valid { + Ok(()) + } else { + Err(stale_lease_error(token)) + } +} + +async fn verify_lease_and_fence( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, + fence_generation: i64, +) -> Result<()> { + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests request \ + JOIN communities community ON community.id = request.community_id \ + JOIN community_deletion_approvals approval ON approval.request_id = request.id \ + AND approval.community_id = request.community_id \ + AND approval.inventory_digest = request.inventory_digest \ + WHERE request.id = $1 AND request.community_id = $6 \ + AND request.stage = $2 AND request.lease_owner = $3 \ + AND request.lease_generation = $4 AND request.lease_until >= now() \ + AND request.blocked_at IS NULL AND request.fence_generation = $5 \ + AND community.deletion_state IN ('fenced', 'tombstone') \ + AND community.deletion_fence_generation = $5)", + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(&token.owner) + .bind(token.generation) + .bind(fence_generation) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut **tx) + .await?; + if valid { + Ok(()) + } else { + Err(DbError::AccessDenied(format!( + "stale lease or fencing generation for deletion {}", + token.request_id + ))) + } +} + +async fn set_executor_gucs( + tx: &mut Transaction<'_, Postgres>, + community: CommunityId, + generation: i64, +) -> Result<()> { + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', $2, true)", + ) + .bind(community.to_string()) + .bind(generation.to_string()) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn advance_request_tx( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + from: DeletionStage, + to: DeletionStage, + fence_generation: Option, +) -> Result<()> { + if from.next() != Some(to) { + return Err(DbError::DeletionSafety(format!( + "illegal deletion transition {from} -> {to}" + ))); + } + let affected = sqlx::query( + "UPDATE community_deletion_requests \ + SET stage = $5, fence_generation = COALESCE($6, fence_generation), \ + updated_at = now(), retry_count = 0, retry_stage = NULL, \ + last_error = NULL, last_error_at = NULL \ + WHERE id = $1 AND stage = $4 AND lease_owner = $2 \ + AND lease_generation = $3 AND lease_until >= now() AND blocked_at IS NULL", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(from.to_string()) + .bind(to.to_string()) + .bind(fence_generation) + .execute(&mut **tx) + .await? + .rows_affected(); + if affected == 1 { + Ok(()) + } else { + Err(stale_lease_error(token)) + } +} + +async fn checkpoint_completed_tx( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + detail: serde_json::Value, +) -> Result<()> { + sqlx::query( + r#" + INSERT INTO community_deletion_checkpoints + (request_id, stage, unit_key, status, lease_generation, detail, completed_at) + VALUES ($1, $2, $3, 'completed', $4, $5, now()) + ON CONFLICT (request_id, stage, unit_key) DO UPDATE + SET status = 'completed', lease_generation = EXCLUDED.lease_generation, + attempts = community_deletion_checkpoints.attempts + 1, + detail = EXCLUDED.detail, error = NULL, completed_at = now() + "#, + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(unit_key) + .bind(token.generation) + .bind(detail) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn checkpoint_failed_tx( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + error: &str, +) -> Result<()> { + sqlx::query( + r#" + INSERT INTO community_deletion_checkpoints + (request_id, stage, unit_key, status, lease_generation, error) + VALUES ($1, $2, $3, 'failed', $4, $5) + ON CONFLICT (request_id, stage, unit_key) DO UPDATE + SET status = 'failed', lease_generation = EXCLUDED.lease_generation, + attempts = community_deletion_checkpoints.attempts + 1, + error = EXCLUDED.error, completed_at = NULL + "#, + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(unit_key) + .bind(token.generation) + .bind(error) + .execute(&mut **tx) + .await?; + Ok(()) +} + +fn serialize_community_id( + community: &CommunityId, + serializer: S, +) -> std::result::Result +where + S: serde::Serializer, +{ + serializer.serialize_str(&community.to_string()) +} + +fn row_to_request(row: sqlx::postgres::PgRow) -> Result { + let community_id: Uuid = row.try_get("community_id")?; + let digest: Option> = row.try_get("inventory_digest")?; + Ok(DeletionRequest { + id: row.try_get("id")?, + community_id: CommunityId::from_uuid(community_id), + community_host: row.try_get("community_host")?, + stage: row.try_get::("stage")?.parse()?, + retry_stage: row + .try_get::, _>("retry_stage")? + .map(|stage| stage.parse()) + .transpose()?, + requested_by: row.try_get("requested_by")?, + reason: row.try_get("reason")?, + schema_manifest: row.try_get("schema_manifest")?, + storage_manifest: row.try_get("storage_manifest")?, + destructive_storage_manifest: row.try_get("destructive_storage_manifest")?, + inventory_manifest: row.try_get("inventory_manifest")?, + inventory_digest: digest.map(hex::encode), + fence_generation: row.try_get("fence_generation")?, + lease_owner: row.try_get("lease_owner")?, + lease_generation: row.try_get("lease_generation")?, + lease_until: row.try_get("lease_until")?, + attempts: row.try_get("attempts")?, + retry_count: row.try_get("retry_count")?, + last_error: row.try_get("last_error")?, + next_attempt_at: row.try_get("next_attempt_at")?, + blocked_reason: row.try_get("blocked_reason")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + pre_quiesce_archived_at: row.try_get("pre_quiesce_archived_at")?, + quiescing_started_at: row.try_get("quiescing_started_at")?, + aborted_by: row.try_get("aborted_by")?, + abort_reason: row.try_get("abort_reason")?, + aborted_at: row.try_get("aborted_at")?, + completed_at: row.try_get("completed_at")?, + }) +} + +/// Return whether an error is the deletion store's typed ownership-loss class. +pub fn is_stale_deletion_lease(error: &DbError) -> bool { + matches!(error, DbError::AccessDenied(message) if message.starts_with("stale deletion lease ") || message.starts_with("stale lease or fencing generation for deletion ")) +} + +fn stale_lease_error(token: &LeaseToken) -> DbError { + DbError::AccessDenied(format!( + "stale deletion lease {} owner {:?} generation {}", + token.request_id, token.owner, token.generation + )) +} + +fn require_fence_generation(token: &LeaseToken) -> Result { + token.fence_generation.ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {} has no durable fence generation", + token.request_id + )) + }) +} + +fn bound_text(input: &str, max: usize) -> String { + if input.len() <= max { + return input.to_owned(); + } + let mut end = max; + while !input.is_char_boundary(end) { + end -= 1; + } + input[..end].to_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_prefix(prefix: &str) -> PrefixManifest { + PrefixManifest { + prefix: prefix.to_string(), + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + } + } + + fn storage_manifest() -> StorageManifest { + StorageManifest { + version: 4, + prefixes: vec![ + empty_prefix("_meta/c/"), + empty_prefix("_uploads/c/"), + empty_prefix("repos/c/"), + ], + } + } + + #[test] + fn stage_order_is_exact_and_terminal() { + let mut stage = DeletionStage::Submitted; + let mut seen = vec![stage]; + while let Some(next) = stage.next() { + stage = next; + seen.push(stage); + } + assert_eq!( + seen, + vec![ + DeletionStage::Submitted, + DeletionStage::Inventoried, + DeletionStage::Approved, + DeletionStage::Fenced, + DeletionStage::Drained, + DeletionStage::BindingsRemoved, + DeletionStage::PostgresPurged, + DeletionStage::CachePurged, + DeletionStage::LogicallyVerified, + DeletionStage::RetentionPending, + ] + ); + assert!(!DeletionStage::Submitted.runnable()); + assert!(!DeletionStage::Inventoried.runnable()); + assert!(DeletionStage::Approved.runnable()); + assert!(!DeletionStage::RetentionPending.runnable()); + assert!(!DeletionStage::Aborted.runnable()); + } + + #[test] + fn stale_lease_classifier_does_not_swallow_other_access_denials() { + let stale = stale_lease_error(&LeaseToken { + request_id: Uuid::new_v4(), + owner: "owner".to_string(), + generation: 1, + community_id: CommunityId::from_uuid(Uuid::new_v4()), + fence_generation: None, + }); + assert!(is_stale_deletion_lease(&stale)); + assert!(!is_stale_deletion_lease(&DbError::AccessDenied( + "ordinary authorization failure".to_string() + ))); + } + + #[test] + fn storage_manifest_shape_invariants_fail_closed() { + assert!(validate_storage_manifest(&storage_manifest()).is_ok()); + + let mut unsorted = storage_manifest(); + unsorted.prefixes.swap(0, 1); + assert!(validate_storage_manifest(&unsorted).is_err()); + + let mut whole_bucket = storage_manifest(); + whole_bucket.prefixes[0].prefix = String::new(); + assert!(validate_storage_manifest(&whole_bucket).is_err()); + + let mut malformed_digest = storage_manifest(); + malformed_digest.prefixes[0].keys_digest = "not-hex".to_string(); + assert!(validate_storage_manifest(&malformed_digest).is_err()); + } + + #[test] + fn key_stream_digest_requires_strict_order_and_is_chunking_invariant() { + let keys = ["a/1", "a/2", "a/3"]; + let mut whole = KeyStreamDigest::new(); + for key in keys { + whole.fold(key).expect("ascending fold"); + } + // The digest must not depend on where chunk boundaries fall. + let mut split = KeyStreamDigest::new(); + split.fold(keys[0]).expect("chunk one"); + split.fold(keys[1]).expect("chunk one"); + split.fold(keys[2]).expect("chunk two"); + assert_eq!(whole.finish(), split.finish()); + + let mut out_of_order = KeyStreamDigest::new(); + out_of_order.fold("b").expect("first key"); + assert!(out_of_order.fold("a").is_err()); + let mut duplicate = KeyStreamDigest::new(); + duplicate.fold("a").expect("first key"); + assert!(duplicate.fold("a").is_err()); + } + + #[test] + fn manifest_key_chunks_must_hash_to_the_frozen_summaries() { + let keys = vec!["_meta/c/1".to_string(), "_meta/c/2".to_string()]; + let mut digest = KeyStreamDigest::new(); + for key in &keys { + digest.fold(key).expect("fold"); + } + let (hex_digest, count) = digest.finish(); + let mut manifest = storage_manifest(); + manifest.prefixes[0].object_count = count; + manifest.prefixes[0].keys_digest = hex_digest; + + let chunk = |chunk_no: i64, keys: &[String]| { + ( + chunk_no, + "_meta/c/".to_string(), + sqlx::types::Json(keys.to_vec()), + ) + }; + assert!(validate_manifest_key_chunks( + &manifest, + &[chunk(0, &keys[..1]), chunk(1, &keys[1..])] + ) + .is_ok()); + // Missing, reordered, or extra keys change the digest. + assert!(validate_manifest_key_chunks(&manifest, &[chunk(0, &keys[..1])]).is_err()); + // A gap in the chunk sequence is an interrupted write, not a manifest. + assert!(validate_manifest_key_chunks(&manifest, &[chunk(1, &keys)]).is_err()); + // A key outside its chunk's prefix must never freeze. + let foreign = vec!["_uploads/other/1".to_string()]; + assert!( + validate_manifest_key_chunks(&manifest, &[chunk(0, &keys), chunk(1, &foreign)]) + .is_err() + ); + // No chunks at all only matches an all-empty manifest. + assert!(validate_manifest_key_chunks(&manifest, &[]).is_err()); + assert!(validate_manifest_key_chunks(&storage_manifest(), &[]).is_ok()); + } + + #[test] + fn frozen_inventory_digest_is_stable() { + let inventory = FrozenInventory { + schema: SchemaManifest { + scoped_tables: vec!["events".to_string()], + row_counts: BTreeMap::from([("events".to_string(), 3)]), + fenced_tables: vec!["events".to_string()], + }, + storage: storage_manifest(), + }; + assert_eq!(inventory.digest().unwrap(), inventory.digest().unwrap()); + assert_eq!(inventory.digest().unwrap().len(), 32); + } + + #[test] + fn errors_are_utf8_bounded() { + let input = format!("{}🛸", "x".repeat(4095)); + let bounded = bound_text(&input, 4096); + assert!(bounded.len() <= 4096); + assert!(std::str::from_utf8(bounded.as_bytes()).is_ok()); + } +} + +#[cfg(test)] +mod postgres_tests { + use super::*; + use crate::{CreateCommunityWithOwnerResult, Db, DbConfig}; + + async fn store() -> (Db, DeletionStore) { + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + let db = Db::new(&DbConfig { + database_url, + max_connections: 5, + min_connections: 0, + ..DbConfig::default() + }) + .await + .expect("connect deletion test DB"); + db.migrate().await.expect("migrate deletion test DB"); + let store = db.deletion_store(); + (db, store) + } + + fn empty_prefix_manifest(prefix: String) -> PrefixManifest { + PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + } + } + + fn empty_storage_manifest(community: CommunityId) -> StorageManifest { + StorageManifest { + version: 4, + prefixes: vec![ + empty_prefix_manifest(format!("_meta/{community}/")), + empty_prefix_manifest(format!("_uploads/{community}/")), + empty_prefix_manifest(format!("repos/{community}/")), + ], + } + } + + async fn inventoried_request( + db: &Db, + store: &DeletionStore, + ) -> (DeletionRequest, FrozenInventory) { + let host = format!("deletion-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("create community"); + let submitted = store + .submit(&host, "test-operator", Some("test deletion")) + .await + .expect("submit"); + assert_eq!(submitted.community_id, community.id); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community.id) + .await + .expect("schema inventory"), + storage: empty_storage_manifest(community.id), + }; + let request = store + .freeze_inventory(submitted.id, &inventory) + .await + .expect("freeze inventory"); + (request, inventory) + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn approval_boundary_blocks_claim_until_exact_inventory_is_approved() { + let (db, store) = store().await; + let (request, inventory) = inventoried_request(&db, &store).await; + assert_eq!(request.stage, DeletionStage::Inventoried); + assert!(store + .claim_specific(request.id, "executor-a", DEFAULT_LEASE_DURATION) + .await + .expect("claim before approval") + .is_none()); + + // Row counts are frozen observational evidence. Ordinary serving + // churn after inventory does not invalidate approval: execution fences, + // purges, and verifies the live tenant state independently. + db.add_to_allowlist(request.community_id, &[7_u8; 32], &[8_u8; 32], None) + .await + .expect("post-inventory serving write"); + let current_schema = store + .inventory_schema(request.community_id) + .await + .expect("live schema after row churn"); + assert_eq!( + current_schema.row_counts["pubkey_allowlist"], + inventory.schema.row_counts["pubkey_allowlist"] + 1 + ); + assert_eq!(current_schema.scoped_tables, inventory.schema.scoped_tables); + assert_eq!(current_schema.fenced_tables, inventory.schema.fenced_tables); + + let mismatched_insert = sqlx::query( + "INSERT INTO community_deletion_approvals \ + (request_id, community_id, inventory_digest, approved_by) \ + VALUES ($1, $2, $3, 'tampered')", + ) + .bind(request.id) + .bind(*request.community_id.as_uuid()) + .bind(vec![0_u8; 32]) + .execute(&db.pool) + .await; + assert!( + mismatched_insert.is_err(), + "a mismatched approval must be unrepresentable" + ); + let approved = store + .approve(request.id, "approver-a", Some("reviewed")) + .await + .expect("approve"); + assert_eq!(approved.stage, DeletionStage::Approved); + assert_eq!( + approved.inventory_digest, + Some(hex::encode(inventory.digest().unwrap())) + ); + let mismatched_approval = sqlx::query( + "UPDATE community_deletion_approvals SET inventory_digest = $2 WHERE request_id = $1", + ) + .bind(request.id) + .bind(vec![0_u8; 32]) + .execute(&db.pool) + .await; + assert!( + mismatched_approval.is_err(), + "approval digest must remain database-bound to the frozen request digest" + ); + let mismatched_request = sqlx::query( + "UPDATE community_deletion_requests SET inventory_digest = $2 WHERE id = $1", + ) + .bind(request.id) + .bind(vec![1_u8; 32]) + .execute(&db.pool) + .await; + assert!( + mismatched_request.is_err(), + "the frozen request digest must remain bound to its approval" + ); + assert!(store + .claim_specific(request.id, "executor-a", DEFAULT_LEASE_DURATION) + .await + .expect("claim approved") + .is_some()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn approved_request_cannot_be_retargeted_rewritten_or_claimed_without_approval() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + let other_host = format!("control-{}.example", Uuid::new_v4().simple()); + let control = db + .ensure_configured_community(&other_host) + .await + .expect("create control community"); + + for mutation in [ + sqlx::query("UPDATE community_deletion_requests SET community_id = $2 WHERE id = $1") + .bind(request.id) + .bind(*control.id.as_uuid()) + .execute(&db.pool) + .await, + sqlx::query("UPDATE community_deletion_requests SET community_host = $2 WHERE id = $1") + .bind(request.id) + .bind(&other_host) + .execute(&db.pool) + .await, + sqlx::query( + "UPDATE community_deletion_requests SET inventory_manifest = '{}'::jsonb WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + sqlx::query( + "UPDATE community_deletion_requests SET storage_manifest = '{}'::jsonb WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + ] { + assert!(mutation.is_err(), "frozen deletion target and inventory must be immutable"); + } + + store + .approve(request.id, "approver", None) + .await + .expect("approve request"); + let claim = store + .claim_specific(request.id, "forged-executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim approved request") + .expect("approved request is claimable"); + let approval_delete = + sqlx::query("DELETE FROM community_deletion_approvals WHERE request_id = $1") + .bind(request.id) + .execute(&db.pool) + .await; + assert!( + approval_delete.is_err(), + "approval evidence must be immutable" + ); + for approval_update in [ + "UPDATE community_deletion_approvals SET approved_by = 'forged' WHERE request_id = $1", + "UPDATE community_deletion_approvals SET approved_at = now() + interval '1 hour' WHERE request_id = $1", + "UPDATE community_deletion_approvals SET note = 'rewritten' WHERE request_id = $1", + ] { + assert!( + sqlx::query(approval_update) + .bind(request.id) + .execute(&db.pool) + .await + .is_err(), + "approval evidence updates must be rejected" + ); + } + store + .verify_execution_token(&claim.lease, DeletionStage::Approved) + .await + .expect("matching approval keeps lease valid"); + sqlx::query( + "UPDATE community_deletion_requests \ + SET blocked_at = now(), blocked_reason = 'operator hold' WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await + .expect("block claimed request"); + assert!( + store + .heartbeat(&claim.lease, "worker", DEFAULT_LEASE_DURATION, false,) + .await + .is_err(), + "blocked requests must not renew destructive leases" + ); + + let (forged, _) = inventoried_request(&db, &store).await; + sqlx::query("UPDATE community_deletion_requests SET stage = 'approved' WHERE id = $1") + .bind(forged.id) + .execute(&db.pool) + .await + .expect("forge runnable stage without approval"); + assert!(store + .claim_specific(forged.id, "forged-executor-2", DEFAULT_LEASE_DURATION) + .await + .expect("claim forged request") + .is_none()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn retry_exhaustion_blocks_only_the_consecutive_stage_and_progress_resets_it() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + + for attempt in 1..=8 { + let claim = store + .claim_specific( + request.id, + &format!("executor-{attempt}"), + DEFAULT_LEASE_DURATION, + ) + .await + .expect("claim retryable request") + .expect("request remains claimable before exhaustion"); + store + .record_retry( + &claim.lease, + DeletionStage::Approved, + "dependency", + "dependency unavailable", + Duration::ZERO, + ) + .await + .expect("record retry"); + + let observed = store.get(request.id).await.expect("load retry state"); + assert_eq!(observed.retry_count, attempt); + assert_eq!(observed.retry_stage, Some(DeletionStage::Approved)); + assert_eq!(observed.blocked_reason.is_some(), attempt == 8); + } + assert!(store + .claim_specific(request.id, "blocked-executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim blocked request") + .is_none()); + + let recovered = store + .unblock(request.id, "operator", "dependency repaired") + .await + .expect("unblock exhausted request"); + assert_eq!(recovered.retry_count, 0); + assert_eq!(recovered.retry_stage, None); + assert!(recovered.blocked_reason.is_none()); + + let claim = store + .claim_specific(request.id, "successor", DEFAULT_LEASE_DURATION) + .await + .expect("claim recovered request") + .expect("recovered request is claimable"); + store + .begin_quiescing(&claim.lease) + .await + .expect("begin quiescing after recovery"); + store.fence(&claim.lease).await.expect("advance stage"); + let advanced = store.get(request.id).await.expect("load advanced request"); + assert_eq!(advanced.stage, DeletionStage::Fenced); + assert_eq!(advanced.retry_count, 0); + assert_eq!(advanced.retry_stage, None); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn abort_serializes_before_quiescing_without_deadlock() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + let mut gate = db.pool.begin().await.expect("begin lock gate"); + sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") + .bind(request.community_id.as_uuid()) + .execute(&mut *gate) + .await + .expect("hold community lock"); + + let abort_store = store.clone(); + let aborting = tokio::spawn(async move { + abort_store + .abort(request.id, "operator", "race recovery") + .await + }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !aborting.is_finished(), + "abort must wait for the community lock" + ); + let forward_store = store.clone(); + let lease = claim.lease.clone(); + let forwarding = tokio::spawn(async move { forward_store.begin_quiescing(&lease).await }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !forwarding.is_finished(), + "forward transition must queue on the same lock" + ); + gate.commit().await.expect("release lock gate"); + + let aborted = tokio::time::timeout(Duration::from_secs(5), aborting) + .await + .expect("abort must not deadlock") + .expect("abort task") + .expect("abort wins lock queue"); + assert_eq!(aborted.stage, DeletionStage::Aborted); + let forward_error = tokio::time::timeout(Duration::from_secs(5), forwarding) + .await + .expect("forward transition must not deadlock") + .expect("forward task") + .expect_err("post-lock lease verification rejects aborted request"); + assert!( + !matches!( + &forward_error, + DbError::Sqlx(sqlx::Error::Database(error)) if error.code().as_deref() == Some("40P01") + ), + "serialization must not report a PostgreSQL deadlock" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn abort_serializes_before_fence_without_deadlock() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + let mut gate = db.pool.begin().await.expect("begin lock gate"); + sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") + .bind(request.community_id.as_uuid()) + .execute(&mut *gate) + .await + .expect("hold community lock"); + + let abort_store = store.clone(); + let aborting = tokio::spawn(async move { + abort_store + .abort(request.id, "operator", "race recovery") + .await + }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !aborting.is_finished(), + "abort must wait for the community lock" + ); + let forward_store = store.clone(); + let lease = claim.lease.clone(); + let forwarding = tokio::spawn(async move { forward_store.fence(&lease).await }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !forwarding.is_finished(), + "fence must queue on the same lock" + ); + gate.commit().await.expect("release lock gate"); + + let aborted = tokio::time::timeout(Duration::from_secs(5), aborting) + .await + .expect("abort must not deadlock") + .expect("abort task") + .expect("abort wins lock queue"); + assert_eq!(aborted.stage, DeletionStage::Aborted); + let forward_error = tokio::time::timeout(Duration::from_secs(5), forwarding) + .await + .expect("fence must not deadlock") + .expect("fence task") + .expect_err("post-lock lease verification rejects aborted request"); + assert!( + !matches!( + &forward_error, + DbError::Sqlx(sqlx::Error::Database(error)) if error.code().as_deref() == Some("40P01") + ), + "serialization must not report a PostgreSQL deadlock" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn abort_preserves_audit_and_allows_fresh_request() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let aborted = store + .abort(request.id, "operator", "cancel deletion") + .await + .expect("abort"); + assert_eq!(aborted.stage, DeletionStage::Aborted); + + let replacement = store + .submit( + &request.community_host, + "second-operator", + Some("fresh review"), + ) + .await + .expect("submit replacement request"); + assert_ne!(replacement.id, request.id); + assert_eq!(replacement.stage, DeletionStage::Submitted); + assert!(replacement.inventory_digest.is_none()); + assert_eq!( + store.get(request.id).await.expect("preserved audit").stage, + DeletionStage::Aborted + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn preclaim_setup_failure_is_durable_without_a_lease() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let blocked = store + .block_preclaim_setup( + request.id, + "pre_claim:service_setup", + "BUZZ_S3_ENDPOINT is required", + ) + .await + .expect("record setup failure"); + assert_eq!(blocked.stage, DeletionStage::Approved); + assert_eq!( + blocked.blocked_reason.as_deref(), + Some("BUZZ_S3_ENDPOINT is required") + ); + assert_eq!( + blocked.last_error.as_deref(), + Some("BUZZ_S3_ENDPOINT is required") + ); + assert!(blocked.lease_owner.is_none()); + let inspection = store.inspect(request.id).await.expect("inspect failure"); + let checkpoint = inspection + .checkpoints + .iter() + .find(|checkpoint| checkpoint.unit_key == "pre_claim:service_setup") + .expect("setup failure checkpoint"); + assert_eq!(checkpoint.status, "failed"); + assert_eq!( + checkpoint.error.as_deref(), + Some("BUZZ_S3_ENDPOINT is required") + ); + assert_eq!(checkpoint.attempts, 1); + assert!(checkpoint.completed_at.is_none()); + + store + .unblock(request.id, "operator", "dependency repaired") + .await + .expect("unblock after first setup failure"); + let blocked_again = store + .block_preclaim_setup( + request.id, + "pre_claim:service_setup", + "BUZZ_REDIS_URL is required", + ) + .await + .expect("record repeated setup failure"); + assert_eq!( + blocked_again.blocked_reason.as_deref(), + Some("BUZZ_REDIS_URL is required") + ); + assert_eq!( + blocked_again.last_error.as_deref(), + Some("BUZZ_REDIS_URL is required") + ); + let repeated = store + .inspect(request.id) + .await + .expect("inspect repeated failure"); + let checkpoint = repeated + .checkpoints + .iter() + .find(|checkpoint| checkpoint.unit_key == "pre_claim:service_setup") + .expect("repeated setup failure checkpoint"); + assert_eq!(checkpoint.status, "failed"); + assert_eq!(checkpoint.attempts, 2); + assert_eq!( + checkpoint.error.as_deref(), + Some("BUZZ_REDIS_URL is required") + ); + assert_eq!( + checkpoint + .detail + .get("error") + .and_then(|value| value.as_str()), + Some("BUZZ_REDIS_URL is required") + ); + assert!(checkpoint.completed_at.is_none()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn operator_unblock_preserves_approval_and_records_recovery() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("approved request is claimable"); + store + .block( + &claim.lease, + DeletionStage::Approved, + "dependency", + "operator repair required", + ) + .await + .expect("block request"); + + assert!(store.unblock(request.id, "", "repair").await.is_err()); + let recovered = store + .unblock(request.id, "operator", "bucket policy repaired") + .await + .expect("unblock after remediation"); + assert_eq!(recovered.stage, DeletionStage::Approved); + assert!(recovered.blocked_reason.is_none()); + assert!(recovered.last_error.is_none()); + assert_eq!(recovered.inventory_digest, request.inventory_digest); + assert_eq!(recovered.fence_generation, request.fence_generation); + assert!(store + .unblock(request.id, "operator", "again") + .await + .is_err()); + assert!(store + .claim_specific(request.id, "successor", DEFAULT_LEASE_DURATION) + .await + .expect("claim recovered request") + .is_some()); + + let inspection = store.inspect(request.id).await.expect("inspect recovery"); + assert!(inspection.checkpoints.iter().any(|checkpoint| { + checkpoint.unit_key.starts_with("operator_unblock:") + && checkpoint.detail["unblocked_by"] == "operator" + && checkpoint.detail["reason"] == "bucket policy repaired" + && checkpoint.detail["previous_block"] == "operator repair required" + })); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn stale_claim_and_fence_generation_fail_closed() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + let mut stale = claim.lease.clone(); + stale.generation -= 1; + assert!( + store.fence(&stale).await.is_err(), + "stale lease must reject" + ); + let mut wrong_community = claim.lease.clone(); + wrong_community.community_id = db + .ensure_configured_community(&format!( + "wrong-lease-community-{}.example", + Uuid::new_v4().simple() + )) + .await + .expect("create unrelated community") + .id; + assert!( + store.begin_quiescing(&wrong_community).await.is_err(), + "a lease token must remain bound to its durable request community" + ); + + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + let generation = store.fence(&claim.lease).await.expect("fence"); + let mut wrong_fence = claim.lease.clone(); + wrong_fence.fence_generation = Some(generation + 1); + assert!( + store.mark_drained(&wrong_fence).await.is_err(), + "wrong fence generation must reject" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn fence_waits_for_open_write_and_rejects_it_after_transition() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + + let mut open_write = db + .begin_transaction() + .await + .expect("open write transaction"); + sqlx::query("INSERT INTO pubkey_allowlist (community_id, pubkey) VALUES ($1, $2)") + .bind(request.community_id.as_uuid()) + .bind(vec![7_u8; 32]) + .execute(&mut *open_write) + .await + .expect("write acquires shared deletion lock"); + + let store_for_fence = store.clone(); + let lease = claim.lease.clone(); + let fencing = tokio::spawn(async move { + store_for_fence.begin_quiescing(&lease).await?; + store_for_fence.fence(&lease).await + }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !fencing.is_finished(), + "exclusive fence must wait for open writer" + ); + open_write + .commit() + .await + .expect("pre-fence writer commits first"); + fencing.await.expect("fence task").expect("fence completes"); + + assert!( + db.add_to_allowlist(request.community_id, &[8_u8; 32], &[9_u8; 32], None) + .await + .is_err(), + "post-fence serving write must fail" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn write_assertion_rejects_pinned_snapshot_isolation_before_authorization() { + let (db, _) = store().await; + let community = db + .ensure_configured_community(&format!( + "isolation-guard-{}.example", + Uuid::new_v4().simple() + )) + .await + .expect("create community") + .id; + + for isolation in ["REPEATABLE READ", "SERIALIZABLE"] { + let mut tx = db.pool.begin().await.expect("begin isolation probe"); + sqlx::query(sqlx::AssertSqlSafe(format!( + "SET TRANSACTION ISOLATION LEVEL {isolation}" + ))) + .execute(&mut *tx) + .await + .expect("set transaction isolation"); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '0', true)", + ) + .bind(community.to_string()) + .execute(&mut *tx) + .await + .expect("forge executor authorization"); + let error = sqlx::query("SELECT assert_community_write_allowed($1)") + .bind(community.as_uuid()) + .execute(&mut *tx) + .await + .expect_err("pinned snapshot isolation must fail before authorization"); + assert_eq!( + error + .as_database_error() + .and_then(|error| error.code()) + .as_deref(), + Some("25000") + ); + } + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn quiescing_rejects_new_leases_but_renews_admitted_lease_until_release() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + let mut serving = store + .acquire_serving_write_lease( + request.community_id, + "test_external", + "test-owner", + DEFAULT_LEASE_DURATION, + ) + .await + .expect("serving lease"); + + store + .begin_quiescing(&claim.lease) + .await + .expect("persist quiescing"); + assert!(matches!( + store + .acquire_serving_write_lease( + request.community_id, + "late_external", + "late-owner", + DEFAULT_LEASE_DURATION, + ) + .await, + Err(DbError::AccessDenied(_)) + )); + assert!(store.verify_serving_write_lease(&serving).await.is_ok()); + let lease_until_before_renewal = serving.lease_until; + tokio::time::sleep(Duration::from_millis(10)).await; + store + .renew_serving_write_lease(&mut serving, DEFAULT_LEASE_DURATION) + .await + .expect("admitted lease renews while quiescing"); + assert!( + serving.lease_until > lease_until_before_renewal, + "renewal must extend the admitted lease" + ); + assert!(matches!( + store.fence(&claim.lease).await, + Err(DbError::ServingWritesNotDrained { + active_count: 1, + .. + }) + )); + assert!(store + .release_serving_write_lease(&serving) + .await + .expect("release")); + assert_eq!(store.fence(&claim.lease).await.expect("fence"), 1); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn sustained_admission_cannot_starve_fence_after_quiescing() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + + for attempt in 0..100 { + assert!(matches!( + store + .acquire_serving_write_lease( + request.community_id, + "sustained_admission", + &format!("owner-{attempt}"), + DEFAULT_LEASE_DURATION, + ) + .await, + Err(DbError::AccessDenied(_)) + )); + } + assert_eq!(store.fence(&claim.lease).await.expect("fence"), 1); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn serving_lease_reaper_is_bounded_and_reports_stats() { + let (db, store) = store().await; + let host = format!("lease-reaper-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("community") + .id; + for owner in ["expired-a", "expired-b", "expired-c"] { + let lease = store + .acquire_serving_write_lease( + community, + "reaper_test", + owner, + Duration::from_secs(1), + ) + .await + .expect("lease"); + sqlx::query("UPDATE community_serving_write_leases SET lease_until = now() - interval '1 second' WHERE id = $1") + .bind(lease.id) + .execute(&db.pool) + .await + .expect("expire lease"); + } + let before = store.serving_lease_stats().await.expect("stats before"); + assert!(before.expired >= 3); + assert_eq!(store.reap_expired_serving_write_leases(2).await.unwrap(), 2); + let after = store.serving_lease_stats().await.expect("stats after"); + assert_eq!(after.expired, before.expired - 2); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn serving_lease_reaper_remains_global_across_tombstoned_tenant() { + let (db, store) = store().await; + let active_a = db + .ensure_configured_community(&format!("lease-a-{}.example", Uuid::new_v4().simple())) + .await + .expect("active A") + .id; + let target = db + .ensure_configured_community(&format!("lease-t-{}.example", Uuid::new_v4().simple())) + .await + .expect("target T") + .id; + let active_x = db + .ensure_configured_community(&format!("lease-x-{}.example", Uuid::new_v4().simple())) + .await + .expect("active X") + .id; + store + .reap_expired_serving_write_leases(10_000) + .await + .expect("clear unrelated expired leases"); + for (community, owner) in [(active_a, "a"), (target, "t"), (active_x, "x")] { + let lease = store + .acquire_serving_write_lease( + community, + "global_reaper_test", + owner, + DEFAULT_LEASE_DURATION, + ) + .await + .expect("acquire lease"); + sqlx::query( + "UPDATE community_serving_write_leases \ + SET lease_until = now() - interval '1 second' WHERE id = $1", + ) + .bind(lease.id) + .execute(&db.pool) + .await + .expect("expire lease"); + } + let mut lifecycle = db.pool.begin().await.expect("begin target lifecycle"); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '1', true)", + ) + .bind(target.to_string()) + .execute(&mut *lifecycle) + .await + .expect("authorize tombstone fixture"); + sqlx::query( + "UPDATE communities SET deletion_state = 'tombstone', \ + deletion_fence_generation = 1, deleted_at = now() WHERE id = $1", + ) + .bind(target.as_uuid()) + .execute(&mut *lifecycle) + .await + .expect("tombstone target"); + lifecycle.commit().await.expect("commit tombstone fixture"); + + assert_eq!( + store + .reap_expired_serving_write_leases(10) + .await + .expect("global lease reap"), + 3 + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn checkpointed_resume_is_idempotent_and_tombstone_blocks_name_reuse() { + let (db, store) = store().await; + let (request, inventory) = inventoried_request(&db, &store).await; + let host = request.community_host.clone(); + let read_state_d_tag = format!("read-state:{}", "a".repeat(32)); + sqlx::query( + "INSERT INTO events \ + (community_id, id, pubkey, created_at, kind, tags, content, sig, d_tag) \ + VALUES ($1, $2, $3, now(), 30078, $4, '', $5, $6)", + ) + .bind(request.community_id.as_uuid()) + .bind(vec![1_u8; 32]) + .bind(vec![2_u8; 32]) + .bind(serde_json::json!([ + ["d", &read_state_d_tag], + ["t", "read-state"] + ])) + .bind(vec![3_u8; 64]) + .bind(&read_state_d_tag) + .execute(&db.pool) + .await + .expect("insert guarded NIP-RS row"); + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + let generation = store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await + .expect("freeze destructive storage"); + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await + .expect("identical destructive manifest retry"); + let mut drifted_storage = inventory.storage.clone(); + let mut drifted_digest = KeyStreamDigest::new(); + drifted_digest + .fold("media/drifted-after-fence") + .expect("fold drifted key"); + let (drifted_hex, drifted_count) = drifted_digest.finish(); + drifted_storage.prefixes[0].object_count = drifted_count; + drifted_storage.prefixes[0].keys_digest = drifted_hex; + assert!(matches!( + store + .freeze_destructive_storage_manifest(&token, &drifted_storage) + .await, + Err(DbError::DeletionSafety(_)) + )); + for mutation in [ + sqlx::query( + "UPDATE community_deletion_requests \ + SET destructive_storage_manifest = '{}'::jsonb WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + sqlx::query( + "UPDATE community_deletion_requests \ + SET destructive_storage_frozen_at = destructive_storage_frozen_at + interval '1 second' \ + WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + ] { + assert!( + mutation.is_err(), + "frozen destructive storage evidence must be immutable" + ); + } + store.mark_drained(&token).await.expect("drain"); + store + .mark_bindings_removed(&token, serde_json::json!({"keys": 0})) + .await + .expect("bindings"); + let first = store.purge_postgres(&token).await.expect("purge postgres"); + assert_eq!(first.len(), EXPECTED_SCOPED_TABLES.len()); + assert!( + store.purge_postgres(&token).await.is_err(), + "completed stage cannot be replayed under stale checkpoint state" + ); + store + .mark_cache_purged(&token, serde_json::json!({"keys": 0})) + .await + .expect("cache"); + store + .verify_postgres_logically_deleted(&token) + .await + .expect("logical postgres verify"); + store + .mark_logically_verified(&token, serde_json::json!({"all": true})) + .await + .expect("mark verified"); + store + .mark_retention_pending(&token, serde_json::json!({"shared_cas": "retained"})) + .await + .expect("terminal"); + + let terminal = store.get(request.id).await.expect("terminal request"); + assert_eq!(terminal.stage, DeletionStage::RetentionPending); + let recreated = db + .create_community_with_owner( + &host, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ) + .await + .expect("recreate attempt"); + assert_eq!(recreated, CreateCommunityWithOwnerResult::HostExists); + assert!(db + .lookup_community_by_host_for_management(&host) + .await + .expect("tombstone lookup") + .is_some()); + assert!(db + .lookup_community_by_host(&host) + .await + .expect("serving lookup") + .is_none()); + let direct_delete = sqlx::query("DELETE FROM communities WHERE id = $1") + .bind(request.community_id.as_uuid()) + .execute(&db.pool) + .await + .expect_err("tombstone row must be permanent"); + assert!(direct_delete + .to_string() + .contains("tombstones are permanent")); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn taxonomy_sweep_uses_database_completion_order() { + let (_, store) = store().await; + store + .record_taxonomy_sweep(Utc::now() + chrono::Duration::minutes(1), 1, 0, &[], 100) + .await + .expect("record skewed clean sweep"); + let dirty = store + .record_taxonomy_sweep(Utc::now(), 1, 1, &["unknown".to_string()], 100) + .await + .expect("record later dirty sweep"); + + assert_eq!( + store + .latest_taxonomy_sweep() + .await + .expect("latest sweep") + .unwrap() + .id, + dirty.id + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn manifest_key_chunks_bind_freeze_execution_and_cleanup() { + let (db, store) = store().await; + let (request, inventory) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + let generation = store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; + + let meta_prefix = format!("_meta/{}/", request.community_id); + let keys = vec![ + format!("{meta_prefix}{}.json", "a".repeat(64)), + format!("{meta_prefix}{}.json", "b".repeat(64)), + ]; + store + .append_manifest_key_chunk(&token, 0, &meta_prefix, &keys[..1]) + .await + .expect("append chunk 0"); + store + .append_manifest_key_chunk(&token, 1, &meta_prefix, &keys[1..]) + .await + .expect("append chunk 1"); + + // A manifest whose digests do not cover the chunk stream must not freeze. + assert!(matches!( + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await, + Err(DbError::DeletionSafety(_)) + )); + let mut digest = KeyStreamDigest::new(); + for key in &keys { + digest.fold(key).expect("fold key"); + } + let (hex_digest, count) = digest.finish(); + let mut storage = inventory.storage.clone(); + storage.prefixes[0].object_count = count; + storage.prefixes[0].total_bytes = 2; + storage.prefixes[0].keys_digest = hex_digest; + store + .freeze_destructive_storage_manifest(&token, &storage) + .await + .expect("freeze manifest matching chunks"); + + // Frozen chunks are immutable working data until terminal cleanup. + assert!(sqlx::query( + "UPDATE community_deletion_manifest_keys SET keys = '[]'::jsonb \ + WHERE request_id = $1 AND chunk_no = 0", + ) + .bind(request.id) + .execute(&db.pool) + .await + .is_err()); + assert!( + sqlx::query("DELETE FROM community_deletion_manifest_keys WHERE request_id = $1") + .bind(request.id) + .execute(&db.pool) + .await + .is_err() + ); + assert!(store.clear_manifest_key_chunks(&token).await.is_err()); + assert!( + sqlx::query( + "INSERT INTO community_deletion_manifest_keys \ + (request_id, chunk_no, prefix, keys) VALUES ($1, 2, $2, $3)", + ) + .bind(request.id) + .bind(&meta_prefix) + .bind(sqlx::types::Json(&keys[..1])) + .execute(&db.pool) + .await + .is_err(), + "the database must reject chunks appended after freeze" + ); + + store.mark_drained(&token).await.expect("drained"); + let first = store + .next_pending_manifest_chunk(&token) + .await + .expect("pending chunk") + .expect("chunk 0 pending"); + assert_eq!(first.chunk_no, 0); + assert_eq!(first.keys, keys[..1]); + store + .mark_manifest_chunk_deleted(&token, 0, serde_json::json!({"deleted": 1})) + .await + .expect("stamp chunk 0"); + assert!( + matches!( + store + .mark_manifest_chunk_deleted(&token, 0, serde_json::json!({})) + .await, + Err(DbError::DeletionSafety(_)) + ), + "a chunk stamp is one-way" + ); + let second = store + .next_pending_manifest_chunk(&token) + .await + .expect("pending chunk") + .expect("chunk 1 pending after resume"); + assert_eq!(second.chunk_no, 1); + store + .mark_manifest_chunk_deleted(&token, 1, serde_json::json!({"deleted": 1})) + .await + .expect("stamp chunk 1"); + assert!(store + .next_pending_manifest_chunk(&token) + .await + .expect("pending chunk") + .is_none()); + assert_eq!( + store + .manifest_chunk_progress(request.id) + .await + .expect("progress"), + (2, 2) + ); + + store + .mark_bindings_removed(&token, serde_json::json!({"deleted_keys": 2})) + .await + .expect("bindings removed"); + store.purge_postgres(&token).await.expect("purge postgres"); + store + .mark_cache_purged(&token, serde_json::json!({"keys": 0})) + .await + .expect("cache purged"); + store + .verify_postgres_logically_deleted(&token) + .await + .expect("verify postgres"); + store + .mark_logically_verified(&token, serde_json::json!({"all": true})) + .await + .expect("logically verified"); + assert_eq!( + store + .manifest_chunk_progress(request.id) + .await + .expect("progress after terminal cleanup"), + (0, 0) + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn destructive_stages_serialize_with_migrations_and_fail_closed_on_new_scoped_tables() { + // The probe table below mutates the live catalog, which every other + // test in the shared database validates against. Run the whole + // scenario in a dedicated database so concurrent purge/verify tests + // never observe the drifted surface; advisory locks are also + // per-database, so the parked migration lock cannot stall them. + let base_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + let admin = PgPool::connect(&base_url) + .await + .expect("connect admin database"); + let probe_db = format!("buzz_lock_probe_{}", Uuid::new_v4().simple()); + sqlx::query(AssertSqlSafe(format!("CREATE DATABASE {probe_db}"))) + .execute(&admin) + .await + .expect("create probe database"); + let (base_prefix, _) = base_url.rsplit_once('/').expect("database url has a path"); + let db = Db::new(&DbConfig { + database_url: format!("{base_prefix}/{probe_db}"), + max_connections: 5, + min_connections: 0, + ..DbConfig::default() + }) + .await + .expect("connect probe database"); + db.migrate().await.expect("migrate probe database"); + let store = db.deletion_store(); + let (request, inventory) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + let generation = store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await + .expect("freeze destructive storage"); + store.mark_drained(&token).await.expect("drain"); + store + .mark_bindings_removed(&token, serde_json::json!({"keys": 0})) + .await + .expect("bindings"); + + // Park a migration mid-run through the production lock path: the op + // runs on the same connection that owns the exclusive session lock, + // exactly as `run_migrations` executes migration SQL. + let probe_table = format!("deletion_probe_{}", Uuid::new_v4().simple()); + let create_probe = + format!("CREATE TABLE {probe_table} (community_id UUID NOT NULL, payload TEXT)"); + let attach_probe = + format!("SELECT attach_community_write_fence('{probe_table}'::regclass)"); + let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>(); + let migration_pool = db.pool.clone(); + let (create_probe_sql, attach_probe_sql) = (create_probe.clone(), attach_probe.clone()); + let migration_run = tokio::spawn(async move { + crate::migration::with_exclusive_schema_destruction_lock( + &migration_pool, + move |mut conn| async move { + let _ = started_tx.send(()); + let _ = release_rx.await; + // Explicit DDL transaction: the new scoped table commits + // before the production path releases the exclusive lock. + let outcome: Result<()> = async { + let mut ddl = sqlx::Connection::begin(&mut conn).await?; + sqlx::query(AssertSqlSafe(create_probe_sql)) + .execute(&mut *ddl) + .await?; + sqlx::query(AssertSqlSafe(attach_probe_sql)) + .execute(&mut *ddl) + .await?; + ddl.commit().await?; + Ok(()) + } + .await; + (conn, outcome) + }, + ) + .await + }); + started_rx.await.expect("parked migration holds the lock"); + let blocked = + tokio::time::timeout(Duration::from_millis(750), store.purge_postgres(&token)).await; + assert!( + blocked.is_err(), + "purge must wait for the in-flight migration instead of validating a stale surface" + ); + + // The migration commits its new fenced scoped table, then finishes. + release_tx.send(()).expect("unpark migration"); + migration_run + .await + .expect("join migration run") + .expect("locked migration op"); + + // Purge revalidates inside its own transaction and fails closed on + // the surface this executor does not know. + let denied = store.purge_postgres(&token).await; + let denied_on_probe = matches!( + &denied, + Err(DbError::DeletionSafety(message)) if message.contains(&probe_table) + ); + sqlx::query(AssertSqlSafe(format!("DROP TABLE {probe_table}"))) + .execute(&db.pool) + .await + .expect("drop probe table"); + assert!( + denied_on_probe, + "purge must fail closed on a migration-committed scoped table: {denied:?}" + ); + let after_denied = store.get(request.id).await.expect("request after denial"); + assert_eq!(after_denied.stage, DeletionStage::BindingsRemoved); + + store + .purge_postgres(&token) + .await + .expect("purge after catalog restored"); + store + .mark_cache_purged(&token, serde_json::json!({"keys": 0})) + .await + .expect("cache"); + + // A scoped table committed after the purge must fail the absence + // proof closed rather than silently escaping verification. + sqlx::query(AssertSqlSafe(create_probe)) + .execute(&db.pool) + .await + .expect("recreate probe scoped table"); + sqlx::query(AssertSqlSafe(attach_probe)) + .execute(&db.pool) + .await + .expect("attach probe fence again"); + let verify_denied = store.verify_postgres_logically_deleted(&token).await; + let verify_denied_on_probe = matches!( + &verify_denied, + Err(DbError::DeletionSafety(message)) if message.contains(&probe_table) + ); + sqlx::query(AssertSqlSafe(format!("DROP TABLE {probe_table}"))) + .execute(&db.pool) + .await + .expect("drop probe table again"); + assert!( + verify_denied_on_probe, + "verification must fail closed on a post-purge scoped table: {verify_denied:?}" + ); + + store + .verify_postgres_logically_deleted(&token) + .await + .expect("verify after catalog restored"); + store + .mark_logically_verified(&token, serde_json::json!({"all": true})) + .await + .expect("mark verified"); + store + .mark_retention_pending(&token, serde_json::json!({"probe": "clean"})) + .await + .expect("terminal"); + + db.pool.close().await; + sqlx::query(AssertSqlSafe(format!( + "DROP DATABASE {probe_db} WITH (FORCE)" + ))) + .execute(&admin) + .await + .expect("drop probe database"); + } + + /// A database bootstrapped from `schema/schema.sql` (the pgschema + /// desired-state path — no migrations) must carry the complete 0028 + /// deletion surface and run a deletion through every stage. + /// + /// Before the parity restoration this wedged post-fence: + /// `freeze_destructive_storage_manifest` hit the missing + /// `community_deletion_manifest_keys` relation only after the write fence + /// was already up, leaving the request with no forward path — and even a + /// hand-created table would have lacked the immutability guard trigger. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn desired_state_schema_bootstrap_progresses_beyond_fencing() { + let base_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + let admin = PgPool::connect(&base_url) + .await + .expect("connect admin database"); + let probe_db = format!("buzz_desired_state_{}", Uuid::new_v4().simple()); + sqlx::query(AssertSqlSafe(format!("CREATE DATABASE {probe_db}"))) + .execute(&admin) + .await + .expect("create probe database"); + let (base_prefix, _) = base_url.rsplit_once('/').expect("database url has a path"); + let probe_url = format!("{base_prefix}/{probe_db}"); + + let schema_sql = std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../schema/schema.sql"), + ) + .expect("read schema/schema.sql"); + let bootstrap = PgPool::connect(&probe_url) + .await + .expect("connect probe database"); + sqlx::raw_sql(AssertSqlSafe(schema_sql)) + .execute(&bootstrap) + .await + .expect("apply desired-state schema"); + bootstrap.close().await; + + let db = Db::new(&DbConfig { + database_url: probe_url, + max_connections: 5, + min_connections: 0, + ..DbConfig::default() + }) + .await + .expect("connect desired-state database"); + let store = db.deletion_store(); + let (request, inventory) = inventoried_request(&db, &store).await; + + // The immutability guard must exist and enforce: chunk rows are + // rejected outside an unfrozen fenced request (this request is still + // `inventoried`). + let premature_chunk = sqlx::query( + "INSERT INTO community_deletion_manifest_keys (request_id, chunk_no, prefix, keys) \ + VALUES ($1, 0, '_meta/premature/', '[]'::jsonb)", + ) + .bind(request.id) + .execute(&db.pool) + .await; + let guard_enforced = matches!( + &premature_chunk, + Err(sqlx::Error::Database(db_err)) if db_err.code().as_deref() == Some("23000") + ); + assert!( + guard_enforced, + "manifest-keys immutability guard must reject pre-fence chunks \ + with integrity_constraint_violation: {premature_chunk:?}" + ); + + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + let generation = store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; + // The previously wedging stage: first touch of the manifest-keys + // relation happens here, after the fence is already up. + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await + .expect("freeze destructive storage on desired-state bootstrap"); + store.mark_drained(&token).await.expect("drain"); + store + .mark_bindings_removed(&token, serde_json::json!({"keys": 0})) + .await + .expect("bindings"); + store.purge_postgres(&token).await.expect("purge postgres"); + store + .mark_cache_purged(&token, serde_json::json!({"keys": 0})) + .await + .expect("cache"); + store + .verify_postgres_logically_deleted(&token) + .await + .expect("verify postgres"); + store + .mark_logically_verified(&token, serde_json::json!({"all": true})) + .await + .expect("logically verified"); + store + .mark_retention_pending(&token, serde_json::json!({"bootstrap": "desired-state"})) + .await + .expect("terminal"); + let terminal = store.get(request.id).await.expect("terminal request"); + assert_eq!(terminal.stage, DeletionStage::RetentionPending); + + db.pool.close().await; + sqlx::query(AssertSqlSafe(format!( + "DROP DATABASE {probe_db} WITH (FORCE)" + ))) + .execute(&admin) + .await + .expect("drop probe database"); + } +} diff --git a/crates/buzz-db/src/error.rs b/crates/buzz-db/src/error.rs index f8b8a2eb56b..593eea1cca6 100644 --- a/crates/buzz-db/src/error.rs +++ b/crates/buzz-db/src/error.rs @@ -45,6 +45,25 @@ pub enum DbError { #[error("invalid data: {0}")] InvalidData(String), + /// A serving write admitted before the lifecycle transition is still live. + /// This is an ordinary retryable drain condition, not a safety violation. + #[error( + "community {community_id} still has {active_count} active serving write lease(s): {operations:?}" + )] + ServingWritesNotDrained { + /// Community whose lifecycle transition must retry. + community_id: uuid::Uuid, + /// Number of currently unexpired serving-write leases. + active_count: i64, + /// Distinct operation categories holding those leases. + operations: Vec, + }, + + /// A deletion safety invariant is structurally violated and requires + /// operator/code remediation rather than blind retry. + #[error("deletion safety error: {0}")] + DeletionSafety(String), + /// A stored timestamp value could not be interpreted. #[error("invalid timestamp: {0}")] InvalidTimestamp(i64), diff --git a/crates/buzz-db/src/event.rs b/crates/buzz-db/src/event.rs index e1b45aa3a1d..db150571719 100644 --- a/crates/buzz-db/src/event.rs +++ b/crates/buzz-db/src/event.rs @@ -1111,7 +1111,7 @@ pub struct ThreadMetadataParams<'a> { pub broadcast: bool, } -async fn insert_event_with_thread_metadata_tx( +pub(crate) async fn insert_event_with_thread_metadata_tx( tx: &mut Transaction<'_, Postgres>, community_id: CommunityId, event: &Event, diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index cbb14173c98..1ba0909bbfb 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -17,6 +17,8 @@ pub mod api_token; pub mod archived_identities; /// Channel and membership persistence. pub mod channel; +/// Durable whole-community deletion lifecycle and PostgreSQL adapter. +pub mod deletion; /// Direct message channel persistence. pub mod dm; /// Database error types. @@ -651,7 +653,7 @@ impl Db { /// `buzz.created_at_floor` GUC — this is what makes the replica fence /// proof hold for every insert path that goes through this pool. pub async fn new(config: &DbConfig) -> Result { - let pool = Self::connect_pool(config, &config.database_url, true).await?; + let pool = Self::connect_pool(config, &config.database_url).await?; let read_max_connections = config .read_max_connections .unwrap_or(config.max_connections); @@ -671,31 +673,39 @@ impl Db { }) } - /// Connect one pool with the sizing knobs from `config`. + /// Connect the writer pool with all session-level safety premises. /// - /// `arm_floor_guard` sets the `buzz.created_at_floor` session GUC on - /// every connection, arming the deferred commit-time trigger from - /// migration 0021. Writer pools must arm it; replica pools are read-only - /// so the trigger never fires there. - async fn connect_pool(config: &DbConfig, url: &str, arm_floor_guard: bool) -> Result { - let mut options = PgPoolOptions::new() + /// SQLx stores one `after_connect` hook, so the floor guard and transaction + /// isolation assertion must remain in this single closure. Registering a + /// second hook replaces the first and silently disarms the floor trigger. + async fn connect_pool(config: &DbConfig, url: &str) -> Result { + let options = PgPoolOptions::new() .max_connections(config.max_connections) .min_connections(config.min_connections) .acquire_timeout(Duration::from_secs(config.acquire_timeout_secs)) .max_lifetime(Duration::from_secs(config.max_lifetime_secs)) - .idle_timeout(Duration::from_secs(config.idle_timeout_secs)); - if arm_floor_guard { - options = options.after_connect(|conn, _meta| { + .idle_timeout(Duration::from_secs(config.idle_timeout_secs)) + .after_connect(|conn, _meta| { Box::pin(async move { // `SET` cannot take bind parameters; `set_config` can. sqlx::query("SELECT set_config('buzz.created_at_floor', $1, false)") .bind(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - .execute(conn) + .execute(&mut *conn) + .await?; + let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") + .fetch_one(&mut *conn) .await?; + if isolation != "read committed" { + return Err(sqlx::Error::Configuration( + format!( + "writer pool requires READ COMMITTED transaction isolation, got {isolation}" + ) + .into(), + )); + } Ok(()) }) }); - } Ok(options.connect(url).await?) } @@ -720,8 +730,9 @@ impl Db { /// are dialed only on first acquire; the ~10-minute reaper never tops /// the pool back up, which is fine — routed reads re-fill it on demand. /// - /// No floor guard: replica sessions are read-only, the trigger never - /// fires there (see [`Db::connect_pool`]). + /// No floor guard or writer-isolation assertion: replica sessions are + /// read-only, so the commit-time trigger from migration 0021 never fires + /// here and the write fence that depends on READ COMMITTED is never reached. fn connect_read_pool(config: &DbConfig, url: &str, max_connections: u32) -> Result { Ok(PgPoolOptions::new() .max_connections(max_connections) @@ -1021,6 +1032,16 @@ impl Db { sqlx::query("SELECT 1").execute(&self.pool).await.is_ok() } + /// Validate the minimum deletion fence catalog required by serving paths. + pub async fn validate_deletion_serving_catalog(&self) -> Result<()> { + self.deletion_store().validate_serving_catalog().await + } + + /// Validate the exact live community-deletion tenant catalog for destruction. + pub async fn validate_deletion_catalog(&self) -> Result<()> { + self.deletion_store().validate_catalog().await + } + /// Returns pool utilisation stats for metrics emission. /// /// `size` — total connections (idle + active) @@ -1198,6 +1219,11 @@ impl Db { usage::community_hosts(&self.pool).await } + /// Return the shared durable whole-community deletion adapter. + pub fn deletion_store(&self) -> deletion::DeletionStore { + deletion::DeletionStore::new(self.pool.clone()) + } + /// Begin a database transaction for atomic multi-statement operations. /// /// Returns a `'static` transaction because `PgPool` is `Arc`-backed internally. @@ -1221,6 +1247,8 @@ impl Db { FROM communities WHERE lower(host) = lower($1) AND archived_at IS NULL + AND deleted_at IS NULL + AND deletion_state = 'active' "#, ) .bind(normalized_host) @@ -1243,7 +1271,7 @@ impl Db { #[datastore_span(name = "is_community_active", system = "postgresql")] pub async fn is_community_active(&self, community_id: CommunityId) -> Result { let active = sqlx::query_scalar::<_, bool>( - "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 AND archived_at IS NULL)", + "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 AND archived_at IS NULL AND deleted_at IS NULL AND deletion_state = 'active')", ) .bind(community_id.as_uuid()) .fetch_one(&self.pool) @@ -1331,6 +1359,8 @@ impl Db { FROM communities WHERE id = $1 AND archived_at IS NULL + AND deleted_at IS NULL + AND deletion_state = 'active' "#, ) .bind(community_id.as_uuid()) @@ -1404,12 +1434,19 @@ impl Db { INSERT INTO communities (host) VALUES ($1) ON CONFLICT (lower(host)) DO UPDATE SET host = communities.host + WHERE communities.deletion_state = 'active' + AND communities.deleted_at IS NULL RETURNING id, host, (xmax = 0) AS created "#, ) .bind(normalized_host) - .fetch_one(&self.pool) - .await?; + .fetch_optional(&self.pool) + .await? + .ok_or_else(|| { + DbError::AccessDenied(format!( + "community host {normalized_host:?} is permanently tombstoned" + )) + })?; let id: Uuid = row.try_get("id")?; let host: String = row.try_get("host")?; @@ -1490,6 +1527,8 @@ impl Db { AND lower(rm.pubkey) = lower($2) AND rm.role = 'owner' AND c.archived_at IS NULL + AND c.deletion_state = 'active' + AND c.deleted_at IS NULL "#, ) .bind(normalized_host) @@ -1529,6 +1568,8 @@ impl Db { AND lower(rm.pubkey) = lower($2) AND rm.role = 'owner' AND lower(c.host) <> lower($3) + AND c.deletion_state = 'active' + AND c.deleted_at IS NULL RETURNING c.id, c.host, c.archived_at"#, ) .bind(normalized_host) @@ -1561,6 +1602,8 @@ impl Db { AND rm.community_id = c.id AND lower(rm.pubkey) = lower($2) AND rm.role = 'owner' + AND c.deletion_state = 'active' + AND c.deleted_at IS NULL RETURNING c.id, c.host"#, ) .bind(normalized_host) @@ -1665,6 +1708,49 @@ impl Db { Ok(result) } + /// Insert an event while holding and validating an admitted serving-write + /// lease under the community ordering lock through commit. + /// + /// External side effects use a durable lease rather than one long-lived DB + /// transaction. Their final database mutation presents that exact lease so + /// it may finish during quiescing without admitting any new serving work. + pub async fn insert_event_with_serving_write_guard( + &self, + lease: &deletion::ServingWriteLease, + event: &nostr::Event, + channel_id: Option, + ) -> Result<(StoredEvent, bool)> { + let community_id = lease.community_id; + let kind_u16 = event.kind.as_u16(); + let kind_u32 = u32::from(kind_u16); + if kind_u32 == buzz_core::kind::KIND_AUTH { + return Err(DbError::AuthEventRejected); + } + if buzz_core::kind::is_ephemeral(kind_u32) { + return Err(DbError::EphemeralEventRejected(kind_u16)); + } + + let mut tx = self.pool.begin().await?; + self.deletion_store() + .guard_transaction_with_serving_lease(&mut tx, lease) + .await?; + let result = event::insert_event_with_thread_metadata_tx( + &mut tx, + community_id, + event, + channel_id, + None, + ) + .await?; + tx.commit().await?; + if result.1 { + if let Err(e) = insert_mentions(&self.pool, community_id, event, channel_id).await { + tracing::warn!(event_id = %event.id, "Failed to insert mentions: {e}"); + } + } + Ok(result) + } + /// Queries events matching the given filter parameters. /// /// Always reads from the WRITER pool. If the result influences a write @@ -4086,7 +4172,8 @@ impl Db { WHERE elem->>0 = 'd' LIMIT 1), \ '' \ ) \ - WHERE kind BETWEEN 30000 AND 39999 AND d_tag IS NULL", + WHERE kind BETWEEN 30000 AND 39999 AND d_tag IS NULL \ + AND community_write_allowed(community_id)", ) .execute(&self.pool) .await?; @@ -8524,6 +8611,76 @@ mod tests { drop_scratch_db(&admin, pool, &name).await; } + #[test] + fn writer_pool_safety_hook_is_single_and_composed() { + let source = include_str!("lib.rs"); + let connect_pool = source + .split("async fn connect_pool") + .nth(1) + .and_then(|tail| tail.split("const READER_ACQUIRE_TIMEOUT").next()) + .expect("connect_pool source block"); + assert_eq!( + connect_pool.matches(".after_connect(").count(), + 1, + "SQLx replaces after_connect hooks; writer safety must use exactly one" + ); + assert!(connect_pool.contains("buzz.created_at_floor")); + assert!(connect_pool.contains("SHOW transaction_isolation")); + assert!(!connect_pool.contains("arm_floor_guard")); + assert!(!connect_pool.contains("_arm_floor_guard")); + assert!(!connect_pool.contains("allow(unused_variables)")); + + let reader_doc = source + .split("fn connect_read_pool") + .next() + .and_then(|prefix| prefix.rsplit("/// Connect the read-replica").next()) + .expect("reader pool documentation"); + assert!(reader_doc.contains("replica sessions are")); + assert!(reader_doc.contains("read-only")); + assert!(!reader_doc.contains("Db::connect_pool")); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn writer_pool_rejects_non_read_committed_database_default() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (seed_pool, name) = create_scratch_db(&admin, "writer_isolation").await; + sqlx::query(sqlx::AssertSqlSafe(format!( + "ALTER DATABASE {name} SET default_transaction_isolation = 'repeatable read'" + ))) + .execute(&admin) + .await + .expect("set unsafe database default"); + seed_pool.close().await; + + let base = admin_url().await; + let idx = base.rfind('/').expect("db url has a path segment"); + let scratch_url = format!("{}/{}", &base[..idx], name); + let error = Db::new(&DbConfig { + database_url: scratch_url, + max_connections: 1, + min_connections: 1, + acquire_timeout_secs: 1, + ..DbConfig::default() + }) + .await + .expect_err("writer pool must reject pinned-snapshot database defaults"); + assert!( + error.to_string().contains("requires READ COMMITTED") + || error.to_string().contains("pool timed out"), + "unexpected isolation rejection: {error}" + ); + + sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE {name} WITH (FORCE)" + ))) + .execute(&admin) + .await + .expect("drop isolation test database"); + } + /// The armed writer pool (`Db::new`) must enforce the floor end-to-end /// through the public insert APIs, and the session GUC must be verifiably /// set on pooled connections. @@ -8563,6 +8720,14 @@ mod tests { crate::replica_fence::CREATED_AT_FLOOR_SECS.to_string(), "writer pool must arm the floor guard on every connection" ); + let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") + .fetch_one(&db.pool) + .await + .expect("SHOW writer isolation"); + assert_eq!( + isolation, "read committed", + "the same writer after_connect hook must enforce the isolation premise" + ); let now_secs = chrono::Utc::now().timestamp() as u64; let floor = crate::replica_fence::CREATED_AT_FLOOR_SECS as u64; diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 43f464526c3..4ccf987f140 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -4,16 +4,37 @@ //! multi-tenant rewrite owns a clean consolidated `0001`; legacy single-tenant //! cutover/backfill is a separate operator script, not startup migration state. -use sqlx::PgPool; +use std::future::Future; +use sqlx::{Connection, PgConnection, PgPool}; + +use crate::deletion::SCHEMA_DESTRUCTION_LOCK_KEY; use crate::Result; static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("../../migrations"); /// Run all pending Buzz database migrations. +/// +/// The entire run holds the exclusive [`SCHEMA_DESTRUCTION_LOCK_KEY`] session +/// lock, serializing schema changes against destructive deletion transactions +/// (which take the shared counterpart while they validate the live catalog +/// and act on it). Every migration statement executes on the same backend +/// that owns the lock — see [`with_exclusive_schema_destruction_lock`] for +/// why that binding, not the explicit unlock, is the safety contract. +/// Migration execution must never bypass this wrapper — a source lint +/// (`migration_execution_cannot_bypass_schema_destruction_lock`) enforces +/// that `MIGRATOR.run` has no other call site. pub async fn run_migrations(pool: &PgPool) -> Result<()> { - reject_legacy_nip_rs_cardinality_ambiguity(pool).await?; - MIGRATOR.run(pool).await?; + with_exclusive_schema_destruction_lock(pool, |mut conn| async move { + let outcome = run_migrations_locked(&mut conn).await; + (conn, outcome) + }) + .await +} + +async fn run_migrations_locked(conn: &mut PgConnection) -> Result<()> { + reject_legacy_nip_rs_cardinality_ambiguity(conn).await?; + MIGRATOR.run(&mut *conn).await?; // The replica-fence proof (see `replica_fence`) requires the commit-time // `created_at` floor trigger from migration 0021 — correctly shaped — on // the `events` parent and every partition. `CREATE TABLE .. PARTITION OF` @@ -21,25 +42,61 @@ pub async fn run_migrations(pool: &PgPool) -> Result<()> { // PARTITION` or created by an older code path would silently escape the // guard, so migration fails closed if any is missing. (The fence probe // re-runs this same check at startup on non-migrating relays.) - crate::replica_fence::verify_floor_guard_catalog(pool).await?; + crate::replica_fence::verify_floor_guard_catalog(&mut *conn).await?; Ok(()) } +/// Run `op` while holding the exclusive schema/destruction session lock. +/// +/// `op` receives ownership of the detached connection that owns the advisory +/// lock and must run every statement on it, handing the same connection back +/// with its outcome. That same-backend lifetime — not the explicit unlock — +/// is the safety contract: PostgreSQL releases a session lock only when its +/// backend finishes, so cancelling this future (dropping the connection while +/// a migration statement is still executing server-side) cannot expose the +/// lock to shared destructive holders before that statement's backend +/// terminates. On completion the lock is explicitly released on the returned +/// connection (success and error alike) and the connection is closed, never +/// returning a locked session to the pool. +pub(crate) async fn with_exclusive_schema_destruction_lock( + pool: &PgPool, + op: F, +) -> Result +where + F: FnOnce(PgConnection) -> Fut, + Fut: Future)>, +{ + let mut lock_conn = pool.acquire().await?.detach(); + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + .execute(&mut lock_conn) + .await?; + let (mut lock_conn, outcome) = op(lock_conn).await; + let unlock = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + .execute(&mut lock_conn) + .await; + let _ = lock_conn.close().await; + let value = outcome?; + unlock?; + Ok(value) +} + /// Migration 0007 is checksum-frozen and predates exact NIP-RS tag-cardinality /// enforcement. A populated database still on 0001-0006 must not let 0007 /// irreversibly purge duplicate-tag history. Fail before sqlx starts its /// migration transaction so an operator can inspect and repair those rows. -async fn reject_legacy_nip_rs_cardinality_ambiguity(pool: &PgPool) -> Result<()> { +async fn reject_legacy_nip_rs_cardinality_ambiguity(conn: &mut PgConnection) -> Result<()> { let migrations_table: Option = sqlx::query_scalar("SELECT to_regclass('_sqlx_migrations')::text") - .fetch_one(pool) + .fetch_one(&mut *conn) .await?; if migrations_table.is_none() { return Ok(()); } let applied: Option = sqlx::query_scalar("SELECT max(version) FROM _sqlx_migrations WHERE success") - .fetch_one(pool) + .fetch_one(&mut *conn) .await?; if applied.is_none_or(|version| version >= 7) { return Ok(()); @@ -83,7 +140,7 @@ async fn reject_legacy_nip_rs_cardinality_ambiguity(pool: &PgPool) -> Result<()> )\ )", ) - .fetch_one(pool) + .fetch_one(conn) .await?; if ambiguous { @@ -348,6 +405,13 @@ mod tests { "push_gateway_delivery_request_replays", "product_feedback", "replica_heartbeat", + "community_deletion_requests", + "community_deletion_approvals", + "community_deletion_checkpoints", + "community_deletion_manifest_keys", + "storage_taxonomy_sweeps", + "community_serving_write_leases", + "community_deletion_executor_heartbeats", ] { if normalized[insert_pos..].contains(&format!("'{value}'")) { globals.insert(value.to_owned()); @@ -561,8 +625,8 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - // FORK-LOCAL PATCH (adrienlacombe/buzz): upstream ships 28 migrations; this - // fork adds two of its own, so the count is 30 here. + // FORK-LOCAL PATCH (adrienlacombe/buzz): upstream ships 30 migrations; this + // fork adds two of its own, so the count is 32 here. // // The fork's 0027 and 0028 belong to the NIP-SW wallet binding, which this // fork has since removed in favour of the Nostr key controlling the Starknet @@ -576,10 +640,12 @@ mod tests { // // Because the fork holds 0027 and 0028, upstream's own new migrations arrive // renumbered above them: upstream's `0027_channels_id_lookup_index.sql` is - // `0029_channels_id_lookup_index.sql` here, and its - // `0028_long_reaction_payloads.sql` is `0030_long_reaction_payloads.sql`. - // See the assertions for both below. - assert_eq!(migrations.len(), 30); + // `0029_channels_id_lookup_index.sql` here, its + // `0028_long_reaction_payloads.sql` is `0030_long_reaction_payloads.sql`, + // its `0029_community_deletion.sql` is `0031_community_deletion.sql`, and + // its `0030_community_deletion_recovery.sql` is + // `0032_community_deletion_recovery.sql`. See the assertions for each below. + assert_eq!(migrations.len(), 32); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -952,14 +1018,8 @@ mod tests { assert!(channel_id_index.contains("idx_channels_id_live")); assert!(channel_id_index.contains("INCLUDE (community_id)")); assert!(channel_id_index.contains("WHERE deleted_at IS NULL")); - assert!( - !channel_id_index.contains("CREATE UNIQUE INDEX"), - "channels.id is not unique across communities — index must not be UNIQUE", - ); - assert!( - desired_schema.contains("idx_channels_id_live"), - "desired-state schema must carry the channel-id lookup index", - ); + assert!(!channel_id_index.contains("CREATE UNIQUE INDEX")); + assert!(desired_schema.contains("idx_channels_id_live")); // Long reaction payloads (upstream's 0028, FORK-LOCAL PATCH // (adrienlacombe/buzz): renumbered to 0030 here because this fork already @@ -970,6 +1030,46 @@ mod tests { long_reactions.contains("ALTER TABLE reactions ALTER COLUMN emoji TYPE VARCHAR(66)") ); assert!(desired_schema.contains("emoji VARCHAR(66) NOT NULL")); + + // Durable whole-community deletion control plane and universal DB fence + // (upstream's 0029, FORK-LOCAL PATCH (adrienlacombe/buzz): renumbered to + // 0031 here because this fork already holds 0027 and 0028). + assert_eq!(migrations[30].version, 31); + let deletion = migrations[30].sql.as_str(); + assert!(deletion.contains("CREATE TABLE community_deletion_requests")); + assert!(deletion.contains("CREATE TABLE community_deletion_approvals")); + assert!(deletion.contains("CREATE TABLE community_deletion_checkpoints")); + assert!(deletion.contains("CREATE TABLE community_serving_write_leases")); + assert!(deletion.contains("CREATE TABLE community_deletion_executor_heartbeats")); + assert!(deletion.contains("CREATE FUNCTION community_write_allowed")); + assert!(deletion.contains("LANGUAGE plpgsql VOLATILE")); + assert!(deletion.contains("CREATE FUNCTION assert_community_write_allowed")); + assert!(deletion.contains("current_setting('transaction_isolation') <> 'read committed'")); + assert!(deletion.contains("ERRCODE = 'invalid_transaction_state'")); + assert!(deletion.contains("CREATE FUNCTION enforce_community_write_fence")); + assert!(deletion.contains("CREATE FUNCTION attach_community_write_fence")); + assert!(deletion.contains("community_write_fence_excluded_table")); + assert!(deletion.contains("CREATE FUNCTION enforce_community_tombstone")); + assert!(deletion.contains("community tombstones are permanent")); + assert!(deletion.contains("SET LOCAL lock_timeout = '5s'")); + assert!(deletion.contains("'active', 'quiescing', 'fenced', 'tombstone'")); + assert!(deletion.contains("_operator_global_tables")); + assert!(deletion.contains("'submitted', 'inventoried', 'approved', 'fenced', 'drained'")); + assert!(deletion.contains("UNIQUE (id, community_id, inventory_digest)")); + assert!(deletion.contains("FOREIGN KEY (request_id, community_id, inventory_digest)")); + assert!(deletion.contains("prevent_community_deletion_request_retargeting")); + assert!(deletion.contains("prevent_community_deletion_approval_removal")); + + assert!(deletion.contains("retry_stage TEXT CHECK")); + assert!(desired_schema.contains("retry_stage TEXT CHECK")); + + // Recovery migration alters populated tables and must preserve the same + // fail-fast lock behavior as the deletion migration (upstream's 0030, + // FORK-LOCAL PATCH (adrienlacombe/buzz): renumbered to 0032 here because + // this fork already holds 0027 and 0028). + assert_eq!(migrations[31].version, 32); + let deletion_recovery = migrations[31].sql.as_str(); + assert!(deletion_recovery.contains("SET LOCAL lock_timeout = '5s'")); } #[test] @@ -1115,6 +1215,552 @@ mod tests { ); } + #[test] + fn migration_execution_cannot_bypass_schema_destruction_lock() { + fn rust_sources(dir: &std::path::Path, files: &mut Vec) { + for entry in std::fs::read_dir(dir).expect("read workspace source dir") { + let path = entry.expect("read workspace source entry").path(); + if path.is_dir() { + if path.file_name().is_some_and(|name| name == "target") { + continue; + } + rust_sources(&path, files); + } else if path.extension().is_some_and(|ext| ext == "rs") { + files.push(path); + } + } + } + fn count(haystack: &str, needle: &str) -> usize { + haystack.matches(needle).count() + } + + // Build the needles so this test's own source never matches them. + let migrate_macro = ["sqlx", "::migrate!"].concat(); + let migrator_run = ["MIGRATOR", ".run("].concat(); + + let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let this_file = manifest_dir.join("src/migration.rs"); + let crates_dir = manifest_dir.parent().expect("workspace crates dir"); + // The push gateway migrates its own dedicated authority database; it + // never holds relay tenant tables, so it is exempt from the relay + // schema/destruction lock. The community_id check below keeps that + // exemption honest. + let push_gateway_exception = crates_dir.join("buzz-push-gateway/src/postgres.rs"); + let push_gateway_migrations = crates_dir.join("buzz-push-gateway/migrations"); + for entry in + std::fs::read_dir(&push_gateway_migrations).expect("read push gateway migrations") + { + let path = entry.expect("read push gateway migration entry").path(); + let sql = std::fs::read_to_string(&path).expect("read push gateway migration"); + assert!( + !sql.to_ascii_lowercase().contains("community_id"), + "{} defines community-scoped data; its migrator would bypass the \ + schema/destruction lock and must move under buzz-db migrations", + path.display() + ); + } + let mut files = Vec::new(); + rust_sources(crates_dir, &mut files); + for path in &files { + let source = std::fs::read_to_string(path).expect("read rust source"); + let (macro_hits, run_hits) = ( + count(&source, &migrate_macro), + count(&source, &migrator_run), + ); + if *path == this_file { + assert_eq!( + (macro_hits, run_hits), + (1, 1), + "migration.rs must embed the migrator once and run it exactly once, \ + inside the locked wrapper" + ); + } else if *path == push_gateway_exception { + continue; + } else { + assert_eq!( + (macro_hits, run_hits), + (0, 0), + "{} embeds or runs a SQLx migrator outside the schema/destruction \ + lock contract; route migration execution through \ + buzz_db migration::run_migrations", + path.display() + ); + } + } + + // Within migration.rs, the single run site must sit inside + // `run_migrations_locked`, and the only public entry point must wrap + // it in the exclusive session lock. + let source = std::fs::read_to_string(&this_file).expect("read migration.rs"); + let entry = source + .find("pub async fn run_migrations(") + .expect("public migration entry point"); + let locked = source + .find("async fn run_migrations_locked(") + .expect("locked migration body"); + let wrapper = source + .find("async fn with_exclusive_schema_destruction_lock") + .expect("exclusive lock wrapper"); + let run_site = source.find(&migrator_run).expect("migrator run site"); + assert!( + source[entry..locked].contains("with_exclusive_schema_destruction_lock("), + "run_migrations must delegate through the exclusive schema/destruction lock" + ); + assert!( + run_site > locked && run_site < wrapper, + "the migrator run site must live inside run_migrations_locked" + ); + assert!( + source[wrapper..].contains("pg_advisory_lock($1)") + && source[wrapper..].contains("pg_advisory_unlock($1)"), + "the lock wrapper must acquire and explicitly release the session lock" + ); + } + + /// Structural parity between migration 0029's deletion surface and the + /// desired-state bootstrap schema (`schema/schema.sql`). + /// + /// Compares parsed statements, not substrings: every deletion control- + /// plane table, function, trigger, and index 0028 creates must exist in + /// schema.sql with an identical normalized definition; every operator- + /// global registry row 0028 inserts must be inserted by schema.sql; the + /// write-fence attachment target sets must be equal; and every column + /// 0028 adds to `communities` must exist in the desired-state + /// `communities` table. A desired-state bootstrap that passes this test + /// cannot silently omit part of the deletion surface the way the + /// pre-parity schema.sql omitted `community_deletion_manifest_keys` (and + /// its immutability trigger) and `storage_taxonomy_sweeps` — booting + /// healthy, then wedging post-fence when the freeze stage first touched + /// the missing relation. + #[test] + fn deletion_surface_parity_between_migration_0029_and_schema_sql() { + use std::collections::BTreeMap; + + #[derive(Default)] + struct DeletionSurface { + tables: BTreeMap, + functions: BTreeMap, + triggers: BTreeMap, + indexes: BTreeSet, + registry_rows: BTreeSet<(String, String)>, + fence_attachments: BTreeSet, + communities_added_columns: BTreeSet, + } + + fn quoted_strings(statement: &str) -> Vec { + let mut strings = Vec::new(); + let mut current: Option = None; + let mut chars = statement.chars().peekable(); + while let Some(ch) = chars.next() { + match (&mut current, ch) { + (None, '\'') => current = Some(String::new()), + (Some(literal), '\'') => { + if chars.peek() == Some(&'\'') { + literal.push('\''); + chars.next(); + } else { + strings.push(current.take().expect("open literal")); + } + } + (Some(literal), other) => literal.push(other), + (None, _) => {} + } + } + strings + } + + fn surface(sql: &str) -> DeletionSurface { + let mut surface = DeletionSurface::default(); + for statement in split_sql_statements(sql) { + let normalized = normalize_sql(&statement); + if normalized.starts_with("create table") { + let table = identifier_after_keyword(&statement, "create table") + .expect("table identifier"); + surface.tables.insert(table, normalized.clone()); + } else if normalized.starts_with("create function") + || normalized.starts_with("create or replace function") + { + let function = identifier_after_keyword(&statement, "function") + .expect("function identifier"); + surface.functions.insert(function, normalized.clone()); + } else if normalized.starts_with("create trigger") { + let trigger = identifier_after_keyword(&statement, "create trigger") + .expect("trigger identifier"); + surface.triggers.insert(trigger, normalized.clone()); + } else if normalized.starts_with("create index") + || normalized.starts_with("create unique index") + { + surface.indexes.insert(normalized.clone()); + } else if normalized.starts_with("insert into _operator_global_tables") { + let literals = quoted_strings(&statement); + assert!( + literals.len().is_multiple_of(2), + "operator-global registry insert must be (table_name, reason) rows" + ); + for row in literals.chunks(2) { + surface + .registry_rows + .insert((row[0].clone(), row[1].clone())); + } + } else if normalized.starts_with("alter table communities") { + for added in normalized.split("add column ").skip(1) { + let column = added + .split_whitespace() + .next() + .expect("added column name") + .to_owned(); + surface.communities_added_columns.insert(column); + } + } + if let Some(position) = normalized.find("attach_community_write_fence('") { + let target = normalized[position + "attach_community_write_fence('".len()..] + .split('\'') + .next() + .expect("fence attachment target") + .to_owned(); + surface.fence_attachments.insert(target); + } + } + surface + } + + // FORK-LOCAL PATCH (adrienlacombe/buzz): upstream's deletion migration is + // 0029; it is 0031 here because this fork already holds 0027 and 0028, so + // upstream's own new migrations arrive renumbered above them. Only this + // version literal moves — the surrounding names and assertion messages are + // left as upstream wrote them so this stays one hunk for the next merge. + let migration_0029: &str = MIGRATOR + .iter() + .find(|migration| migration.version == 31) + .expect("embedded migration 0031 (upstream's 0029)") + .sql + .as_ref(); + let workspace_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(std::path::Path::parent) + .expect("workspace root"); + let schema_sql = std::fs::read_to_string(workspace_root.join("schema/schema.sql")) + .expect("read schema/schema.sql"); + + let migration = surface(migration_0029); + let schema = surface(&schema_sql); + + assert_eq!( + migration.tables.len(), + 7, + "0029 deletion control plane must define exactly the known tables: {:?}", + migration.tables.keys().collect::>() + ); + assert!(!migration.fence_attachments.is_empty()); + assert!(!migration.registry_rows.is_empty()); + + for (table, definition) in &migration.tables { + let in_schema = schema + .tables + .get(table) + .unwrap_or_else(|| panic!("schema.sql is missing deletion table {table}")); + if table != "community_deletion_requests" { + assert_eq!( + in_schema, definition, + "schema.sql definition of {table} drifted from migration 0029" + ); + } + } + for (function, definition) in &migration.functions { + let in_schema = schema + .functions + .get(function) + .unwrap_or_else(|| panic!("schema.sql is missing deletion function {function}")); + if function != "community_write_fence_excluded_table" { + assert_eq!( + in_schema, definition, + "schema.sql definition of {function}() drifted from migration 0029" + ); + } + } + for (trigger, definition) in &migration.triggers { + let in_schema = schema + .triggers + .get(trigger) + .unwrap_or_else(|| panic!("schema.sql is missing deletion trigger {trigger}")); + assert_eq!( + in_schema, definition, + "schema.sql definition of trigger {trigger} drifted from migration 0029" + ); + } + for index in &migration.indexes { + assert!( + schema.indexes.contains(index), + "schema.sql is missing (or drifted on) deletion index: {index}" + ); + } + for row in &migration.registry_rows { + assert!( + schema.registry_rows.contains(row), + "schema.sql is missing operator-global registry row {row:?}" + ); + } + let mut expected_fences = migration.fence_attachments.clone(); + expected_fences.remove("product_feedback"); + expected_fences.remove("rate_limit_violations"); + assert_eq!( + expected_fences, schema.fence_attachments, + "write-fence attachment targets differ after recovery policy" + ); + + // 0029's ALTER TABLE additions are expressed inline by the + // desired-state `communities` definition; require the columns to + // exist there (exact definition equality is impossible across the + // ALTER/inline representations — behavior is pinned by the + // desired-state bootstrap deletion test). + let communities_columns = split_sql_statements(&schema_sql) + .into_iter() + .find_map(|statement| { + let (table, body) = create_table_body(&statement)?; + (table == "communities").then_some(body) + }) + .expect("schema.sql defines communities"); + let column_names: BTreeSet = communities_columns + .iter() + .filter_map(|definition| column_definition_name(definition)) + .collect(); + for column in &migration.communities_added_columns { + assert!( + column_names.contains(column), + "schema.sql communities table is missing 0028 column {column}" + ); + } + assert!(!migration.communities_added_columns.is_empty()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn schema_destruction_lock_excludes_shared_holders_and_releases_on_both_paths() { + let pool = connect_test_pool().await; + async fn assert_exclusive_lock_free(pool: &PgPool) { + let mut probe = pool.acquire().await.expect("acquire lock probe"); + let free: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock($1)") + .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + .fetch_one(&mut *probe) + .await + .expect("probe try-lock"); + assert!(free, "schema/destruction session lock must be released"); + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + .execute(&mut *probe) + .await + .expect("probe unlock"); + } + + let probe_pool = pool.clone(); + with_exclusive_schema_destruction_lock(&pool, move |conn| async move { + // While a migration run is in flight, destructive transactions + // must be unable to take their shared counterpart. + let mut probe = probe_pool.acquire().await.expect("acquire shared probe"); + let shared_available: bool = + sqlx::query_scalar("SELECT pg_try_advisory_lock_shared($1)") + .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + .fetch_one(&mut *probe) + .await + .expect("probe shared try-lock"); + assert!( + !shared_available, + "exclusive migration lock must exclude shared destructive holders" + ); + (conn, Ok(())) + }) + .await + .expect("locked migration op"); + assert_exclusive_lock_free(&pool).await; + + let failed: Result<()> = with_exclusive_schema_destruction_lock(&pool, |conn| async { + ( + conn, + Err(crate::DbError::InvalidData( + "forced migration failure".into(), + )), + ) + }) + .await; + assert!(failed.is_err(), "op failure must propagate"); + assert_exclusive_lock_free(&pool).await; + } + + /// Cancellation must not release the exclusion contract while migration + /// SQL is still executing server-side. + /// + /// The op parks an `ALTER TABLE` behind an ACCESS EXCLUSIVE table lock + /// held by another session, then the whole locked run is aborted. Because + /// the advisory lock lives on the same backend that runs the DDL, + /// dropping the client future cannot release it: the backend keeps the + /// session lock until it finishes the statement and dies on the closed + /// socket. The shared (destructive) counterpart must stay unavailable for + /// that entire interval — and the orphaned DDL really does commit after + /// cancellation, which is exactly the window the lock has to cover. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn cancelled_migration_cannot_expose_shared_lock_while_ddl_backend_lives() { + use std::time::Instant; + + use sqlx::AssertSqlSafe; + + // Dedicated database: the probe table and the orphaned backend must + // stay invisible to concurrent tests in the shared database. + let base_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| TEST_DB_URL.to_owned()); + let admin = PgPool::connect(&base_url) + .await + .expect("connect admin database"); + let probe_db = format!("buzz_lock_cancel_{}", uuid::Uuid::new_v4().simple()); + sqlx::query(AssertSqlSafe(format!("CREATE DATABASE {probe_db}"))) + .execute(&admin) + .await + .expect("create probe database"); + let (base_prefix, _) = base_url.rsplit_once('/').expect("database url has a path"); + let pool = PgPool::connect(&format!("{base_prefix}/{probe_db}")) + .await + .expect("connect probe database"); + sqlx::query("CREATE TABLE schema_lock_cancel_probe (id int)") + .execute(&pool) + .await + .expect("create probe table"); + + // Park the migration DDL server-side: the op's ALTER TABLE waits on + // this ACCESS EXCLUSIVE lock, pinning the backend mid-statement. + let mut blocker = pool.begin().await.expect("open blocker transaction"); + sqlx::query("LOCK TABLE schema_lock_cancel_probe IN ACCESS EXCLUSIVE MODE") + .execute(&mut *blocker) + .await + .expect("hold probe table lock"); + + let (pid_tx, pid_rx) = tokio::sync::oneshot::channel::(); + let task_pool = pool.clone(); + let locked_run = tokio::spawn(async move { + with_exclusive_schema_destruction_lock(&task_pool, move |mut conn| async move { + let outcome: Result<()> = async { + let pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut conn) + .await?; + let _ = pid_tx.send(pid); + sqlx::query( + "ALTER TABLE schema_lock_cancel_probe \ + ADD COLUMN committed_after_cancel int", + ) + .execute(&mut conn) + .await?; + Ok(()) + } + .await; + (conn, outcome) + }) + .await + }); + let ddl_pid = pid_rx.await.expect("locked op reports its backend pid"); + let deadline = Instant::now() + std::time::Duration::from_secs(10); + loop { + let waiting: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity \ + WHERE pid = $1 AND wait_event_type = 'Lock')", + ) + .bind(ddl_pid) + .fetch_one(&pool) + .await + .expect("poll DDL wait state"); + if waiting { + break; + } + assert!( + Instant::now() < deadline, + "migration DDL never parked on the table lock" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + + locked_run.abort(); + let joined = locked_run.await; + assert!( + joined.is_err_and(|err| err.is_cancelled()), + "locked migration run must abort mid-statement" + ); + + // The client future is gone, but the DDL backend is alive: the shared + // destructive lock must remain unavailable for that whole interval. + for _ in 0..20 { + let backend_alive: bool = + sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid = $1)") + .bind(ddl_pid) + .fetch_one(&pool) + .await + .expect("poll DDL backend liveness"); + assert!( + backend_alive, + "parked DDL backend must outlive client cancellation" + ); + let shared_free: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock_shared($1)") + .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + .fetch_one(&pool) + .await + .expect("probe shared lock"); + assert!( + !shared_free, + "cancellation must not expose the shared lock while migration DDL is executing" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + + // Release the table lock: the orphaned backend finishes the ALTER, + // commits, then exits on the dead socket — only then may shared + // destructive holders enter. + blocker.rollback().await.expect("release probe table lock"); + let mut probe = pool.acquire().await.expect("acquire shared-lock probe"); + let deadline = Instant::now() + std::time::Duration::from_secs(30); + loop { + let shared_free: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock_shared($1)") + .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + .fetch_one(&mut *probe) + .await + .expect("probe shared lock after backend exit"); + if shared_free { + sqlx::query("SELECT pg_advisory_unlock_shared($1)") + .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + .execute(&mut *probe) + .await + .expect("release shared probe lock"); + break; + } + assert!( + Instant::now() < deadline, + "shared lock must become available once the DDL backend exits" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + drop(probe); + + // The cancelled statement committed after the client vanished — + // exactly the interval the same-backend lock covered. + let committed: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM information_schema.columns \ + WHERE table_name = 'schema_lock_cancel_probe' \ + AND column_name = 'committed_after_cancel')", + ) + .fetch_one(&pool) + .await + .expect("inspect orphaned DDL outcome"); + assert!( + committed, + "orphaned migration DDL commits after cancellation; the lock must cover it" + ); + + pool.close().await; + sqlx::query(AssertSqlSafe(format!( + "DROP DATABASE {probe_db} WITH (FORCE)" + ))) + .execute(&admin) + .await + .expect("drop probe database"); + } + async fn connect_test_pool() -> PgPool { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) @@ -1212,7 +1858,15 @@ mod tests { run_migrations(&pool) .await .expect("retry succeeds after operator repair"); - assert_eq!(applied_versions(&pool).await.last().copied(), Some(30)); + let latest_version = MIGRATOR + .iter() + .map(|migration| migration.version) + .max() + .expect("embedded migrator is non-empty"); + assert_eq!( + applied_versions(&pool).await.last().copied(), + Some(latest_version) + ); } #[tokio::test] @@ -1334,5 +1988,163 @@ mod tests { search_expression.contains("ELSE NULL::tsvector"), "fresh installs must default non-allowlisted kinds to NULL: {search_expression}" ); + + let active_a = uuid::Uuid::new_v4(); + let active_b = uuid::Uuid::new_v4(); + let to_fence = uuid::Uuid::new_v4(); + for (community, label) in [ + (active_a, "active-a"), + (active_b, "active-b"), + (to_fence, "to-fence"), + ] { + sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") + .bind(community) + .bind(format!("late-fence-{label}-{}.example", community.simple())) + .execute(&pool) + .await + .expect("insert late-table test community"); + } + sqlx::query( + "CREATE TABLE late_created_scoped (\ + community_id UUID NOT NULL, id BIGINT PRIMARY KEY, value TEXT NOT NULL\ + )", + ) + .execute(&pool) + .await + .expect("create late scoped table"); + sqlx::query("SELECT attach_community_write_fence('late_created_scoped'::regclass)") + .execute(&pool) + .await + .expect("attach late create fence"); + sqlx::query("CREATE TABLE late_altered_scoped (id BIGINT PRIMARY KEY)") + .execute(&pool) + .await + .expect("create table before late alter"); + sqlx::query("ALTER TABLE late_altered_scoped ADD COLUMN community_id UUID NOT NULL") + .execute(&pool) + .await + .expect("add late community id"); + sqlx::query("SELECT attach_community_write_fence('late_altered_scoped'::regclass)") + .execute(&pool) + .await + .expect("attach late alter fence"); + let attached: Vec = sqlx::query_scalar( + "SELECT c.relname FROM pg_trigger trigger \ + JOIN pg_class c ON c.oid = trigger.tgrelid \ + JOIN pg_proc procedure ON procedure.oid = trigger.tgfoid \ + WHERE c.relname IN ('late_created_scoped', 'late_altered_scoped') \ + AND procedure.proname = 'enforce_community_write_fence' \ + AND NOT trigger.tgisinternal ORDER BY c.relname", + ) + .fetch_all(&pool) + .await + .expect("read late trigger catalog"); + assert_eq!(attached, vec!["late_altered_scoped", "late_created_scoped"]); + let malformed_fence_triggers: i64 = sqlx::query_scalar( + "SELECT count(*)::BIGINT FROM pg_trigger trigger \ + JOIN pg_class c ON c.oid = trigger.tgrelid \ + JOIN pg_proc procedure ON procedure.oid = trigger.tgfoid \ + WHERE c.relname IN ('late_created_scoped', 'late_altered_scoped') \ + AND procedure.proname = 'enforce_community_write_fence' \ + AND NOT trigger.tgisinternal \ + AND (trigger.tgenabled <> 'O' OR (trigger.tgtype & 31) <> 31)", + ) + .fetch_one(&pool) + .await + .expect("validate late trigger mode and operations"); + assert_eq!(malformed_fence_triggers, 0); + + sqlx::query( + "INSERT INTO late_created_scoped (community_id, id, value) \ + VALUES ($1, 1, 'same'), ($2, 2, 'source-fenced'), \ + ($1, 3, 'destination-fenced'), ($1, 4, 'opposite-a'), \ + ($3, 5, 'opposite-b')", + ) + .bind(active_a) + .bind(to_fence) + .bind(active_b) + .execute(&pool) + .await + .expect("seed late table while communities active"); + sqlx::query("UPDATE late_created_scoped SET value = 'same-ok' WHERE id = 1") + .execute(&pool) + .await + .expect("same-tenant active update"); + sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 1") + .bind(active_b) + .execute(&pool) + .await + .expect("active-to-active update"); + + let mut fence_connection = pool.acquire().await.expect("fence connection"); + sqlx::query("BEGIN") + .execute(&mut *fence_connection) + .await + .expect("begin direct fence"); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '1', true)", + ) + .bind(to_fence.to_string()) + .execute(&mut *fence_connection) + .await + .expect("authorize direct fence"); + sqlx::query( + "UPDATE communities SET deletion_state = 'fenced', \ + deletion_fence_generation = 1, archived_at = now() WHERE id = $1", + ) + .bind(to_fence) + .execute(&mut *fence_connection) + .await + .expect("fence test destination"); + sqlx::query("COMMIT") + .execute(&mut *fence_connection) + .await + .expect("commit direct fence"); + + let active_to_fenced = + sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 3") + .bind(to_fence) + .execute(&pool) + .await + .expect_err("active to fenced destination must fail"); + assert!(active_to_fenced + .to_string() + .contains("community write fenced")); + let fenced_to_active = + sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 2") + .bind(active_a) + .execute(&pool) + .await + .expect_err("fenced source to active destination must fail"); + assert!(fenced_to_active + .to_string() + .contains("community write fenced")); + let row_locations: Vec<(i64, uuid::Uuid)> = sqlx::query_as( + "SELECT id, community_id FROM late_created_scoped WHERE id IN (2, 3) ORDER BY id", + ) + .fetch_all(&pool) + .await + .expect("failed moves preserve row location"); + assert_eq!(row_locations, vec![(2, to_fence), (3, active_a)]); + + let move_a = sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 4") + .bind(active_b) + .execute(&pool); + let move_b = sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 5") + .bind(active_a) + .execute(&pool); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + let (a, b) = tokio::join!(move_a, move_b); + a.expect("opposite active move A"); + b.expect("opposite active move B"); + }) + .await + .expect("opposite cross-tenant updates must not deadlock"); + + sqlx::query("DROP TABLE late_created_scoped, late_altered_scoped") + .execute(&pool) + .await + .expect("drop late-table fixtures"); } } diff --git a/crates/buzz-db/src/push.rs b/crates/buzz-db/src/push.rs index 04b6a7ae363..0b3245ffcc2 100644 --- a/crates/buzz-db/src/push.rs +++ b/crates/buzz-db/src/push.rs @@ -852,6 +852,7 @@ where WHERE attempts < $3 AND next_attempt_at <= now() AND (state = 'pending' OR (state = 'matching' AND lease_until < now())) + AND community_write_allowed(community_id) ORDER BY next_attempt_at, created_at LIMIT 1 ), @@ -933,7 +934,8 @@ where pub async fn reap_exhausted_matches(pool: &PgPool) -> Result { Ok(sqlx::query( "DELETE FROM push_match_queue WHERE attempts >= $1 \ - AND (state='pending' OR (state='matching' AND lease_until < now()))", + AND (state='pending' OR (state='matching' AND lease_until < now())) \ + AND community_write_allowed(community_id)", ) .bind(MAX_MATCH_ATTEMPTS) .execute(pool) @@ -2121,6 +2123,128 @@ mod tests { ); } + async fn seed_matcher_fixture( + pool: &PgPool, + community: CommunityId, + marker: u8, + attempts: i32, + age_seconds: i64, + ) { + let event_id = vec![marker; 32]; + sqlx::query( + "INSERT INTO events \ + (community_id, id, pubkey, created_at, kind, tags, content, sig) \ + VALUES ($1, $2, $3, to_timestamp(1), 9, '[]', '', $4)", + ) + .bind(community.as_uuid()) + .bind(&event_id) + .bind(vec![marker.saturating_add(20); 32]) + .bind(vec![marker.saturating_add(30); 64]) + .execute(pool) + .await + .expect("seed source event"); + sqlx::query( + "INSERT INTO push_match_queue \ + (community_id, event_id, attempts, next_attempt_at) \ + VALUES ($1, $2, $3, now() - make_interval(secs => $4))", + ) + .bind(community.as_uuid()) + .bind(event_id) + .bind(attempts) + .bind(age_seconds) + .execute(pool) + .await + .expect("seed matcher row"); + } + + async fn quiesce_test_community(pool: &PgPool, community: CommunityId) { + let mut lifecycle = pool.begin().await.expect("begin lifecycle fixture"); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '0', true)", + ) + .bind(community.to_string()) + .execute(&mut *lifecycle) + .await + .expect("authorize target lifecycle"); + sqlx::query("UPDATE communities SET deletion_state = 'quiescing' WHERE id = $1") + .bind(community.as_uuid()) + .execute(&mut *lifecycle) + .await + .expect("quiesce target"); + lifecycle.commit().await.expect("commit target lifecycle"); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn matcher_claim_skips_quiescing_tenant_while_active_bystanders_progress() { + let pool = setup_pool().await; + sqlx::query("DELETE FROM push_match_queue WHERE community_write_allowed(community_id)") + .execute(&pool) + .await + .expect("drain active matcher queue"); + let active_a = make_community(&pool).await; + let target = make_community(&pool).await; + let active_x = make_community(&pool).await; + seed_matcher_fixture(&pool, active_a, 1, 0, 30).await; + seed_matcher_fixture(&pool, target, 2, 0, 40).await; + seed_matcher_fixture(&pool, active_x, 3, 0, 20).await; + quiesce_test_community(&pool, target).await; + + let batch = claim_due_match_batch(&pool, 16, Utc::now() + chrono::Duration::minutes(1)) + .await + .expect("claim active bystander") + .expect("active A is claimable despite older target row"); + assert_eq!(batch.community, active_a); + let target_state: String = + sqlx::query_scalar("SELECT state FROM push_match_queue WHERE community_id = $1") + .bind(target.as_uuid()) + .fetch_one(&pool) + .await + .expect("target row remains attributed"); + assert_eq!(target_state, "pending"); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn exhausted_match_reaper_skips_quiescing_tenant_and_reaps_active_bystanders() { + let pool = setup_pool().await; + sqlx::query("DELETE FROM push_match_queue WHERE community_write_allowed(community_id)") + .execute(&pool) + .await + .expect("drain active matcher queue"); + let active_a = make_community(&pool).await; + let target = make_community(&pool).await; + let active_x = make_community(&pool).await; + seed_matcher_fixture(&pool, active_a, 4, MAX_MATCH_ATTEMPTS, 30).await; + seed_matcher_fixture(&pool, target, 5, MAX_MATCH_ATTEMPTS, 40).await; + seed_matcher_fixture(&pool, active_x, 6, MAX_MATCH_ATTEMPTS, 20).await; + quiesce_test_community(&pool, target).await; + + assert_eq!( + reap_exhausted_matches(&pool) + .await + .expect("reap active bystanders"), + 2 + ); + let target_remaining: i64 = + sqlx::query_scalar("SELECT count(*) FROM push_match_queue WHERE community_id = $1") + .bind(target.as_uuid()) + .fetch_one(&pool) + .await + .expect("target exhausted row remains attributed"); + assert_eq!(target_remaining, 1); + for active in [active_a, active_x] { + let active_remaining: i64 = + sqlx::query_scalar("SELECT count(*) FROM push_match_queue WHERE community_id = $1") + .bind(active.as_uuid()) + .fetch_one(&pool) + .await + .expect("active bystander is drained"); + assert_eq!(active_remaining, 0); + } + } + /// T2b batch contract: one claim returns jobs from exactly ONE community /// (so downstream lease/membership loads are single statements), the /// set-wise complete and retry honor the claim fence, and a retried job diff --git a/crates/buzz-db/src/relay_invite.rs b/crates/buzz-db/src/relay_invite.rs index 82b71b07bb0..14331b022f5 100644 --- a/crates/buzz-db/src/relay_invite.rs +++ b/crates/buzz-db/src/relay_invite.rs @@ -111,6 +111,14 @@ pub async fn mint_relay_invite( let now = Utc::now(); let expires_at = now + chrono::Duration::seconds(ttl_secs as i64); + // Mint a v2 opaque invite inside the same lifecycle gate as every other + // community-scoped database write. The trigger remains the final backstop, + // but this typed guard keeps a quiescing community from surfacing as an + // opaque SQLSTATE/HTTP 500 at the API boundary. + let mut tx = pool.begin().await?; + crate::deletion::DeletionStore::new(pool.clone()) + .guard_transaction(&mut tx, community) + .await?; let row = sqlx::query( "INSERT INTO relay_invites (community_id, token_hash, max_uses, expires_at, created_by) \ VALUES ($1, $2, $3, $4, $5) \ @@ -121,8 +129,9 @@ pub async fn mint_relay_invite( .bind(max_uses) .bind(expires_at) .bind(created_by) - .fetch_one(pool) + .fetch_one(&mut *tx) .await?; + tx.commit().await?; let invite_id: uuid::Uuid = row.try_get("id")?; @@ -167,6 +176,7 @@ pub async fn reap_expired_relay_invites(pool: &PgPool, cutoff: DateTime) -> WHERE (community_id, id) IN (\ SELECT community_id, id FROM relay_invites \ WHERE expires_at < $1 \ + AND community_write_allowed(community_id) \ ORDER BY expires_at \ LIMIT $2\ )", @@ -374,20 +384,53 @@ pub async fn claim_relay_invite( mod tests { use super::*; use crate::relay_members::is_relay_member; + use sha2::Digest; use sqlx::PgPool; use uuid::Uuid; const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1 async fn setup_pool() -> PgPool { - let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") - .or_else(|_| std::env::var("DATABASE_URL")) - .unwrap_or_else(|_| TEST_DB_URL.to_owned()); - PgPool::connect(&database_url) + PgPool::connect(&test_database_url()) .await .expect("connect to test DB") } + fn test_database_url() -> String { + std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| TEST_DB_URL.to_owned()) + } + + async fn create_scratch_database(prefix: &str) -> (PgPool, String, String) { + let admin_url = test_database_url(); + let admin = PgPool::connect(&admin_url) + .await + .expect("connect to test database server"); + let name = format!("{}_{}", prefix, Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}"))) + .execute(&admin) + .await + .expect("create scratch database"); + let path_start = admin_url + .rfind('/') + .expect("database URL has a path segment"); + let scratch_url = format!("{}/{}", &admin_url[..path_start], name); + (admin, name, scratch_url) + } + + async fn drop_scratch_database(admin: PgPool, db: crate::Db, name: &str) { + db.pool.close().await; + drop(db); + sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE IF EXISTS {name} WITH (FORCE)" + ))) + .execute(&admin) + .await + .expect("drop scratch database"); + admin.close().await; + } + async fn make_test_community(pool: &PgPool) -> CommunityId { let id = Uuid::new_v4(); sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") @@ -448,6 +491,95 @@ mod tests { } } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn mint_after_quiescing_returns_typed_fence_without_persisting() { + let (admin, database_name, database_url) = + create_scratch_database("relay_invite_fence").await; + let db = crate::Db::new(&crate::DbConfig { + database_url, + max_connections: 5, + min_connections: 0, + ..crate::DbConfig::default() + }) + .await + .expect("connect invite deletion test DB"); + db.migrate().await.expect("migrate invite deletion test DB"); + let pool = db.pool.clone(); + let store = db.deletion_store(); + let host = format!("relay-invite-fence-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("create fenced invite community") + .id; + let request = store + .submit(&host, "owner", None) + .await + .expect("submit deletion request"); + let empty_digest = hex::encode(sha2::Sha256::digest([])); + let inventory = crate::deletion::FrozenInventory { + schema: store + .inventory_schema(community) + .await + .expect("inventory schema"), + storage: crate::deletion::StorageManifest { + version: 4, + prefixes: [ + format!("_meta/{community}/"), + format!("_uploads/{community}/"), + format!("repos/{community}/"), + ] + .into_iter() + .map(|prefix| crate::deletion::PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: empty_digest.clone(), + }) + .collect(), + }, + }; + store + .freeze_inventory(request.id, &inventory) + .await + .expect("freeze inventory"); + store + .approve(request.id, "owner", None) + .await + .expect("approve deletion"); + let claim = store + .claim_specific( + request.id, + "executor", + crate::deletion::DEFAULT_LEASE_DURATION, + ) + .await + .expect("claim deletion") + .expect("runnable deletion"); + store + .begin_quiescing(&claim.lease) + .await + .expect("begin quiescing"); + + let error = mint_relay_invite(&pool, community, "owner", 3600, Some(1)) + .await + .expect_err("quiescing must reject invite minting"); + assert!(matches!(error, crate::error::DbError::AccessDenied(_))); + + let invite_count: i64 = + sqlx::query_scalar("SELECT count(*) FROM relay_invites WHERE community_id = $1") + .bind(community.as_uuid()) + .fetch_one(&pool) + .await + .expect("count relay invites"); + assert_eq!(invite_count, 0, "rejected mint must not persist an invite"); + + drop(store); + drop(pool); + drop_scratch_database(admin, db, &database_name).await; + } + #[tokio::test] #[ignore = "requires Postgres"] async fn bounded_claim_exhausts_and_existing_member_retry_does_not_consume() { @@ -615,6 +747,73 @@ mod tests { delete_test_community(&pool, community).await; } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn retention_sweep_skips_quiescing_tenant_while_active_bystanders_progress() { + let (admin, database_name, database_url) = + create_scratch_database("relay_invite_liveness").await; + let db = crate::Db::new(&crate::DbConfig { + database_url, + max_connections: 5, + min_connections: 0, + ..crate::DbConfig::default() + }) + .await + .expect("connect invite liveness database"); + db.migrate() + .await + .expect("migrate invite liveness database"); + let pool = db.pool.clone(); + let active_a = make_test_community(&pool).await; + let target = make_test_community(&pool).await; + let active_x = make_test_community(&pool).await; + let cutoff = Utc::now(); + for community in [active_a, target, active_x] { + sqlx::query( + "INSERT INTO relay_invites \ + (community_id, token_hash, expires_at, created_by) \ + VALUES ($1, $2, $3, 'test')", + ) + .bind(community.as_uuid()) + .bind(sha2::Sha256::digest(community.as_uuid().as_bytes()).as_slice()) + .bind(cutoff - chrono::Duration::seconds(1)) + .execute(&pool) + .await + .expect("seed expired invite"); + } + let mut lifecycle = pool.begin().await.expect("begin lifecycle fixture"); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '0', true)", + ) + .bind(target.to_string()) + .execute(&mut *lifecycle) + .await + .expect("authorize lifecycle fixture"); + sqlx::query("UPDATE communities SET deletion_state = 'quiescing' WHERE id = $1") + .bind(target.as_uuid()) + .execute(&mut *lifecycle) + .await + .expect("quiesce target"); + lifecycle.commit().await.expect("commit lifecycle fixture"); + + assert_eq!( + reap_expired_relay_invites(&pool, cutoff) + .await + .expect("reap active bystanders"), + 2 + ); + let remaining: Vec = + sqlx::query_scalar("SELECT community_id FROM relay_invites ORDER BY community_id") + .fetch_all(&pool) + .await + .expect("read remaining invite attribution"); + assert_eq!(remaining, vec![*target.as_uuid()]); + + drop(pool); + drop_scratch_database(admin, db, &database_name).await; + } + #[tokio::test] #[ignore = "requires Postgres"] async fn unlimited_invites_count_each_new_member() { diff --git a/crates/buzz-db/src/replica_fence.rs b/crates/buzz-db/src/replica_fence.rs index 98bdd9850eb..83322bea141 100644 --- a/crates/buzz-db/src/replica_fence.rs +++ b/crates/buzz-db/src/replica_fence.rs @@ -317,8 +317,13 @@ impl ReplicaFence { /// /// This is a name-and-shape check only; it cannot detect a sabotaged /// function body. [`verify_floor_guard_behavior`] proves the semantics. +/// +/// Generic over the executor so the migration path can run it on the +/// lock-holding connection while the startup probe keeps using the pool. #[datastore_span(name = "replica_fence_verify_catalog", system = "postgresql")] -pub async fn verify_floor_guard_catalog(pool: &PgPool) -> crate::Result<()> { +pub async fn verify_floor_guard_catalog<'e>( + executor: impl sqlx::PgExecutor<'e>, +) -> crate::Result<()> { // tgtype bits: 1 = ROW, 2 = BEFORE, 4 = INSERT, 16 = UPDATE, 64 = INSTEAD. // Required: ROW + INSERT + UPDATE set, BEFORE + INSTEAD clear. let missing: Vec = sqlx::query_scalar( @@ -345,7 +350,7 @@ pub async fn verify_floor_guard_catalog(pool: &PgPool) -> crate::Result<()> { ) "#, ) - .fetch_all(pool) + .fetch_all(executor) .await?; if !missing.is_empty() { return Err(crate::error::DbError::InvalidData(format!( diff --git a/crates/buzz-db/src/workflow.rs b/crates/buzz-db/src/workflow.rs index 7a2396c1fd7..ad1fd3a9396 100644 --- a/crates/buzz-db/src/workflow.rs +++ b/crates/buzz-db/src/workflow.rs @@ -602,6 +602,7 @@ pub async fn prune_scheduled_workflow_fires_before( r#" DELETE FROM scheduled_workflow_fires WHERE claimed_at < $1 + AND community_write_allowed(community_id) "#, ) .bind(older_than) diff --git a/crates/buzz-deletion/Cargo.toml b/crates/buzz-deletion/Cargo.toml new file mode 100644 index 00000000000..8c308b0a8c0 --- /dev/null +++ b/crates/buzz-deletion/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "buzz-deletion" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Durable whole-community deletion engine for Buzz" + +[dependencies] +anyhow = { workspace = true } +thiserror = { workspace = true } +buzz-core = { workspace = true } +buzz-db = { workspace = true } +buzz-media = { workspace = true } +chrono = { workspace = true } +clap = { version = "4", features = ["derive"] } +deadpool-redis = { workspace = true } +hex = { workspace = true } +redis = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +sqlx = { workspace = true } diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs new file mode 100644 index 00000000000..4e27b85fe9f --- /dev/null +++ b/crates/buzz-deletion/src/lib.rs @@ -0,0 +1,2063 @@ +#![deny(unsafe_code)] +#![warn(missing_docs)] +//! Shared durable whole-community deletion engine and store adapters. + +#[cfg(test)] +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use buzz_db::deletion::{ + ClaimedDeletion, DeletionRequest, DeletionStage, DeletionStore, FrozenInventory, + KeyStreamDigest, LeaseToken, PrefixManifest, StorageManifest, DEFAULT_LEASE_DURATION, +}; +use buzz_db::{Db, DbConfig}; +use buzz_media::{is_tenant_owned_key, tenant_prefixes, MediaStorage}; +use clap::Subcommand; +use serde::Serialize; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +/// Fleet-wide object cap for one observational taxonomy sweep. +const DEFAULT_SWEEP_OBJECT_CAP: u64 = 10_000_000; +/// Keys per frozen side-table chunk (one `DeleteObjects`-sized unit × 10). +const DEFAULT_MANIFEST_CHUNK_KEYS: usize = 10_000; +/// Unknown keys retained verbatim on a sweep record for diagnosis. +const SWEEP_UNKNOWN_KEY_SAMPLE: usize = 100; +/// One S3 LIST page. +const LIST_PAGE_SIZE: usize = 1000; +const RETRY_DELAY: Duration = Duration::from_secs(30); +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(10); + +fn heartbeat_interval() -> Duration { + HEARTBEAT_INTERVAL +} + +#[derive(Debug, Clone, thiserror::Error)] +#[error("deletion execution lease heartbeat failed")] +struct DeletionLeaseLost; + +#[derive(Debug, Clone, thiserror::Error)] +#[error("{message}")] +struct ServingWriteLeaseLost { + message: String, +} + +/// Return the shared durable deletion store for relay and operator paths. +pub fn store(db: &Db) -> DeletionStore { + db.deletion_store() +} + +/// Durable, heartbeated lease for a serving-path external side effect. +pub struct ServingWriteGuard { + store: DeletionStore, + lease: buzz_db::deletion::ServingWriteLease, + cancel: CancellationToken, + lost: CancellationToken, + finished: bool, +} + +impl ServingWriteGuard { + /// Verify this side-effect lease is still current before an irreversible call. + pub async fn verify(&self) -> Result<()> { + if self.lost.is_cancelled() { + return Err(ServingWriteLeaseLost { + message: "serving write lease heartbeat was lost".to_string(), + } + .into()); + } + self.store + .verify_serving_write_lease(&self.lease) + .await + .map_err(|error| ServingWriteLeaseLost { + message: error.to_string(), + })?; + Ok(()) + } + + /// Run an external side effect while observing lease-heartbeat loss. + /// + /// Dropping the operation future on lease loss prevents a stale caller from + /// continuing network I/O after its durable exclusion proof disappears. + pub async fn protect(&self, operation: F) -> Result + where + F: std::future::Future, + { + self.verify().await?; + let output = tokio::select! { + biased; + output = operation => output, + _ = self.lost.cancelled() => { + return Err(ServingWriteLeaseLost { + message: "serving write lease heartbeat was lost".to_string(), + } + .into()) + } + }; + self.verify().await?; + Ok(output) + } + + /// Whether an error represents loss of a durable serving-write lease. + pub fn is_lease_lost(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() + } + + /// Whether a serving-write acquisition failed because the tenant is fenced. + pub fn acquisition_is_fenced(error: &anyhow::Error) -> bool { + matches!( + error.downcast_ref::(), + Some(buzz_db::DbError::AccessDenied(_)) + ) + } + + /// Signal fired if the background lease heartbeat fails. + pub fn lost(&self) -> CancellationToken { + self.lost.clone() + } + + /// The durable lease token presented to a final database mutation. + pub fn lease(&self) -> &buzz_db::deletion::ServingWriteLease { + &self.lease + } + + /// Release the lease after the side effect completes. + pub async fn finish(mut self) -> Result<()> { + self.cancel.cancel(); + let released = self.store.release_serving_write_lease(&self.lease).await?; + self.finished = true; + if !released { + return Err(ServingWriteLeaseLost { + message: "serving write lease was already stale or released".to_string(), + } + .into()); + } + Ok(()) + } +} + +impl Drop for ServingWriteGuard { + fn drop(&mut self) { + self.cancel.cancel(); + if self.finished { + return; + } + let store = self.store.clone(); + let lease = self.lease.clone(); + tokio::spawn(async move { + let _ = store.release_serving_write_lease(&lease).await; + }); + } +} + +/// Acquire a serving-side external-effect lease without holding a pool connection. +/// +/// A separate short database lease per effect is intentional: it is the only +/// durable proof that deletion can drain S3/Redis/push work across replicas. +/// PostgreSQL lease-table churn is reaped and exported by the relay pool-metrics +/// task; operators should watch the deletion lease gauges documented by Helm. +pub async fn acquire_serving_write( + db: &Db, + community: buzz_core::CommunityId, + operation: &str, +) -> Result { + acquire_serving_write_with_heartbeat(db, community, operation, heartbeat_interval()).await +} + +async fn acquire_serving_write_with_heartbeat( + db: &Db, + community: buzz_core::CommunityId, + operation: &str, + heartbeat_period: Duration, +) -> Result { + let store = store(db); + let owner = default_executor_id(); + let lease = store + .acquire_serving_write_lease(community, operation, &owner, DEFAULT_LEASE_DURATION) + .await?; + let heartbeat_store = store.clone(); + let mut heartbeat_lease = lease.clone(); + let cancel = CancellationToken::new(); + let heartbeat_cancel = cancel.clone(); + let lost = CancellationToken::new(); + let heartbeat_lost = lost.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(heartbeat_period); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + tokio::select! { + _ = heartbeat_cancel.cancelled() => return, + _ = interval.tick() => { + if heartbeat_store + .renew_serving_write_lease( + &mut heartbeat_lease, + DEFAULT_LEASE_DURATION, + ) + .await + .is_err() + { + heartbeat_lost.cancel(); + return; + } + } + } + } + }); + Ok(ServingWriteGuard { + store, + lease, + cancel, + lost, + finished: false, + }) +} + +/// CLI-only whole-community deletion commands. +#[derive(Subcommand)] +pub enum Command { + /// Persist a deletion request and freeze its initial cross-store inventory. + Submit { + /// Canonical community host. Defaults to RELAY_URL's authority. + #[arg(long)] + host: Option, + /// Operator identity recorded on the request. + #[arg(long)] + requested_by: String, + /// Optional reason for the request. + #[arg(long)] + reason: Option, + }, + /// List deletion requests as JSON. + List { + /// Maximum records. + #[arg(long, default_value_t = 100, value_parser = clap::value_parser!(u16).range(1..=1000))] + limit: u16, + }, + /// Inspect one request, including approval/checkpoints/errors. + Inspect { + /// Deletion request UUID. + id: Uuid, + }, + /// Explicitly approve the exact frozen inventory digest. + Approve { + /// Deletion request UUID. + id: Uuid, + /// Approving operator identity. + #[arg(long)] + approved_by: String, + /// Optional approval note. + #[arg(long)] + note: Option, + }, + /// Terminally cancel before irreversible object deletion begins. + Abort { + /// Deletion request UUID. + id: Uuid, + /// Aborting operator identity. + #[arg(long)] + aborted_by: String, + /// Reason recorded in immutable audit evidence. + #[arg(long)] + reason: String, + }, + /// Resume a blocked request after remediating its recorded failure. + Unblock { + /// Deletion request UUID. + id: Uuid, + /// Operator identity recorded in the recovery checkpoint. + #[arg(long)] + unblocked_by: String, + /// Remediation or change reference recorded in the checkpoint. + #[arg(long)] + reason: String, + }, + /// Claim and run one request until terminal/blocked. + Run { + /// Deletion request UUID. + id: Uuid, + /// Executor identity (defaults to hostname/pid). + #[arg(long)] + executor_id: Option, + }, + /// Drain the currently runnable deletion queue, then exit. + Drain { + /// Executor identity (defaults to hostname/pid). + #[arg(long)] + executor_id: Option, + }, + /// Sweep the whole bucket's key taxonomy and record observational evidence. + /// + /// This is independent of community deletion. It reports unknown writer + /// shapes but never gates submission or destructive progress. + Sweep, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LoopMode { + Run, + Drain, +} + +impl LoopMode { + const fn as_str(self) -> &'static str { + match self { + Self::Run => "run", + Self::Drain => "drain", + } + } +} + +#[derive(Clone)] +struct Services { + store: DeletionStore, + media: Arc, + redis: deadpool_redis::Pool, +} + +#[derive(Debug, thiserror::Error)] +enum EngineError { + #[error("permanent deletion safety failure: {0}")] + Permanent(String), + #[error("transient deletion dependency failure: {0}")] + Transient(String), +} + +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +struct PermanentSource(#[from] anyhow::Error); + +fn permanent(message: impl Into) -> anyhow::Error { + EngineError::Permanent(message.into()).into() +} + +fn permanent_source(error: impl Into) -> anyhow::Error { + PermanentSource(error.into()).into() +} + +fn transient(message: impl Into) -> anyhow::Error { + EngineError::Transient(message.into()).into() +} + +fn is_permanent_error(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause.is::() + || matches!( + cause.downcast_ref::(), + Some(buzz_db::DbError::DeletionSafety(_)) + ) + || cause + .downcast_ref::() + .is_some_and(|error| matches!(error, EngineError::Permanent(_))) + }) +} + +#[derive(Debug, Serialize)] +struct RunOutput { + request_id: Uuid, + stage: DeletionStage, + retry_count: i32, + last_error: Option, + next_attempt_at: chrono::DateTime, + blocked_reason: Option, +} + +/// Execute one nested deletion command. +pub async fn run(command: Command) -> Result { + match command { + Command::List { limit } => { + let store = connect_store().await?; + print_json(&store.list(i64::from(limit)).await?)?; + Ok(0) + } + Command::Inspect { id } => { + let store = connect_store().await?; + print_json(&store.inspect(id).await?)?; + Ok(0) + } + Command::Approve { + id, + approved_by, + note, + } => { + let store = connect_store().await?; + print_json(&store.approve(id, &approved_by, note.as_deref()).await?)?; + Ok(0) + } + Command::Abort { + id, + aborted_by, + reason, + } => { + let store = connect_store().await?; + print_json(&store.abort(id, &aborted_by, &reason).await?)?; + Ok(0) + } + Command::Unblock { + id, + unblocked_by, + reason, + } => { + let store = connect_store().await?; + print_json(&store.unblock(id, &unblocked_by, &reason).await?)?; + Ok(0) + } + Command::Run { id, executor_id } => { + let store = connect_store().await?; + let services = match connect_services_with_store(store.clone()).await { + Ok(services) => services, + Err(error) => { + let message = format!("{error:#}"); + store + .block_preclaim_setup(id, "pre_claim:service_setup", &message) + .await + .context("record pre-claim service setup failure")?; + return Err(error); + } + }; + run_loop( + services, + LoopMode::Run, + Some(id), + executor_id.unwrap_or_else(default_executor_id), + ) + .await + } + command => run_with_services(command, connect_services().await?).await, + } +} + +async fn run_with_services(command: Command, services: Services) -> Result { + match command { + Command::Submit { + host, + requested_by, + reason, + } => { + let relay_url = std::env::var("RELAY_URL").ok(); + let host = resolve_submit_host(host.as_deref(), relay_url.as_deref())?; + let request = services + .store + .submit(&host, &requested_by, reason.as_deref()) + .await?; + let inventory = build_inventory(&services, &request).await?; + let request = services + .store + .freeze_inventory(request.id, &inventory) + .await?; + print_json(&request)?; + Ok(0) + } + Command::Run { id, executor_id } => { + run_loop( + services, + LoopMode::Run, + Some(id), + executor_id.unwrap_or_else(default_executor_id), + ) + .await + } + Command::Drain { executor_id } => { + run_loop( + services, + LoopMode::Drain, + None, + executor_id.unwrap_or_else(default_executor_id), + ) + .await + } + Command::Sweep => { + let started_at = chrono::Utc::now(); + let cap = sweep_object_cap(); + let media = Arc::clone(&services.media); + let outcome = + buzz_media::sweep_bucket_taxonomy(cap, SWEEP_UNKNOWN_KEY_SAMPLE, move |token| { + let media = Arc::clone(&media); + async move { media.list_page(token, LIST_PAGE_SIZE).await } + }) + .await?; + let sweep = services + .store + .record_taxonomy_sweep( + started_at, + outcome.listed_objects, + outcome.unknown_object_count, + &outcome.unknown_key_sample, + cap, + ) + .await?; + print_json(&sweep)?; + Ok(i32::from(sweep.unknown_object_count > 0)) + } + Command::List { .. } + | Command::Inspect { .. } + | Command::Approve { .. } + | Command::Abort { .. } + | Command::Unblock { .. } => { + anyhow::bail!("database-only command reached full-service dispatcher") + } + } +} + +fn resolve_submit_host(host: Option<&str>, relay_url: Option<&str>) -> Result { + if let Some(host) = host { + let host = host.trim(); + if host.is_empty() { + anyhow::bail!("--host must not be empty"); + } + return Ok(host.to_owned()); + } + + let relay_url = relay_url + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + anyhow::anyhow!("cannot derive community host; pass --host or set RELAY_URL") + })?; + let host = buzz_core::tenant::relay_url_authority(relay_url); + if host.is_empty() { + anyhow::bail!( + "cannot derive community host from RELAY_URL; pass --host or set a valid RELAY_URL" + ); + } + Ok(host) +} + +async fn connect_store() -> Result { + let database_url = required_env("DATABASE_URL")?; + let db = Db::new(&DbConfig { + database_url, + max_connections: env_parse("BUZZ_DB_POOL_SIZE", 20), + ..DbConfig::default() + }) + .await?; + Ok(store(&db)) +} + +fn resolve_s3_region(buzz_region: Option, aws_region: Option) -> String { + buzz_region + .and_then(nonempty_s3_region) + .or_else(|| aws_region.and_then(nonempty_s3_region)) + .unwrap_or_else(|| "us-east-1".to_string()) +} + +fn nonempty_s3_region(region: String) -> Option { + let region = region.trim(); + (!region.is_empty()).then(|| region.to_string()) +} + +fn s3_region_from_env() -> String { + resolve_s3_region( + std::env::var("BUZZ_S3_REGION").ok(), + std::env::var("AWS_REGION").ok(), + ) +} + +async fn connect_services() -> Result { + let store = connect_store().await?; + connect_services_with_store(store).await +} + +async fn connect_services_with_store(store: DeletionStore) -> Result { + let media_config = buzz_media::MediaConfig { + s3_endpoint: required_env("BUZZ_S3_ENDPOINT")?, + s3_access_key: required_env("BUZZ_S3_ACCESS_KEY")?, + s3_secret_key: required_env("BUZZ_S3_SECRET_KEY")?, + s3_bucket: required_env("BUZZ_S3_BUCKET")?, + s3_region: s3_region_from_env(), + s3_addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") + .unwrap_or_else(|_| "path".to_string()) + .parse() + .map_err(anyhow::Error::msg)?, + max_image_bytes: 1, + max_gif_bytes: 1, + max_video_bytes: 1, + max_file_bytes: 1, + public_base_url: "http://localhost/media".to_string(), + upload_records_enabled: false, + upload_ip_header: None, + upload_port_header: None, + }; + let media = Arc::new(MediaStorage::new(&media_config)?); + let redis_url = required_env("REDIS_URL")?; + let mut redis_config = deadpool_redis::Config::from_url(&redis_url); + redis_config.pool = Some(deadpool_redis::PoolConfig::new(env_parse( + "BUZZ_REDIS_POOL_SIZE", + 16, + ))); + let redis = redis_config + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .context("create deletion Redis pool")?; + Ok(Services { + store, + media, + redis, + }) +} + +fn required_env(name: &str) -> Result { + std::env::var(name) + .ok() + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) + .ok_or_else(|| anyhow::anyhow!("{name} is required for community deletion")) +} + +fn env_parse(name: &str, default: T) -> T +where + T: std::str::FromStr, +{ + std::env::var(name) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) +} + +fn validate_frozen_inventory(request: &DeletionRequest) -> Result { + let frozen: FrozenInventory = serde_json::from_value( + request + .inventory_manifest + .clone() + .ok_or_else(|| permanent("approved request has no frozen inventory"))?, + ) + .map_err(permanent_source)?; + let expected_digest = request + .inventory_digest + .as_deref() + .ok_or_else(|| permanent("approved request has no frozen inventory digest"))?; + let actual_digest = hex::encode(frozen.digest().map_err(permanent_source)?); + if actual_digest != expected_digest { + return Err(permanent("approved frozen inventory digest mismatch")); + } + validate_storage_ownership(request, &frozen.storage)?; + Ok(frozen) +} + +fn validate_storage_ownership(request: &DeletionRequest, manifest: &StorageManifest) -> Result<()> { + buzz_db::deletion::validate_storage_manifest(manifest)?; + let expected = tenant_prefixes(*request.community_id.as_uuid()); + let actual = manifest + .prefixes + .iter() + .map(|prefix| prefix.prefix.as_str()) + .collect::>(); + if actual != expected.iter().map(String::as_str).collect::>() { + return Err(permanent( + "storage manifest prefixes are not the deletion target's tenant prefixes", + )); + } + Ok(()) +} + +async fn build_inventory( + services: &Services, + request: &DeletionRequest, +) -> Result { + let schema = services + .store + .inventory_schema(request.community_id) + .await?; + let storage = enumerate_tenant_prefixes(services, request, None, None).await?; + Ok(FrozenInventory { schema, storage }) +} + +/// Buffered writer for frozen key chunks during the destructive freeze. +struct ChunkSink<'a> { + token: &'a LeaseToken, + next_chunk_no: i64, + buffered: Vec, +} + +async fn flush_chunk(services: &Services, sink: &mut ChunkSink<'_>, prefix: &str) -> Result<()> { + if sink.buffered.is_empty() { + return Ok(()); + } + services + .store + .append_manifest_key_chunk(sink.token, sink.next_chunk_no, prefix, &sink.buffered) + .await?; + sink.next_chunk_no += 1; + sink.buffered.clear(); + Ok(()) +} + +/// Enumerate the target's three tenant prefixes into per-prefix summaries. +/// +/// Cost is O(tenant objects) regardless of fleet size. Unknown shapes inside +/// one of the owned prefixes fail closed; keys elsewhere in the shared bucket +/// are outside this operation's contract. Memory stays bounded at one listing +/// page plus one buffered chunk — the full key list is never materialized. +/// When `sink` is supplied, keys are also persisted as side-table chunks +/// (never spanning prefixes) for the destructive freeze to bind against these +/// digests. +async fn enumerate_tenant_prefixes( + services: &Services, + request: &DeletionRequest, + heartbeat_lost: Option<&CancellationToken>, + mut sink: Option<&mut ChunkSink<'_>>, +) -> Result { + if services.media.bucket_versioning_detected().await? { + return Err(permanent( + "bucket versioning detected; deletion cannot prove logical absence with delete markers", + )); + } + let community = *request.community_id.as_uuid(); + let chunk_keys = manifest_chunk_keys(); + let mut prefixes = Vec::new(); + for prefix in tenant_prefixes(community) { + let mut digest = KeyStreamDigest::new(); + let mut total_bytes: u64 = 0; + let mut continuation = None; + loop { + if heartbeat_lost.is_some_and(CancellationToken::is_cancelled) { + return Err(DeletionLeaseLost.into()); + } + let page = services + .media + .list_prefix_page(&prefix, continuation.take(), LIST_PAGE_SIZE) + .await?; + for (key, size) in page.objects { + if !is_tenant_owned_key(community, &key) { + return Err(permanent(format!( + "key under a tenant prefix is outside the exact writer taxonomy: {key}" + ))); + } + digest.fold(&key)?; + total_bytes = total_bytes.saturating_add(size); + if let Some(sink) = sink.as_deref_mut() { + sink.buffered.push(key); + if sink.buffered.len() >= chunk_keys { + flush_chunk(services, sink, &prefix).await?; + } + } + } + if !page.is_truncated { + break; + } + continuation = page.next_continuation_token; + if continuation.is_none() { + return Err(transient( + "truncated tenant listing page has no continuation token", + )); + } + } + if let Some(sink) = sink.as_deref_mut() { + flush_chunk(services, sink, &prefix).await?; + } + let (keys_digest, object_count) = digest.finish(); + prefixes.push(PrefixManifest { + prefix, + object_count, + total_bytes, + keys_digest, + }); + } + let manifest = StorageManifest { + version: 4, + prefixes, + }; + buzz_db::deletion::validate_storage_manifest(&manifest)?; + Ok(manifest) +} + +/// Freeze the post-fence, post-drain destructive enumeration: stream the +/// tenant prefixes into side-table chunks, then bind the chunk stream to the +/// request row's digests atomically. +async fn freeze_destructive_manifest( + services: &Services, + request: &DeletionRequest, + token: &LeaseToken, + heartbeat_lost: &CancellationToken, +) -> Result { + validate_frozen_inventory(request)?; + // A prior interrupted freeze may have left partial chunks; they were + // never bound to a committed manifest, so rewrite them from scratch. + services.store.clear_manifest_key_chunks(token).await?; + let mut sink = ChunkSink { + token, + next_chunk_no: 0, + buffered: Vec::new(), + }; + let manifest = + enumerate_tenant_prefixes(services, request, Some(heartbeat_lost), Some(&mut sink)).await?; + services + .store + .freeze_destructive_storage_manifest(token, &manifest) + .await?; + Ok(manifest) +} + +async fn run_loop( + services: Services, + mode: LoopMode, + request_id: Option, + executor_id: String, +) -> Result { + let shutdown = shutdown_token(); + let mut ran = false; + loop { + if shutdown.is_cancelled() { + services + .store + .stop_executor(None, &executor_id) + .await + .context("record executor drain")?; + return Ok(0); + } + let claim = match request_id { + Some(id) => { + services + .store + .claim_specific(id, &executor_id, DEFAULT_LEASE_DURATION) + .await? + } + None => { + services + .store + .claim_next(&executor_id, DEFAULT_LEASE_DURATION) + .await? + } + }; + let Some(claim) = claim else { + if mode == LoopMode::Run && !ran { + anyhow::bail!( + "deletion request is not runnable, is blocked, or is leased by another executor" + ); + } + return Ok(0); + }; + ran = true; + let output = execute_claim(&services, mode, claim, &shutdown).await?; + print_json(&output)?; + let failed = output.last_error.is_some() || output.blocked_reason.is_some(); + if mode == LoopMode::Run || shutdown.is_cancelled() || failed { + return Ok(i32::from(failed)); + } + } +} + +async fn stop_claim_executor( + services: &Services, + mode: LoopMode, + token: &LeaseToken, +) -> Result<()> { + // A failed draining heartbeat must not prevent the generation-checked release + // attempt. `stop_executor` cannot clear a successor's reclaimed lease. + let _ = services + .store + .heartbeat(token, mode.as_str(), DEFAULT_LEASE_DURATION, true) + .await; + services + .store + .stop_executor(Some(token), &token.owner) + .await?; + Ok(()) +} + +async fn record_stage_failure( + services: &Services, + token: &LeaseToken, + stage: DeletionStage, + error: &anyhow::Error, +) -> Result { + let message = format!("{error:#}"); + let result = if is_permanent_error(error) { + services.store.block(token, stage, "stage", &message).await + } else { + services + .store + .record_retry(token, stage, "stage", &message, RETRY_DELAY) + .await + }; + match result { + Ok(()) => Ok(true), + Err(error) if buzz_db::deletion::is_stale_deletion_lease(&error) => Ok(false), + Err(error) => Err(error.into()), + } +} + +async fn execute_claim( + services: &Services, + mode: LoopMode, + mut claim: ClaimedDeletion, + shutdown: &CancellationToken, +) -> Result { + let token = claim.lease.clone(); + loop { + if shutdown.is_cancelled() { + stop_claim_executor(services, mode, &token).await?; + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); + } + services + .store + .heartbeat(&token, mode.as_str(), DEFAULT_LEASE_DURATION, false) + .await?; + let stage_result = run_stage_with_heartbeat(services, mode, &claim, shutdown).await; + match stage_result { + StageOutcome::Completed => {} + StageOutcome::Shutdown => { + stop_claim_executor(services, mode, &token).await?; + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); + } + StageOutcome::Failed(error) => { + let request = services.store.get(token.request_id).await?; + if request.lease_owner.as_deref() != Some(&token.owner) + || request.lease_generation != token.generation + { + return Ok(run_output(request)); + } + if !record_stage_failure(services, &token, claim.request.stage, &error).await? { + // Ownership expired between the stage failure and durable + // error recording. A successor now owns retry/block policy. + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); + } + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); + } + } + let request = services.store.get(token.request_id).await?; + if request.stage == DeletionStage::RetentionPending || request.blocked_reason.is_some() { + return Ok(run_output(request)); + } + claim.request = request.clone(); + claim.lease.fence_generation = request.fence_generation; + } +} + +enum StageOutcome { + Completed, + Shutdown, + Failed(anyhow::Error), +} + +async fn await_stage( + stage: F, + shutdown: &CancellationToken, + heartbeat_error: &CancellationToken, +) -> StageOutcome +where + F: std::future::Future>, +{ + tokio::select! { + biased; + _ = shutdown.cancelled() => StageOutcome::Shutdown, + _ = heartbeat_error.cancelled() => StageOutcome::Failed(DeletionLeaseLost.into()), + result = stage => match result { + Ok(()) => StageOutcome::Completed, + Err(error) => StageOutcome::Failed(error), + }, + } +} + +async fn run_stage_with_heartbeat( + services: &Services, + mode: LoopMode, + claim: &ClaimedDeletion, + shutdown: &CancellationToken, +) -> StageOutcome { + let heartbeat_services = services.clone(); + let heartbeat_token = claim.lease.clone(); + let heartbeat_mode = mode.as_str(); + let heartbeat_shutdown = CancellationToken::new(); + let heartbeat_cancel = heartbeat_shutdown.clone(); + let heartbeat_error = CancellationToken::new(); + let heartbeat_error_signal = heartbeat_error.clone(); + let heartbeat = tokio::spawn(async move { + let mut interval = tokio::time::interval(heartbeat_interval()); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + tokio::select! { + _ = heartbeat_cancel.cancelled() => return, + _ = interval.tick() => { + if heartbeat_services + .store + .heartbeat( + &heartbeat_token, + heartbeat_mode, + DEFAULT_LEASE_DURATION, + false, + ) + .await + .is_err() + { + heartbeat_error_signal.cancel(); + return; + } + } + } + } + }); + + let stage = await_stage( + execute_stage(services, claim, &heartbeat_error), + shutdown, + &heartbeat_error, + ) + .await; + heartbeat_shutdown.cancel(); + match heartbeat.await { + Ok(()) => stage, + Err(error) => { + StageOutcome::Failed(anyhow::anyhow!("deletion heartbeat task failed: {error}")) + } + } +} + +async fn run_guarded_external_step( + services: &Services, + token: &LeaseToken, + stage: DeletionStage, + heartbeat_lost: &CancellationToken, + operation: F, +) -> Result +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + services.store.verify_execution_token(token, stage).await?; + let result = tokio::select! { + biased; + _ = heartbeat_lost.cancelled() => { + return Err(DeletionLeaseLost.into()); + } + result = operation() => result, + }; + let output = result?; + services.store.verify_execution_token(token, stage).await?; + Ok(output) +} + +async fn execute_stage( + services: &Services, + claim: &ClaimedDeletion, + heartbeat_lost: &CancellationToken, +) -> Result<()> { + let request = &claim.request; + let token = token_with_current_fence(&claim.lease, request); + if matches!( + request.stage, + DeletionStage::Approved + | DeletionStage::Fenced + | DeletionStage::Drained + | DeletionStage::BindingsRemoved + | DeletionStage::PostgresPurged + | DeletionStage::CachePurged + | DeletionStage::LogicallyVerified + ) { + validate_frozen_inventory(request)?; + } + match request.stage { + DeletionStage::Approved => { + // Approval binds immutable catalog + community-prefix ownership. + // Live row counts and tenant binding keys are deliberately not + // equality-bound until the durable fence closes all writers. + let live_schema = services + .store + .inventory_schema(request.community_id) + .await?; + let frozen = validate_frozen_inventory(request)?; + if live_schema.scoped_tables != frozen.schema.scoped_tables + || live_schema.fenced_tables != frozen.schema.fenced_tables + { + return Err(permanent( + "approved structural catalog drifted before fencing", + )); + } + services.store.begin_quiescing(&token).await?; + match services.store.fence(&token).await { + Ok(_) => {} + Err(buzz_db::DbError::ServingWritesNotDrained { + active_count, + operations, + .. + }) => { + return Err(transient(format!( + "serving writes not drained before fence: count={active_count}, operations={operations:?}" + ))); + } + Err(error) => return Err(error.into()), + } + } + DeletionStage::Fenced => { + services + .store + .verify_execution_token(&token, DeletionStage::Fenced) + .await?; + let disconnect = tokio::select! { + biased; + _ = heartbeat_lost.cancelled() => Err(DeletionLeaseLost.into()), + result = publish_disconnect_community(&services.redis, request.community_id) => result, + }; + disconnect?; + services + .store + .verify_execution_token(&token, DeletionStage::Fenced) + .await?; + if !services + .store + .serving_writes_drained(request.community_id) + .await? + { + return Err(transient("serving writes have not drained")); + } + // Freeze the destructive enumeration only after the fence closed + // new writers AND every admitted serving write drained: the + // post-drain listing is the final storage state, so no tenant key + // can appear after the freeze. + let destructive = match request.destructive_storage_manifest.clone() { + Some(value) => serde_json::from_value(value)?, + None => { + freeze_destructive_manifest(services, request, &token, heartbeat_lost).await? + } + }; + validate_storage_ownership(request, &destructive)?; + services.store.mark_drained(&token).await?; + } + DeletionStage::Drained => { + let storage: StorageManifest = serde_json::from_value( + request + .destructive_storage_manifest + .clone() + .context("request has no post-fence destructive storage manifest")?, + )?; + validate_storage_ownership(request, &storage)?; + // Resume = first unstamped chunk. Bulk deletes are idempotent + // (missing keys report as deleted), so re-deleting a chunk whose + // stamp was lost to a crash is safe. + let mut removed: u64 = 0; + let mut already_missing: u64 = 0; + while let Some(chunk) = services.store.next_pending_manifest_chunk(&token).await? { + let outcome = run_guarded_external_step( + services, + &token, + DeletionStage::Drained, + heartbeat_lost, + || async { Ok(services.media.delete_objects(&chunk.keys).await?) }, + ) + .await?; + if !outcome.versioned_keys.is_empty() { + return Err(permanent(format!( + "bulk delete produced version artifacts; bucket versioning blocks \ + deletion: {}", + outcome.versioned_keys.join(",") + ))); + } + if !outcome.failed.is_empty() { + let (key, code, message) = &outcome.failed[0]; + return Err(transient(format!( + "bulk delete failed for {} key(s); first: {key}: {code}: {message}", + outcome.failed.len() + ))); + } + let acknowledged = outcome.deleted.saturating_add(outcome.already_missing); + if acknowledged != chunk.keys.len() as u64 { + return Err(transient(format!( + "bulk delete acknowledged {acknowledged} of {} keys in chunk {}", + chunk.keys.len(), + chunk.chunk_no + ))); + } + removed += outcome.deleted; + already_missing += outcome.already_missing; + services + .store + .mark_manifest_chunk_deleted( + &token, + chunk.chunk_no, + serde_json::json!({ + "prefix": chunk.prefix, + "keys": chunk.keys.len(), + "deleted": outcome.deleted, + "already_missing": outcome.already_missing, + }), + ) + .await?; + } + let frozen_keys: u64 = storage + .prefixes + .iter() + .map(|prefix| prefix.object_count) + .sum(); + services + .store + .mark_bindings_removed( + &token, + serde_json::json!({ + "deleted_keys": frozen_keys, + "removed_now": removed, + "already_missing": already_missing, + }), + ) + .await?; + } + DeletionStage::BindingsRemoved => { + services.store.purge_postgres(&token).await?; + } + DeletionStage::PostgresPurged => { + services + .store + .verify_execution_token(&token, DeletionStage::PostgresPurged) + .await?; + let deleted = purge_redis_namespace(&services.redis, request.community_id).await?; + services + .store + .verify_execution_token(&token, DeletionStage::PostgresPurged) + .await?; + services + .store + .mark_cache_purged(&token, serde_json::json!({"deleted_keys": deleted})) + .await?; + } + DeletionStage::CachePurged => { + services + .store + .verify_postgres_logically_deleted(&token) + .await?; + verify_storage_absence(services, request).await?; + verify_redis_absence(&services.redis, request.community_id).await?; + services + .store + .mark_logically_verified( + &token, + serde_json::json!({"postgres": true, "object_store": true, "redis": true}), + ) + .await?; + } + DeletionStage::LogicallyVerified => { + validate_frozen_inventory(request)?; + services + .store + .mark_retention_pending( + &token, + serde_json::json!({ + "policy": "member-erasure and fleet-wide shared-CAS GC are out of V1 scope" + }), + ) + .await?; + } + DeletionStage::Submitted | DeletionStage::Inventoried => { + anyhow::bail!("request has not crossed the explicit approval boundary") + } + DeletionStage::RetentionPending | DeletionStage::Aborted => {} + } + Ok(()) +} + +fn token_with_current_fence(token: &LeaseToken, request: &DeletionRequest) -> LeaseToken { + LeaseToken { + fence_generation: request.fence_generation, + ..token.clone() + } +} + +/// Prove logical absence by listing each tenant prefix and requiring it +/// empty — O(1) requests per prefix, independent of fleet size. +async fn verify_storage_absence(services: &Services, request: &DeletionRequest) -> Result<()> { + for prefix in tenant_prefixes(*request.community_id.as_uuid()) { + let page = services.media.list_prefix_page(&prefix, None, 1).await?; + if let Some((key, _)) = page.objects.first() { + return Err(transient(format!( + "logical verification found a live target object binding: {key}" + ))); + } + } + Ok(()) +} + +async fn publish_disconnect_community( + pool: &deadpool_redis::Pool, + community: buzz_core::CommunityId, +) -> Result<()> { + let mut connection = pool.get().await?; + let channel = format!("buzz:{community}:conn-control"); + let _: u64 = redis::cmd("PUBLISH") + .arg(channel) + .arg(r#"{"op":"DisconnectCommunity"}"#) + .query_async(&mut *connection) + .await?; + Ok(()) +} + +async fn purge_redis_namespace( + pool: &deadpool_redis::Pool, + community: buzz_core::CommunityId, +) -> Result { + let mut connection = pool.get().await?; + let pattern = format!("buzz:{community}:*"); + let mut cursor = 0u64; + let mut deleted = 0u64; + loop { + let (next, keys): (u64, Vec) = redis::cmd("SCAN") + .arg(cursor) + .arg("MATCH") + .arg(&pattern) + .arg("COUNT") + .arg(1000) + .query_async(&mut *connection) + .await?; + if !keys.is_empty() { + let count: u64 = redis::cmd("UNLINK") + .arg(&keys) + .query_async(&mut *connection) + .await?; + deleted = deleted.saturating_add(count); + } + if next == 0 { + break; + } + cursor = next; + } + Ok(deleted) +} + +fn scan_proves_absence(pages: &[(u64, Vec)]) -> bool { + pages.last().is_some_and(|(cursor, _)| *cursor == 0) + && pages.iter().all(|(_, keys)| keys.is_empty()) +} + +async fn scan_redis_namespace( + connection: &mut deadpool_redis::Connection, + pattern: &str, +) -> Result)>> { + let mut cursor = 0u64; + let mut pages = Vec::new(); + loop { + let page: (u64, Vec) = redis::cmd("SCAN") + .arg(cursor) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(1000) + .query_async(&mut **connection) + .await?; + cursor = page.0; + pages.push(page); + if cursor == 0 { + return Ok(pages); + } + } +} + +async fn verify_redis_absence( + pool: &deadpool_redis::Pool, + community: buzz_core::CommunityId, +) -> Result<()> { + let mut connection = pool.get().await?; + let pattern = format!("buzz:{community}:*"); + // SCAN is weakly consistent. Two complete empty passes ensure a cursor + // rollover or concurrent expiry cannot make one sparse pass look absent. + let first = scan_redis_namespace(&mut connection, &pattern).await?; + let second = scan_redis_namespace(&mut connection, &pattern).await?; + if scan_proves_absence(&first) && scan_proves_absence(&second) { + Ok(()) + } else { + Err(transient( + "logical verification found a Redis namespace key", + )) + } +} + +fn sweep_object_cap() -> u64 { + std::env::var("BUZZ_DELETION_SWEEP_MAX_OBJECTS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_SWEEP_OBJECT_CAP) +} + +fn manifest_chunk_keys() -> usize { + std::env::var("BUZZ_DELETION_MANIFEST_CHUNK_KEYS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .map(|value| value.min(100_000)) + .unwrap_or(DEFAULT_MANIFEST_CHUNK_KEYS) +} + +fn default_executor_id() -> String { + let hostname = std::env::var("HOSTNAME").unwrap_or_else(|_| "buzz-admin".to_string()); + format!("{hostname}:{}", std::process::id()) +} + +fn shutdown_token() -> CancellationToken { + let token = CancellationToken::new(); + let signal = token.clone(); + tokio::spawn(async move { + #[cfg(unix)] + { + use tokio::signal::unix::{signal as unix_signal, SignalKind}; + if let Ok(mut terminate) = unix_signal(SignalKind::terminate()) { + tokio::select! { + _ = tokio::signal::ctrl_c() => {}, + _ = terminate.recv() => {}, + } + } else { + let _ = tokio::signal::ctrl_c().await; + } + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } + signal.cancel(); + }); + token +} + +fn run_output(request: DeletionRequest) -> RunOutput { + RunOutput { + request_id: request.id, + stage: request.stage, + retry_count: request.retry_count, + last_error: request.last_error, + next_attempt_at: request.next_attempt_at, + blocked_reason: request.blocked_reason, + } +} + +fn print_json(value: &impl Serialize) -> Result<()> { + println!("{}", serde_json::to_string_pretty(value)?); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn submit_host_prefers_explicit_host() { + assert_eq!( + resolve_submit_host(Some(" community.example "), Some("wss://ignored.example")) + .expect("explicit host"), + "community.example" + ); + } + + #[test] + fn submit_host_derives_from_relay_url() { + assert_eq!( + resolve_submit_host(None, Some("wss://relay.example:8443/path")) + .expect("relay URL host"), + "relay.example:8443" + ); + } + + #[test] + fn submit_host_requires_an_explicit_source() { + for relay_url in [None, Some(""), Some(" ")] { + let error = resolve_submit_host(None, relay_url).expect_err("missing host must fail"); + assert!(error.to_string().contains("pass --host or set RELAY_URL")); + } + } + + #[test] + fn submit_host_rejects_empty_or_invalid_values() { + assert!(resolve_submit_host(Some(" "), Some("wss://relay.example")).is_err()); + assert!(resolve_submit_host(None, Some("not a URL")).is_err()); + } + + fn empty_storage_manifest(community: buzz_core::CommunityId) -> StorageManifest { + StorageManifest { + version: 4, + prefixes: tenant_prefixes(*community.as_uuid()) + .into_iter() + .map(|prefix| PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + }) + .collect(), + } + } + + async fn claimed_test_deletion(prefix: &str) -> (Db, Services, ClaimedDeletion) { + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .expect("BUZZ_TEST_DATABASE_URL or DATABASE_URL is required"); + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect deletion engine test DB"); + let db = Db::from_pool(pool); + db.migrate().await.expect("migrate deletion engine test DB"); + let store = db.deletion_store(); + let host = format!("{prefix}-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("create deletion engine test community"); + let request = store + .submit(&host, "test", None) + .await + .expect("submit deletion request"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community.id) + .await + .expect("inventory schema"), + storage: empty_storage_manifest(community.id), + }; + store + .freeze_inventory(request.id, &inventory) + .await + .expect("freeze deletion inventory"); + store + .approve(request.id, "test", None) + .await + .expect("approve deletion request"); + let claim = store + .claim_specific(request.id, "test-executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim deletion request") + .expect("runnable deletion request"); + let services = Services { + store, + media: Arc::new( + MediaStorage::new(&buzz_media::MediaConfig { + s3_endpoint: "http://127.0.0.1:1".to_string(), + s3_access_key: "unused".to_string(), + s3_secret_key: "unused".to_string(), + s3_bucket: "unused".to_string(), + s3_region: "us-east-1".to_string(), + s3_addressing_style: buzz_media::S3AddressingStyle::Path, + max_image_bytes: 1, + max_gif_bytes: 1, + max_video_bytes: 1, + max_file_bytes: 1, + public_base_url: "http://localhost/media".to_string(), + upload_records_enabled: false, + upload_ip_header: None, + upload_port_header: None, + }) + .expect("construct unused media service"), + ), + redis: deadpool_redis::Config::from_url("redis://127.0.0.1:1") + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .expect("construct unused Redis pool"), + }; + (db, services, claim) + } + + fn deletion_test_media_storage() -> Arc { + let endpoint = std::env::var("BUZZ_TEST_S3_ENDPOINT") + .or_else(|_| std::env::var("BUZZ_S3_ENDPOINT")) + .expect("BUZZ_TEST_S3_ENDPOINT or BUZZ_S3_ENDPOINT is required"); + let access_key = std::env::var("BUZZ_TEST_S3_ACCESS_KEY") + .or_else(|_| std::env::var("BUZZ_S3_ACCESS_KEY")) + .expect("BUZZ_TEST_S3_ACCESS_KEY or BUZZ_S3_ACCESS_KEY is required"); + let secret_key = std::env::var("BUZZ_TEST_S3_SECRET_KEY") + .or_else(|_| std::env::var("BUZZ_S3_SECRET_KEY")) + .expect("BUZZ_TEST_S3_SECRET_KEY or BUZZ_S3_SECRET_KEY is required"); + let bucket = std::env::var("BUZZ_TEST_S3_BUCKET") + .or_else(|_| std::env::var("BUZZ_S3_BUCKET")) + .expect("BUZZ_TEST_S3_BUCKET or BUZZ_S3_BUCKET is required"); + Arc::new( + MediaStorage::new(&buzz_media::MediaConfig { + s3_endpoint: endpoint, + s3_access_key: access_key, + s3_secret_key: secret_key, + s3_bucket: bucket, + s3_region: std::env::var("BUZZ_TEST_S3_REGION") + .or_else(|_| std::env::var("BUZZ_S3_REGION")) + .unwrap_or_else(|_| "us-east-1".to_string()), + s3_addressing_style: buzz_media::S3AddressingStyle::Path, + max_image_bytes: 1, + max_gif_bytes: 1, + max_video_bytes: 1, + max_file_bytes: 1, + public_base_url: "http://localhost/media".to_string(), + upload_records_enabled: false, + upload_ip_header: None, + upload_port_header: None, + }) + .expect("construct deletion test media service"), + ) + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn approved_stage_allows_post_inventory_row_churn_before_fencing() { + let (db, services, claim) = claimed_test_deletion("deletion-row-churn").await; + let frozen: FrozenInventory = serde_json::from_value( + claim + .request + .inventory_manifest + .clone() + .expect("frozen inventory"), + ) + .expect("decode frozen inventory"); + + db.add_to_allowlist(claim.request.community_id, &[0x41; 32], &[0x42; 32], None) + .await + .expect("post-inventory serving write"); + let live = services + .store + .inventory_schema(claim.request.community_id) + .await + .expect("live inventory after serving churn"); + assert_eq!(live.scoped_tables, frozen.schema.scoped_tables); + assert_eq!(live.fenced_tables, frozen.schema.fenced_tables); + assert_eq!( + live.row_counts["pubkey_allowlist"], + frozen.schema.row_counts["pubkey_allowlist"] + 1 + ); + + execute_stage(&services, &claim, &CancellationToken::new()) + .await + .expect("row-count churn must not fail structural revalidation"); + let fenced = services + .store + .get(claim.request.id) + .await + .expect("load fenced request"); + assert_eq!(fenced.stage, DeletionStage::Fenced); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn frozen_inventory_digest_and_storage_ownership_fail_closed() { + let (_, _, claim) = claimed_test_deletion("deletion-integrity").await; + assert!(validate_frozen_inventory(&claim.request).is_ok()); + + let mut digest_tampered = claim.request.clone(); + digest_tampered.inventory_manifest = Some(serde_json::json!({ + "schema": {"scoped_tables": [], "row_counts": {}, "fenced_tables": []}, + "storage": {"version": 4, "prefixes": []} + })); + assert!(validate_frozen_inventory(&digest_tampered).is_err()); + + // A manifest scoped to another community's prefixes is never the + // deletion target's, even when internally valid. + let foreign_manifest = + empty_storage_manifest(buzz_core::CommunityId::from_uuid(Uuid::new_v4())); + assert!(validate_storage_ownership(&claim.request, &foreign_manifest).is_err()); + } + + /// The non-atomic boundary under test: S3 committed a chunk's deletes, + /// then the worker died before the chunk stamp. Resume must re-delete the + /// chunk (missing keys report as deleted — idempotent), stamp it, and + /// finish the stage. + #[tokio::test] + #[ignore = "requires Postgres and S3-compatible storage"] + async fn drained_stage_resumes_chunk_deleted_before_stamp() { + let (_, mut services, claim) = claimed_test_deletion("deletion-chunk-resume").await; + services.media = deletion_test_media_storage(); + let community = claim.request.community_id; + let meta_prefix = format!("_meta/{community}/"); + let keys = vec![ + format!("{meta_prefix}{}.json", "a".repeat(64)), + format!("{meta_prefix}{}.json", "b".repeat(64)), + ]; + for key in &keys { + services + .media + .put(key, b"chunk-resume", "application/json") + .await + .expect("seed object"); + } + + services + .store + .begin_quiescing(&claim.lease) + .await + .expect("quiesce"); + let generation = services.store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease.clone() + }; + // Two single-key chunks so resume order is observable. + services + .store + .append_manifest_key_chunk(&token, 0, &meta_prefix, &keys[..1]) + .await + .expect("append chunk 0"); + services + .store + .append_manifest_key_chunk(&token, 1, &meta_prefix, &keys[1..]) + .await + .expect("append chunk 1"); + let mut digest = KeyStreamDigest::new(); + for key in &keys { + digest.fold(key).expect("fold key"); + } + let (keys_digest, object_count) = digest.finish(); + let mut storage = empty_storage_manifest(community); + storage.prefixes[0] = PrefixManifest { + prefix: meta_prefix.clone(), + object_count, + total_bytes: keys.len() as u64 * "chunk-resume".len() as u64, + keys_digest, + }; + services + .store + .freeze_destructive_storage_manifest(&token, &storage) + .await + .expect("freeze chunked manifest"); + services.store.mark_drained(&token).await.expect("drained"); + + // Simulate the crash window: chunk 0's key is already gone from S3 + // but the chunk was never stamped. + services + .media + .delete(&keys[0]) + .await + .expect("simulate committed delete before stamp"); + + let resumed = ClaimedDeletion { + request: services + .store + .get(token.request_id) + .await + .expect("reload drained request"), + lease: claim.lease, + }; + execute_stage(&services, &resumed, &CancellationToken::new()) + .await + .expect("Drained stage resumes at the unstamped chunk"); + + assert_eq!( + services + .store + .manifest_chunk_progress(token.request_id) + .await + .expect("chunk progress"), + (2, 2) + ); + assert_eq!( + services.store.get(token.request_id).await.unwrap().stage, + DeletionStage::BindingsRemoved + ); + for key in &keys { + assert!( + !services.media.head(key).await.expect("verify absence"), + "tenant binding {key} must be gone" + ); + } + } + + #[test] + fn permanent_failures_are_typed_not_string_classified() { + let permanent_error = permanent("catalog drift"); + let transient_error = transient("temporary catalog service reset"); + let nested = permanent_source(anyhow::anyhow!("schema mismatch")).context("outer"); + let db_permanent = anyhow::Error::from(buzz_db::DbError::DeletionSafety( + "typed catalog drift".to_string(), + )); + let db_transient = anyhow::Error::from(buzz_db::DbError::Sqlx(sqlx::Error::PoolTimedOut)); + assert!(is_permanent_error(&permanent_error)); + assert!(is_permanent_error(&nested)); + assert!(is_permanent_error(&db_permanent)); + assert!(!is_permanent_error(&transient_error)); + assert!(!is_permanent_error(&db_transient)); + } + + #[test] + fn deletion_configuration_requires_every_destructive_dependency() { + let variable = format!("BUZZ_DELETION_REQUIRED_TEST_{}", Uuid::new_v4().simple()); + assert!(required_env(&variable).is_err()); + std::env::set_var(&variable, " "); + assert!(required_env(&variable).is_err()); + std::env::set_var(&variable, "configured"); + assert_eq!( + required_env(&variable).expect("configured environment variable"), + "configured" + ); + std::env::remove_var(&variable); + } + + #[test] + fn deletion_s3_region_matches_relay_precedence_and_default() { + assert_eq!(resolve_s3_region(None, None), "us-east-1"); + assert_eq!( + resolve_s3_region(Some(" ".to_string()), None), + "us-east-1" + ); + assert_eq!( + resolve_s3_region(Some(" ".to_string()), Some(" us-west-2 ".to_string())), + "us-west-2" + ); + assert_eq!( + resolve_s3_region(None, Some("us-west-2".to_string())), + "us-west-2" + ); + assert_eq!( + resolve_s3_region( + Some("eu-central-1".to_string()), + Some("us-west-2".to_string()) + ), + "eu-central-1" + ); + } + + #[test] + fn redis_absence_requires_terminal_cursor_and_all_pages_empty() { + assert!(!scan_proves_absence(&[(9, Vec::new())])); + assert!(!scan_proves_absence(&[ + (9, Vec::new()), + (0, vec!["buzz:tenant:late".to_string()]), + ])); + assert!(scan_proves_absence(&[(9, Vec::new()), (0, Vec::new())])); + } + + #[tokio::test] + #[ignore = "requires Postgres and S3-compatible storage"] + async fn final_storage_verification_rejects_late_target_binding() { + let (_, mut services, claim) = claimed_test_deletion("deletion-late-binding").await; + services.media = deletion_test_media_storage(); + let community = claim.request.community_id; + let late_key = format!("_meta/{community}/{}.json", "a".repeat(64)); + services + .media + .put(&late_key, b"late", "application/json") + .await + .expect("seed late binding"); + let error = verify_storage_absence(&services, &claim.request) + .await + .expect_err("late target binding must fail verification"); + assert!(format!("{error:#}").contains(&late_key)); + services + .media + .delete(&late_key) + .await + .expect("remove late binding"); + verify_storage_absence(&services, &claim.request) + .await + .expect("empty tenant prefixes verify clean"); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn stale_lease_during_failure_recording_is_lost_ownership() { + let (_, services, claim) = claimed_test_deletion("deletion-stale-record").await; + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .expect("test database URL"); + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect stale-record test DB"); + sqlx::query( + "UPDATE community_deletion_requests SET lease_until = now() - interval '1 second' WHERE id = $1", + ) + .bind(claim.request.id) + .execute(&pool) + .await + .expect("expire claim"); + let recorded = record_stage_failure( + &services, + &claim.lease, + claim.request.stage, + &transient("test failure"), + ) + .await + .expect("stale ownership is not fatal"); + assert!(!recorded); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn serving_guard_heartbeats_long_operation_through_quiescing() { + let (db, services, claim) = claimed_test_deletion("serving-guard-quiesce").await; + let community = claim.request.community_id; + let guard = acquire_serving_write_with_heartbeat( + &db, + community, + "test_quiesce", + Duration::from_millis(10), + ) + .await + .expect("serving guard"); + + services + .store + .begin_quiescing(&claim.lease) + .await + .expect("quiesce"); + assert!( + services + .store + .acquire_serving_write_lease( + community, + "late_external", + "late-owner", + DEFAULT_LEASE_DURATION, + ) + .await + .is_err(), + "quiescing must reject newly admitted work" + ); + + let completed = Arc::new(AtomicBool::new(false)); + let operation_completed = Arc::clone(&completed); + let result = guard + .protect(async move { + tokio::time::sleep(Duration::from_millis(75)).await; + operation_completed.store(true, Ordering::Relaxed); + }) + .await; + assert!( + !guard.lost().is_cancelled(), + "quiescing must not cancel the admitted lease heartbeat" + ); + result.expect("admitted operation survives quiescing heartbeats"); + assert!(completed.load(Ordering::Relaxed)); + assert!(matches!( + services.store.fence(&claim.lease).await, + Err(buzz_db::DbError::ServingWritesNotDrained { + active_count: 1, + .. + }) + )); + + guard.finish().await.expect("release serving guard"); + assert_eq!(services.store.fence(&claim.lease).await.expect("fence"), 1); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn serving_guard_cancels_protected_operation_when_heartbeat_is_lost() { + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .expect("BUZZ_TEST_DATABASE_URL or DATABASE_URL is required"); + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect serving guard test DB"); + let db = Db::from_pool(pool.clone()); + db.migrate().await.expect("migrate serving guard test DB"); + let community = db + .ensure_configured_community(&format!( + "serving-guard-{}.example", + Uuid::new_v4().simple() + )) + .await + .expect("create test community") + .id; + let guard = acquire_serving_write_with_heartbeat( + &db, + community, + "test_cancel", + Duration::from_millis(10), + ) + .await + .expect("serving guard"); + sqlx::query("DELETE FROM community_serving_write_leases WHERE community_id = $1") + .bind(community.as_uuid()) + .execute(&pool) + .await + .expect("force heartbeat failure"); + let completed = Arc::new(AtomicBool::new(false)); + let operation_completed = Arc::clone(&completed); + let result = guard + .protect(async move { + tokio::time::sleep(Duration::from_secs(1)).await; + operation_completed.store(true, Ordering::Relaxed); + }) + .await; + assert!(result.is_err(), "lease loss must reject the operation"); + assert!( + !completed.load(Ordering::Relaxed), + "lease loss must cancel the protected operation future" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn guarded_external_step_rejects_preexisting_heartbeat_loss_without_polling_operation() { + let (_, services, claim) = claimed_test_deletion("deletion-heartbeat").await; + let heartbeat_lost = CancellationToken::new(); + heartbeat_lost.cancel(); + let polled = Arc::new(AtomicBool::new(false)); + let operation_polled = Arc::clone(&polled); + let result = run_guarded_external_step( + &services, + &claim.lease, + DeletionStage::Approved, + &heartbeat_lost, + || async move { + operation_polled.store(true, Ordering::Relaxed); + Ok(()) + }, + ) + .await; + assert!(result.is_err(), "heartbeat loss must abort the side effect"); + assert!( + result + .expect_err("heartbeat loss error") + .downcast_ref::() + .is_some(), + "heartbeat loss must stay typed" + ); + assert!( + !polled.load(Ordering::Relaxed), + "a pre-cancelled heartbeat must win before polling the operation" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn shutdown_during_stage_releases_claim_without_recording_retry() { + let (_, services, claim) = claimed_test_deletion("deletion-shutdown").await; + let request_id = claim.request.id; + let retry_count = claim.request.retry_count; + let shutdown = CancellationToken::new(); + let cancel = shutdown.clone(); + let services_for_run = services.clone(); + let executor = tokio::spawn(async move { + execute_claim(&services_for_run, LoopMode::Drain, claim, &shutdown).await + }); + tokio::time::sleep(Duration::from_millis(10)).await; + cancel.cancel(); + let output = tokio::time::timeout(Duration::from_secs(2), executor) + .await + .expect("shutdown must cancel the active stage") + .expect("deletion executor task") + .expect("graceful deletion executor shutdown"); + let request = services + .store + .get(request_id) + .await + .expect("load deletion request after shutdown"); + assert_eq!(output.stage, DeletionStage::Approved); + assert_eq!(request.stage, DeletionStage::Approved); + assert_eq!(request.retry_count, retry_count); + assert!(request.last_error.is_none()); + assert!(request.lease_owner.is_none()); + assert!(request.lease_until.is_none()); + } + + #[tokio::test] + async fn stage_wait_treats_shutdown_as_control_flow() { + let shutdown = CancellationToken::new(); + shutdown.cancel(); + let heartbeat_lost = CancellationToken::new(); + let outcome = await_stage( + std::future::pending::>(), + &shutdown, + &heartbeat_lost, + ) + .await; + assert!(matches!(outcome, StageOutcome::Shutdown)); + } + + #[tokio::test] + async fn stage_wait_prioritizes_heartbeat_loss_over_a_ready_operation() { + let shutdown = CancellationToken::new(); + let heartbeat_lost = CancellationToken::new(); + heartbeat_lost.cancel(); + let outcome = await_stage(async { Ok(()) }, &shutdown, &heartbeat_lost).await; + match outcome { + StageOutcome::Failed(error) => assert!( + error.downcast_ref::().is_some(), + "heartbeat loss must stay typed" + ), + StageOutcome::Completed | StageOutcome::Shutdown => { + panic!("preexisting heartbeat loss must win") + } + } + } +} diff --git a/crates/buzz-media/src/bucket_index.rs b/crates/buzz-media/src/bucket_index.rs index bb83dc517fa..6c78c2e0a16 100644 --- a/crates/buzz-media/src/bucket_index.rs +++ b/crates/buzz-media/src/bucket_index.rs @@ -753,3 +753,258 @@ mod tests { ); } } + +/// The exact community-scoped listing prefixes owned by one tenant, in +/// ascending key order: media sidecars, upload records, and Git repository +/// pointers. Every tenant-owned binding lives under one of these; shared +/// immutable CAS/thumb/probe data is deliberately outside them (fleet-wide +/// physical GC is a separate retention phase). +pub fn tenant_prefixes(community: Uuid) -> [String; 3] { + [ + format!("_meta/{community}/"), + format!("_uploads/{community}/"), + format!("repos/{community}/"), + ] +} + +/// Whether one bucket key is a tenant-owned binding of `community` in the +/// exact writer taxonomy: a media sidecar, an upload record, or a Git +/// repository pointer. A malformed key under a tenant prefix is NOT owned — +/// deletion fails closed on shapes this binary did not write. +pub fn is_tenant_owned_key(community: Uuid, key: &str) -> bool { + match classify_key(key) { + KeyClass::Sidecar { + community: owner, .. + } + | KeyClass::Auxiliary { + community: owner, .. + } => owner == community, + KeyClass::Unknown => git_pointer_community(key) == Some(community), + KeyClass::Blob { .. } | KeyClass::Thumb { .. } => false, + } +} + +/// Whether one bucket key belongs to the fleet's known writer taxonomy: +/// blob/thumb/sidecar/upload shapes, any community's Git pointer, shared Git +/// CAS data, or a `probe/` connectivity key. +pub fn is_known_fleet_key(key: &str) -> bool { + !matches!(classify_key(key), KeyClass::Unknown) + || git_pointer_community(key).is_some() + || is_known_git_shared_key(key) + || key.starts_with("probe/") +} + +/// Durable outcome of one fleet-wide taxonomy sweep. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TaxonomySweepOutcome { + /// Total objects listed. + pub listed_objects: u64, + /// Exact count of keys outside the known writer taxonomy. + pub unknown_object_count: u64, + /// Bounded sample of unknown keys, in listing order. + pub unknown_key_sample: Vec, +} + +/// Fold an entire paginated bucket listing into the fleet taxonomy outcome. +/// +/// Deleting a tenant while the bucket contains a writer shape this binary +/// does not understand is unsafe — but that is a *fleet* invariant, not a +/// per-request one. This sweep records it once; deletion stages then gate on +/// a recent clean sweep instead of re-listing the whole bucket per request. +/// Same pagination/cap contract as [`fold_bucket_listing`]; memory is +/// bounded by `sample_limit`, never the listing size. +pub async fn sweep_bucket_taxonomy( + cap: u64, + sample_limit: usize, + mut fetch_page: F, +) -> Result +where + F: FnMut(Option) -> Fut, + Fut: Future>, +{ + let mut outcome = TaxonomySweepOutcome::default(); + let mut continuation_token = None; + loop { + let page = fetch_page(continuation_token.take()).await?; + outcome.listed_objects += page.objects.len() as u64; + if outcome.listed_objects > cap { + return Err(SweepError::CapExceeded { + seen: outcome.listed_objects, + cap, + }); + } + for (key, _size) in page.objects { + if !is_known_fleet_key(&key) { + outcome.unknown_object_count += 1; + if outcome.unknown_key_sample.len() < sample_limit { + outcome.unknown_key_sample.push(key); + } + } + } + if !page.is_truncated { + break; + } + match page.next_continuation_token { + Some(token) => continuation_token = Some(token), + None => return Err(SweepError::MalformedPage), + } + } + Ok(outcome) +} + +fn git_pointer_community(key: &str) -> Option { + let mut parts = key.split('/'); + if parts.next()? != "repos" { + return None; + } + let community = parse_canonical_uuid(parts.next()?)?; + let owner = parts.next()?; + let repo = parts.next()?; + let pointer = parts.next()?; + if parts.next().is_some() + || owner.len() != 64 + || !owner.bytes().all(|byte| byte.is_ascii_hexdigit()) + || repo.is_empty() + || repo.len() > 64 + || repo.starts_with('.') + || repo.contains("..") + || !repo + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + || pointer != "pointer" + { + return None; + } + Some(community) +} + +fn is_known_git_shared_key(key: &str) -> bool { + ["packs/", "idx/", "manifests/"].iter().any(|prefix| { + key.strip_prefix(prefix).is_some_and(|digest| { + digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) + }) +} + +#[cfg(test)] +mod deletion_taxonomy_tests { + use super::*; + + #[test] + fn tenant_ownership_is_exact_per_community_and_shape() { + let target = Uuid::from_u128(1); + let other = Uuid::from_u128(2); + let sha = "a".repeat(64); + + assert!(is_tenant_owned_key( + target, + &format!("_meta/{target}/{sha}.json") + )); + assert!(is_tenant_owned_key( + target, + &format!("_uploads/{target}/{sha}/01ARZ3NDEKTSV4RRFFQ69G5FAV.json") + )); + assert!(is_tenant_owned_key( + target, + &format!("repos/{target}/{}/repo/pointer", "b".repeat(64)) + )); + // Another tenant's bindings and shared CAS are never owned. + assert!(!is_tenant_owned_key( + target, + &format!("_meta/{other}/{sha}.json") + )); + assert!(!is_tenant_owned_key(target, &format!("{sha}.png"))); + // A malformed key under the tenant's own prefix fails closed. + assert!(!is_tenant_owned_key( + target, + &format!("_meta/{target}/not-a-sidecar") + )); + assert!(!is_tenant_owned_key( + target, + &format!("repos/{target}/stray-file") + )); + } + + #[test] + fn tenant_prefixes_cover_every_owned_shape_and_sort_ascending() { + let community = Uuid::from_u128(7); + let prefixes = tenant_prefixes(community); + assert!(prefixes.windows(2).all(|pair| pair[0] < pair[1])); + let sha = "c".repeat(64); + for key in [ + format!("_meta/{community}/{sha}.json"), + format!("_uploads/{community}/{sha}/01ARZ3NDEKTSV4RRFFQ69G5FAV.json"), + format!("repos/{community}/{}/repo/pointer", "d".repeat(64)), + ] { + assert!( + prefixes + .iter() + .any(|prefix| key.starts_with(prefix.as_str())), + "owned key {key} must live under a tenant prefix" + ); + assert!(is_tenant_owned_key(community, &key)); + } + } + + #[tokio::test] + async fn taxonomy_sweep_counts_all_unknowns_but_bounds_the_sample() { + let community = Uuid::from_u128(1); + let sha = "a".repeat(64); + let known = vec![ + (format!("{sha}.png"), 1), + (format!("{sha}.thumb.jpg"), 1), + (format!("_meta/{community}/{sha}.json"), 1), + (format!("packs/{sha}"), 1), + ( + format!("repos/{community}/{}/repo/pointer", "b".repeat(64)), + 1, + ), + ("probe/cas-123.txt".to_string(), 1), + ]; + let pages = [ + Page { + objects: known, + next_continuation_token: Some("next".to_string()), + is_truncated: true, + }, + Page { + objects: vec![ + ("future-format/one".to_string(), 1), + ("future-format/two".to_string(), 1), + ("future-format/three".to_string(), 1), + ], + next_continuation_token: None, + is_truncated: false, + }, + ]; + let outcome = sweep_bucket_taxonomy(100, 2, |token| { + let page = match token.as_deref() { + None => pages[0].clone(), + Some("next") => pages[1].clone(), + other => panic!("unexpected continuation token {other:?}"), + }; + async move { Ok(page) } + }) + .await + .expect("sweep synthetic listing"); + assert_eq!(outcome.listed_objects, 9); + assert_eq!(outcome.unknown_object_count, 3); + assert_eq!( + outcome.unknown_key_sample, + vec!["future-format/one", "future-format/two"] + ); + } + + #[tokio::test] + async fn taxonomy_sweep_fails_closed_past_the_fleet_cap() { + let result = sweep_bucket_taxonomy(1, 10, |_token| async { + Ok(Page { + objects: vec![("a".to_string(), 1), ("b".to_string(), 1)], + next_continuation_token: None, + is_truncated: false, + }) + }) + .await; + assert!(matches!(result, Err(SweepError::CapExceeded { .. }))); + } +} diff --git a/crates/buzz-media/src/error.rs b/crates/buzz-media/src/error.rs index c3d180402f1..14ce4afe1e8 100644 --- a/crates/buzz-media/src/error.rs +++ b/crates/buzz-media/src/error.rs @@ -54,6 +54,10 @@ pub enum MediaError { InsufficientScope, #[error("relay membership required")] RelayMembershipRequired, + #[error("community writes are fenced")] + CommunityWriteFenced, + #[error("media service temporarily unavailable")] + ServiceUnavailable, #[error("token revoked")] TokenRevoked, #[error("pubkey mismatch")] @@ -138,7 +142,10 @@ impl IntoResponse for MediaError { ) } Self::InsufficientScope => (StatusCode::FORBIDDEN, self.to_string()), - Self::RelayMembershipRequired => (StatusCode::FORBIDDEN, self.to_string()), + Self::RelayMembershipRequired | Self::CommunityWriteFenced => { + (StatusCode::FORBIDDEN, self.to_string()) + } + Self::ServiceUnavailable => (StatusCode::SERVICE_UNAVAILABLE, self.to_string()), Self::UploadRateLimitExceeded | Self::UploadConcurrencyLimitReached => { (StatusCode::TOO_MANY_REQUESTS, self.to_string()) } @@ -164,6 +171,21 @@ impl IntoResponse for MediaError { mod tests { use super::*; + #[test] + fn serving_backend_failures_map_to_5xx_but_fences_remain_403() { + for error in [ + MediaError::ServiceUnavailable, + MediaError::Internal, + MediaError::StorageError("backend".to_string()), + ] { + assert!(error.into_response().status().is_server_error()); + } + assert_eq!( + MediaError::CommunityWriteFenced.into_response().status(), + StatusCode::FORBIDDEN + ); + } + #[test] fn unsupported_media_maps_to_415() { for error in [ diff --git a/crates/buzz-media/src/lib.rs b/crates/buzz-media/src/lib.rs index 67896d4ef22..b2ff12c16e9 100644 --- a/crates/buzz-media/src/lib.rs +++ b/crates/buzz-media/src/lib.rs @@ -14,12 +14,13 @@ pub mod upload_record; pub mod validation; pub use bucket_index::{ - classify_key, fold_bucket_listing, BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass, - Page, SweepError, + classify_key, fold_bucket_listing, is_tenant_owned_key, sweep_bucket_taxonomy, tenant_prefixes, + BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass, Page, SweepError, + TaxonomySweepOutcome, }; pub use config::{MediaConfig, S3AddressingStyle}; pub use error::MediaError; -pub use storage::{BlobHeadMeta, BlobMeta, ByteStream, MediaStorage}; +pub use storage::{BlobHeadMeta, BlobMeta, BulkDeleteOutcome, ByteStream, MediaStorage}; pub use types::BlobDescriptor; pub use upload::{process_file_upload, process_upload, process_video_upload}; pub use upload_record::{ diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index cbf980201fc..0f0aa7af623 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -177,6 +177,47 @@ impl MediaStorage { } } + /// Detect whether the bucket has ever had versioning enabled. + /// + /// rust-s3 exposes no GetBucketVersioning, so this writes and inspects a + /// short-lived fleet probe object instead: versioning-enabled (and + /// versioning-suspended) buckets stamp new writes with a version id. + /// Deletion refuses versioned buckets because bulk deletes without a + /// VersionId would only insert delete markers, not prove logical absence. + pub async fn bucket_versioning_detected(&self) -> Result { + let key = format!("probe/deletion-versioning-{}", uuid::Uuid::new_v4()); + self.put(&key, b"buzz deletion versioning probe", "text/plain") + .await?; + let inspected = self.bucket.head_object(&key).await; + let removed = self.bucket.delete_object(&key).await; + let (head, _) = inspected.map_err(|e| MediaError::StorageError(e.to_string()))?; + removed.map_err(|e| MediaError::StorageError(e.to_string()))?; + Ok(head.version_id.is_some()) + } + + /// Bulk-delete up to one manifest chunk of keys via S3 `DeleteObjects`. + /// + /// Never fails on per-key outcomes: they are folded into + /// [`BulkDeleteOutcome`] so the caller owns retry/fail-closed policy. + /// Historical MinIO releases report already-absent keys as + /// `NoSuchKey`/`NoSuchVersion` errors instead of deleted; both map to + /// `already_missing` to keep checkpointed retry idempotent. + pub async fn delete_objects(&self, keys: &[String]) -> Result { + if keys.is_empty() { + return Ok(BulkDeleteOutcome::default()); + } + let identifiers = keys + .iter() + .map(|key| s3::serde_types::ObjectIdentifier::new(key.clone())) + .collect::>(); + let result = self + .bucket + .delete_objects(identifiers) + .await + .map_err(|e| MediaError::StorageError(e.to_string()))?; + Ok(fold_bulk_delete_result(result)) + } + /// Build the community-scoped sidecar key for a given sha256 (bare hash). /// /// Raw media bytes remain shared content-addressed CAS (`{sha}.{ext}`), but @@ -234,6 +275,11 @@ impl MediaStorage { .map(|m| m.mime_type) } + /// Probe object-store connectivity and bucket access. + pub async fn ping(&self) -> Result<(), MediaError> { + self.list_page(None, 1).await.map(|_| ()) + } + /// One page of a full-bucket listing, for the storage sweep. Wraps /// rust-s3's manual `list_page` (NOT the auto-paginating `list`, which /// has no cap) and converts the result into the storage-agnostic @@ -246,11 +292,28 @@ impl MediaStorage { &self, continuation_token: Option, max_keys: usize, + ) -> Result { + self.list_prefix_page("", continuation_token, max_keys) + .await + } + + /// One page of a prefix-scoped listing. + /// + /// Deletion enumerates the target community's exact key prefixes with + /// this instead of listing the whole fleet bucket: cost stays + /// O(tenant objects) regardless of fleet size. `ListObjectsV2` returns + /// keys in ascending UTF-8 binary order, which callers rely on for + /// streaming key-stream digests. + pub async fn list_prefix_page( + &self, + prefix: &str, + continuation_token: Option, + max_keys: usize, ) -> Result { let (result, _status) = self .bucket .list_page( - String::new(), + prefix.to_string(), None, continuation_token, None, @@ -269,11 +332,93 @@ impl MediaStorage { } } +/// Per-key outcomes of one bulk `DeleteObjects` call. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct BulkDeleteOutcome { + /// Keys the backend reported deleted (S3 reports already-missing keys as + /// deleted too — the API is idempotent by design). + pub deleted: u64, + /// Keys reported absent via legacy MinIO `NoSuchKey`/`NoSuchVersion` + /// per-key errors; equivalent to deleted for retry purposes. + pub already_missing: u64, + /// Keys whose deletion produced a version artifact (delete marker or + /// version id) — evidence of bucket versioning, which deletion must + /// fail closed on. + pub versioned_keys: Vec, + /// Remaining per-key failures as `(key, code, message)`. + pub failed: Vec<(String, String, String)>, +} + +fn fold_bulk_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { + let mut outcome = BulkDeleteOutcome::default(); + for deleted in result.deleted { + if deleted.delete_marker == Some(true) + || deleted.delete_marker_version_id.is_some() + || deleted.version_id.is_some() + { + outcome.versioned_keys.push(deleted.key); + } else { + outcome.deleted += 1; + } + } + for error in result.errors { + if error.code == "NoSuchKey" || error.code == "NoSuchVersion" { + outcome.already_missing += 1; + } else { + outcome.failed.push((error.key, error.code, error.message)); + } + } + outcome +} + #[cfg(test)] mod tests { use super::*; use std::collections::HashMap; + /// The bulk-delete fold is the retry-idempotence contract: legacy MinIO + /// absent-key errors count as success, version artifacts are surfaced for + /// fail-closed handling, and anything else stays a per-key failure. + #[test] + fn bulk_delete_fold_maps_absent_keys_and_version_artifacts() { + use s3::serde_types::{DeleteError, DeleteObjectsResult, DeletedObject}; + let deleted_object = |key: &str, marker: bool| DeletedObject { + key: key.to_string(), + version_id: None, + delete_marker: marker.then_some(true), + delete_marker_version_id: marker.then(|| "v1".to_string()), + }; + let delete_error = |key: &str, code: &str, message: &str| DeleteError { + key: key.to_string(), + code: code.to_string(), + message: message.to_string(), + version_id: None, + }; + let result = DeleteObjectsResult { + deleted: vec![ + deleted_object("plain", false), + deleted_object("marked", true), + ], + errors: vec![ + delete_error("gone", "NoSuchKey", "absent"), + delete_error("gone-version", "NoSuchVersion", "absent"), + delete_error("denied", "AccessDenied", "nope"), + ], + }; + let outcome = fold_bulk_delete_result(result); + assert_eq!(outcome.deleted, 1); + assert_eq!(outcome.already_missing, 2); + assert_eq!(outcome.versioned_keys, vec!["marked".to_string()]); + assert_eq!( + outcome.failed, + vec![( + "denied".to_string(), + "AccessDenied".to_string(), + "nope".to_string() + )] + ); + } + fn tenant(n: u128) -> TenantContext { TenantContext::resolved( CommunityId::from_uuid(uuid::Uuid::from_u128(n)), @@ -351,6 +496,28 @@ mod tests { ); } + #[test] + fn tenant_key_writers_are_covered_by_deletion_taxonomy() { + let ctx = tenant(1); + let community = *ctx.community().as_uuid(); + let sha = "a".repeat(64); + let event_id = "01ARZ3NDEKTSV4RRFFQ69G5FAV"; + let sidecar = MediaStorage::ctx_sidecar_key(&ctx, &sha); + let upload = crate::upload_record::upload_record_key(&ctx, &sha, event_id); + let prefixes = crate::bucket_index::tenant_prefixes(community); + + for key in [sidecar, upload] { + assert!( + prefixes.iter().any(|prefix| key.starts_with(prefix)), + "tenant writer key {key} is outside deletion prefixes" + ); + assert!( + crate::bucket_index::is_tenant_owned_key(community, &key), + "tenant writer key {key} is not recognized by deletion taxonomy" + ); + } + } + #[test] fn sidecar_keys_are_community_scoped() { let a = tenant(1); diff --git a/crates/buzz-relay/Cargo.toml b/crates/buzz-relay/Cargo.toml index 124457fe4e3..deb2e7e16a5 100644 --- a/crates/buzz-relay/Cargo.toml +++ b/crates/buzz-relay/Cargo.toml @@ -20,6 +20,7 @@ buzz-core = { workspace = true } buzz-conformance = { workspace = true } buzz-db = { workspace = true } buzz-datastore-tracing = { workspace = true } +buzz-deletion = { workspace = true } buzz-auth = { workspace = true } buzz-pubsub = { workspace = true } buzz-audit = { workspace = true } diff --git a/crates/buzz-relay/src/api/git/manifest.rs b/crates/buzz-relay/src/api/git/manifest.rs index baf109c1ade..0dfbdb35a4d 100644 --- a/crates/buzz-relay/src/api/git/manifest.rs +++ b/crates/buzz-relay/src/api/git/manifest.rs @@ -474,6 +474,17 @@ mod tests { m.validate().expect("no parent is fine (first push)"); } + #[test] + fn pointer_writer_is_covered_by_deletion_taxonomy() { + let community = CommunityId::from_uuid(uuid::Uuid::from_u128(1)); + let owner = "a".repeat(64); + let key = pointer_key(community, &owner, "repo"); + let prefixes = buzz_media::tenant_prefixes(*community.as_uuid()); + + assert!(prefixes.iter().any(|prefix| key.starts_with(prefix))); + assert!(buzz_media::is_tenant_owned_key(*community.as_uuid(), &key)); + } + #[test] fn pointer_key_strips_dot_git() { let c = CommunityId::from_uuid(uuid::Uuid::from_u128(1)); diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index 53e3f59463c..3b2241046a3 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -1783,6 +1783,21 @@ pub(crate) struct PushContext { pub repo_handle: HydratedRepo, } +#[derive(Default)] +struct FinalizePushHooks { + #[cfg(test)] + post_cas_gate: Option>, + #[cfg(test)] + fail_ref_state_insert: bool, +} + +#[cfg(test)] +#[derive(Default)] +struct PostCasGate { + reached: tokio::sync::Notify, + resume: tokio::sync::Notify, +} + /// Finalize a push request: CAS-commit the new state into the object /// store, derive kind:30618 from the committed manifest, and only then /// build the success response. @@ -1793,6 +1808,17 @@ pub(crate) struct PushContext { /// constructor of a push 2xx, so the seam is structural (not by /// convention). async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { + finalize_push_inner(state, ctx, &FinalizePushHooks::default()).await +} + +async fn finalize_push_inner( + state: &Arc, + ctx: PushContext, + hooks: &FinalizePushHooks, +) -> Response { + #[cfg(not(test))] + let _ = hooks; + // The push fence, part 0 — **a rejected push publishes nothing.** // // `ctx.pack.ok` is false when git aborted the ref updates: either the @@ -1823,10 +1849,41 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { return response; } + // An already-running receive-pack may cross the durable fence after + // request admission. Revalidate immediately before object-store CAS; DB + // trigger fencing alone cannot roll back an S3 pointer mutation. + let serving_write = match buzz_deletion::acquire_serving_write( + &state.db, + ctx.tenant.community(), + "git_publish", + ) + .await + { + Ok(guard) => guard, + Err(error) => { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "push rejected by community deletion fence"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "community writes are fenced", + ) + .into_response(); + } + }; + + if let Err(error) = serving_write.verify().await { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "push lost community serving lease"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "community write lease lost", + ) + .into_response(); + } + // Step 7 (CAS). The PushContext binds `parent_state` (observed at // hydrate) to the CAS predicate here — no re-reading of the pointer - // between hydrate and CAS. - let success = match cas_publish( + // between hydrate and CAS. Observe serving-lease loss throughout the + // potentially long upload/CAS operation, not only at its boundaries. + let publish = cas_publish( &state.git_store, &ctx.tenant, ctx.repo_handle.path(), @@ -1838,72 +1895,87 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { max_pack_bytes: state.config.git_max_pack_bytes, max_repo_bytes: state.config.git_max_repo_bytes, }, - ) - .await - { - Ok(s) => s, - Err(CasError::Conflict { - winner_manifest_key, - .. - }) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo, - winner = %winner_manifest_key, - "push lost CAS race; tempdir dropped, returning 409" - ); - return ( - StatusCode::CONFLICT, - "push superseded by a concurrent writer; pull and retry", - ) - .into_response(); - } - Err(CasError::ManifestInvalid(e)) => { - // 4xx-class: the workspace produced refs/HEAD/oids the - // manifest validator rejects (unsafe refname, malformed oid, - // empty head, malformed parent). Pre-CAS — no pointer was - // written. - warn!( - owner = %ctx.owner, - repo = %ctx.repo, - error = %e, - "push rejected: manifest validation failed" - ); - return ( - StatusCode::BAD_REQUEST, - "push produced invalid manifest state", - ) - .into_response(); - } - Err(CasError::ResourceLimit(e)) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo, - error = %e, - "push rejected: repo exceeds relay resource limits" - ); + ); + let success = match serving_write.protect(publish).await { + Ok(result) => match result { + Ok(s) => s, + Err(CasError::Conflict { + winner_manifest_key, + .. + }) => { + warn!( + owner = %ctx.owner, + repo = %ctx.repo, + winner = %winner_manifest_key, + "push lost CAS race; tempdir dropped, returning 409" + ); + return ( + StatusCode::CONFLICT, + "push superseded by a concurrent writer; pull and retry", + ) + .into_response(); + } + Err(CasError::ManifestInvalid(e)) => { + // 4xx-class: the workspace produced refs/HEAD/oids the + // manifest validator rejects (unsafe refname, malformed oid, + // empty head, malformed parent). Pre-CAS — no pointer was + // written. + warn!( + owner = %ctx.owner, + repo = %ctx.repo, + error = %e, + "push rejected: manifest validation failed" + ); + return ( + StatusCode::BAD_REQUEST, + "push produced invalid manifest state", + ) + .into_response(); + } + Err(CasError::ResourceLimit(e)) => { + warn!( + owner = %ctx.owner, + repo = %ctx.repo, + error = %e, + "push rejected: repo exceeds relay resource limits" + ); + return ( + StatusCode::PAYLOAD_TOO_LARGE, + "repository exceeds relay resource limits", + ) + .into_response(); + } + Err(e) => { + // 5xx-class: ManifestReadFailed (parent corruption), + // Backend, PackCapture. The tempdir drops on scope exit; no + // pointer was written (or, on rare ManifestReadFailed during + // winner-fetch, the winner is already installed and the + // loser's data is unrelated). + error!( + owner = %ctx.owner, + repo = %ctx.repo, + error = %e, + "push failed pre-response" + ); + return (StatusCode::INTERNAL_SERVER_ERROR, "git backend error").into_response(); + } + }, + Err(error) => { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "push lost community serving lease during CAS publish"); return ( - StatusCode::PAYLOAD_TOO_LARGE, - "repository exceeds relay resource limits", + StatusCode::SERVICE_UNAVAILABLE, + "community write lease lost", ) .into_response(); } - Err(e) => { - // 5xx-class: ManifestReadFailed (parent corruption), - // Backend, PackCapture. The tempdir drops on scope exit; no - // pointer was written (or, on rare ManifestReadFailed during - // winner-fetch, the winner is already installed and the - // loser's data is unrelated). - error!( - owner = %ctx.owner, - repo = %ctx.repo, - error = %e, - "push failed pre-response" - ); - return (StatusCode::INTERNAL_SERVER_ERROR, "git backend error").into_response(); - } }; + #[cfg(test)] + if let Some(gate) = &hooks.post_cas_gate { + gate.reached.notify_one(); + gate.resume.notified().await; + } + // Derived after CAS: kind:30618 ref-state event over the *committed* // manifest's refs/head. Spec §Implementation Correspondence: // "kind:30618 is derived after CAS, never the commit." We emit only @@ -1927,7 +1999,7 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { (Some(before), Some(after)) => before != after, _ => true, // first push (parent None) or impossible-shape after key → publish }; - if manifest_changed { + let publication_result: Result<(), String> = if manifest_changed { let inputs = RefStateInputs { repo_id: &ctx.repo_id, head: &success.manifest.head, @@ -1938,11 +2010,23 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { Ok(event) => { // Relay-signed kind:30618 belongs to the same server-resolved // tenant as the git request that committed the pointer. - match state + #[cfg(test)] + let insert_result = if hooks.fail_ref_state_insert { + Err(buzz_db::DbError::InvalidData( + "injected kind:30618 insert failure".to_string(), + )) + } else { + state + .db + .insert_event_with_serving_write_guard(serving_write.lease(), &event, None) + .await + }; + #[cfg(not(test))] + let insert_result = state .db - .insert_event(ctx.tenant.community(), &event, None) - .await - { + .insert_event_with_serving_write_guard(serving_write.lease(), &event, None) + .await; + match insert_result { Ok((stored, true)) => { // Routed through the guarded send path for uniformity; // the access gate no-ops for this globally-scoped @@ -1959,6 +2043,7 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { manifest = %success.manifest_key, "kind:30618 published (derived after CAS)" ); + Ok(()) } Ok((_, false)) => { info!( @@ -1966,26 +2051,41 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { repo = %ctx.repo_id, "kind:30618 deduplicated by relay db" ); + Ok(()) } - Err(e) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo_id, - error = %e, - "kind:30618 insert failed; push remains durable in object store" - ); - } + Err(error) => Err(format!("kind:30618 insert failed: {error}")), } } - Err(e) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo_id, - error = %e, - "kind:30618 build failed; push remains durable in object store" - ); - } + Err(error) => Err(format!("kind:30618 build failed: {error}")), } + } else { + Ok(()) + }; + + // The admitted serving write spans the complete publication attempt. Fence + // acquisition cannot overtake the pointer CAS, durable 30618 insert, or + // local fan-out attempt; only now may the lease be released. + if let Err(error) = serving_write.finish().await { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "failed to release community serving lease after push publication"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "community write lease lost during publication", + ) + .into_response(); + } + if let Err(error) = publication_result { + error!( + owner = %ctx.owner, + repo = %ctx.repo_id, + manifest = %success.manifest_key, + %error, + "push pointer committed but kind:30618 publication failed" + ); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "push committed but ref-state publication failed; retry", + ) + .into_response(); } // Only now — after CAS commit and (optional) 30618 emission — build @@ -2014,12 +2114,14 @@ pub fn git_router(state: Arc) -> Router { #[cfg(test)] mod track_c_tests { use super::*; + use crate::api::git::hydrate::{hydrate_for_write, HydrationOptions}; use crate::api::git::manifest::Manifest; use buzz_core::CommunityId; use nostr::{EventBuilder, Keys, Kind, Tag}; use std::collections::BTreeMap; use std::io::Write; use std::process::Output; + use tempfile::TempDir; fn oid_sha1() -> String { "cb09a769da1c01f458fa6959d4e8eded38fac8d3".to_string() @@ -2160,6 +2262,303 @@ mod track_c_tests { assert!(remote.join("refs/heads/master").exists()); } + async fn run_finalize_git(repo: &Path, args: &[&str]) -> std::process::Output { + let mut command = Command::new("git"); + command.current_dir(repo).args(args); + harden_git_env(&mut command); + let output = command.output().await.expect("spawn git"); + assert!( + output.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + output + } + + async fn finalize_test_state() -> (Arc, sqlx::PgPool) { + const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1 + let mut config = crate::config::Config::from_env().expect("default config loads"); + config.require_relay_membership = false; + config.redis_url = "redis://127.0.0.1:1".to_string(); + config.database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| TEST_DB_URL.to_string()); + let pool = sqlx::PgPool::connect(&config.database_url) + .await + .expect("connect test DB"); + let db = buzz_db::Db::from_pool(pool.clone()); + db.migrate().await.expect("migrate test DB"); + let redis_pool = deadpool_redis::Config::from_url(&config.redis_url) + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .expect("redis pool"); + let pubsub = Arc::new( + buzz_pubsub::PubSubManager::new(&config.redis_url, redis_pool.clone()) + .await + .expect("pubsub manager"), + ); + let audit = buzz_audit::AuditService::new(pool.clone()); + let auth = buzz_auth::AuthService::new(config.auth.clone()); + let search = buzz_search::SearchService::new(pool.clone()); + let workflow_engine = Arc::new(buzz_workflow::WorkflowEngine::new( + db.clone(), + buzz_workflow::WorkflowConfig::default(), + )); + let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let (state, _audit_shutdown) = AppState::new( + config, + db, + redis_pool, + audit, + pubsub, + auth, + search, + workflow_engine, + Keys::generate(), + media_storage, + ); + (Arc::new(state), pool) + } + + async fn approved_deletion( + state: &AppState, + host: &str, + ) -> ( + buzz_db::deletion::DeletionRequest, + buzz_db::deletion::ClaimedDeletion, + ) { + use buzz_db::deletion::{ + FrozenInventory, KeyStreamDigest, PrefixManifest, StorageManifest, + DEFAULT_LEASE_DURATION, + }; + + let store = state.db.deletion_store(); + let request = store + .submit(host, "git-finalize-test", Some("post-CAS lease regression")) + .await + .expect("submit deletion"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(request.community_id) + .await + .expect("schema inventory"), + storage: StorageManifest { + version: 4, + prefixes: buzz_media::tenant_prefixes(*request.community_id.as_uuid()) + .into_iter() + .map(|prefix| PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + }) + .collect(), + }, + }; + store + .freeze_inventory(request.id, &inventory) + .await + .expect("freeze inventory"); + store + .approve(request.id, "git-finalize-test", None) + .await + .expect("approve deletion"); + let claim = store + .claim_specific(request.id, "git-finalize-test", DEFAULT_LEASE_DURATION) + .await + .expect("claim deletion") + .expect("won deletion claim"); + (request, claim) + } + + async fn pushed_context( + state: &AppState, + community: CommunityId, + host: &str, + owner: String, + repo: String, + pusher: nostr::PublicKey, + scratch: &Path, + ) -> PushContext { + let tenant = TenantContext::resolved(community, host); + let (hydrated, parent_state) = hydrate_for_write( + &state.git_store, + &tenant, + &owner, + &repo, + HydrationOptions { + pack_cache: &state.git_pack_cache, + scratch_dir: scratch, + max_pack_bytes: 1024 * 1024, + max_repo_bytes: 2 * 1024 * 1024, + }, + ) + .await + .expect("hydrate empty test repo"); + let source = scratch.join("source"); + tokio::fs::create_dir(&source) + .await + .expect("source directory"); + run_finalize_git(&source, &["init", "--quiet", "--initial-branch=main"]).await; + run_finalize_git(&source, &["config", "user.email", "finalize@test"]).await; + run_finalize_git(&source, &["config", "user.name", "finalize"]).await; + tokio::fs::write(source.join("file.txt"), b"committed\n") + .await + .expect("write source file"); + run_finalize_git(&source, &["add", "file.txt"]).await; + run_finalize_git(&source, &["commit", "--quiet", "-m", "committed"]).await; + let remote = hydrated.path().to_str().expect("hydrated path utf8"); + run_finalize_git(&source, &["push", "--quiet", remote, "main"]).await; + + PushContext { + pack: PackOutput { + stdout: b"push-ok".to_vec(), + ok: true, + }, + parent_state, + owner, + repo: repo.clone(), + repo_id: repo, + pusher, + tenant, + repo_handle: hydrated, + } + } + + #[tokio::test] + #[ignore = "requires Postgres and MinIO"] + async fn finalize_push_holds_serving_lease_through_post_cas_publication() { + let (state, pool) = finalize_test_state().await; + let host = format!("git-finalize-{}.example", uuid::Uuid::new_v4().simple()); + let community = state + .db + .ensure_configured_community(&host) + .await + .expect("create test community") + .id; + let (request, claim) = approved_deletion(&state, &host).await; + let scratch = TempDir::new().expect("scratch"); + let owner = format!("owner-{}", uuid::Uuid::new_v4().simple()); + let repo = format!("repo-{}", uuid::Uuid::new_v4().simple()); + let ctx = pushed_context( + &state, + community, + &host, + owner, + repo.clone(), + Keys::generate().public_key(), + scratch.path(), + ) + .await; + let gate = Arc::new(PostCasGate::default()); + let hooks = FinalizePushHooks { + post_cas_gate: Some(Arc::clone(&gate)), + fail_ref_state_insert: false, + }; + let finalize_state = Arc::clone(&state); + let finalize = + tokio::spawn(async move { finalize_push_inner(&finalize_state, ctx, &hooks).await }); + + gate.reached.notified().await; + state + .db + .deletion_store() + .begin_quiescing(&claim.lease) + .await + .expect("quiesce after CAS"); + let error = state + .db + .deletion_store() + .fence(&claim.lease) + .await + .expect_err("post-CAS serving lease must block fence"); + assert!(matches!( + error, + buzz_db::DbError::ServingWritesNotDrained { .. } + )); + assert!(!state + .db + .deletion_store() + .is_serving_active(community) + .await + .expect("quiescing rejects new serving work")); + + gate.resume.notify_one(); + let response = finalize.await.expect("finalize task"); + assert_eq!(response.status(), StatusCode::OK); + let mut query = buzz_db::event::EventQuery::for_community(community); + query.kinds = Some(vec![30_618]); + query.d_tag = Some(repo); + let events = state.db.query_events(&query).await.expect("query 30618"); + assert_eq!(events.len(), 1, "kind:30618 must be durable before release"); + assert!(state + .db + .deletion_store() + .serving_writes_drained(community) + .await + .expect("serving lease released")); + let generation = state + .db + .deletion_store() + .fence(&claim.lease) + .await + .expect("fence after publication"); + assert_eq!(generation, 1); + assert_eq!( + state + .db + .deletion_store() + .get(request.id) + .await + .expect("fenced request") + .stage, + buzz_db::deletion::DeletionStage::Fenced + ); + drop(state); + pool.close().await; + } + + #[tokio::test] + #[ignore = "requires Postgres and MinIO"] + async fn finalize_push_db_failure_after_cas_is_not_success_and_releases_lease() { + let (state, pool) = finalize_test_state().await; + let host = format!( + "git-finalize-fail-{}.example", + uuid::Uuid::new_v4().simple() + ); + let community = state + .db + .ensure_configured_community(&host) + .await + .expect("create test community") + .id; + let scratch = TempDir::new().expect("scratch"); + let ctx = pushed_context( + &state, + community, + &host, + format!("owner-{}", uuid::Uuid::new_v4().simple()), + format!("repo-{}", uuid::Uuid::new_v4().simple()), + Keys::generate().public_key(), + scratch.path(), + ) + .await; + let hooks = FinalizePushHooks { + post_cas_gate: None, + fail_ref_state_insert: true, + }; + + let response = finalize_push_inner(&state, ctx, &hooks).await; + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!(state + .db + .deletion_store() + .serving_writes_drained(community) + .await + .expect("serving lease released on failure")); + drop(state); + pool.close().await; + } + /// A gzip-encoded request body is transparently inflated before it /// reaches the git subprocess. Git's smart-HTTP client gzips the /// upload-pack/receive-pack request body past a size threshold (fires diff --git a/crates/buzz-relay/src/api/invites.rs b/crates/buzz-relay/src/api/invites.rs index 6104171ccad..d09c7fc6119 100644 --- a/crates/buzz-relay/src/api/invites.rs +++ b/crates/buzz-relay/src/api/invites.rs @@ -260,6 +260,19 @@ async fn authenticate( Ok((tenant, pubkey)) } +fn map_mint_error(error: buzz_db::DbError) -> (StatusCode, Json) { + match error { + buzz_db::DbError::InvalidData(message) | buzz_db::DbError::DeletionSafety(message) => { + api_error(StatusCode::BAD_REQUEST, &message) + } + buzz_db::DbError::AccessDenied(_) => api_error( + StatusCode::SERVICE_UNAVAILABLE, + "community writes are temporarily unavailable", + ), + error => internal_error(&format!("invite mint: {error}")), + } +} + /// Mint an invite code — `POST /api/invites`, NIP-98 signed by an owner/admin. /// /// Returns the code, its expiry, and a shareable landing-page URL on the @@ -304,10 +317,7 @@ pub async fn mint_invite( .db .mint_relay_invite(tenant.community(), &sender_hex, ttl, max_uses) .await - .map_err(|error| match error { - buzz_db::DbError::InvalidData(message) => api_error(StatusCode::BAD_REQUEST, &message), - error => internal_error(&format!("invite mint: {error}")), - })?; + .map_err(map_mint_error)?; // Same TLS-posture logic as nip98_expected_url: wss deployments get an // https landing page URL, ws dev/test deployments get http. @@ -895,6 +905,19 @@ mod tests { } } + #[test] + fn mint_fence_errors_map_to_temporary_unavailability() { + let (status, body) = super::map_mint_error(buzz_db::DbError::AccessDenied( + "community is write-fenced".to_string(), + )); + + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + body.0.get("error").and_then(Value::as_str), + Some("community writes are temporarily unavailable") + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn malformed_and_unknown_v2_codes_are_forbidden_without_v1_fallback() { diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index a2f3640bde5..3b6e07bad66 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -283,6 +283,19 @@ async fn upload_attribution( }) } +fn serving_write_error(error: anyhow::Error) -> MediaError { + if buzz_deletion::ServingWriteGuard::acquisition_is_fenced(&error) { + MediaError::CommunityWriteFenced + } else { + MediaError::ServiceUnavailable + } +} + +fn serving_lease_lost(error: anyhow::Error) -> MediaError { + tracing::warn!(%error, "media serving-write lease lost"); + MediaError::ServiceUnavailable +} + /// PUT `/upload` or the temporary media-only `/media/upload` alias. /// /// Auth is validated via the [`AuthenticatedUpload`] extractor BEFORE the body @@ -310,6 +323,11 @@ pub async fn upload_blob( ) -> Result, MediaError> { let attribution = upload_attribution(&state, &auth, &headers).await; + let serving_write = + buzz_deletion::acquire_serving_write(&state.db, auth.tenant.community(), "media_upload") + .await + .map_err(serving_write_error)?; + if auth.route_mode == UploadRouteMode::LegacyMedia { metrics::counter!("buzz_media_legacy_upload_route_total").increment(1); } @@ -335,69 +353,86 @@ pub async fn upload_blob( } let replay = futures_util::stream::iter(replay_chunks.into_iter().map(Ok)).chain(source); - let mut descriptor = if should_stream_as_video(&sniff) { - // Video path: stream body directly to disk — never fully buffered in RAM. - let content_length = headers - .get("content-length") - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()); - buzz_media::process_video_upload( - &state.media_storage, - &state.config.media, - &auth.tenant, - &auth.auth_event, - replay, - content_length, - attribution, - ) - .await? - } else { - // Non-video path: buffer the body (bounded by the larger of the image - // and generic-file caps), then decide image-vs-generic by sniffed MIME. - // Images go through the thumbnailing pipeline; non-media attachments - // (docs, archives, text, data) take the generic file path and are - // served as downloads. Recognized audio/video cannot fall through it. - let max = state - .config - .media - .max_image_bytes - .max(state.config.media.max_file_bytes); - let bytes = axum::body::to_bytes(axum::body::Body::from_stream(replay), max as usize) - .await - .map_err(|_| MediaError::FileTooLarge { size: 0, max })?; - - let is_image = matches!( - infer::get(&bytes).map(|t| t.mime_type()), - Some("image/jpeg" | "image/png" | "image/gif" | "image/webp") - ); - - if is_image { - buzz_media::process_upload( - &state.media_storage, - &state.config.media, - &auth.tenant, - &auth.auth_event, - bytes, - attribution, - ) - .await? - } else if auth.route_mode == UploadRouteMode::LegacyMedia { - let mime = infer::get(&bytes) - .map(|kind| kind.mime_type().to_string()) - .unwrap_or_else(|| "application/octet-stream".to_string()); - return Err(MediaError::DisallowedContentType(mime)); - } else { - buzz_media::process_file_upload( - &state.media_storage, - &state.config.media, - &auth.tenant, - &auth.auth_event, - bytes, - attribution, - ) - .await? - } - }; + serving_write.verify().await.map_err(serving_lease_lost)?; + + let mut descriptor = serving_write + .protect(async { + Ok(if should_stream_as_video(&sniff) { + // Video path: stream body directly to disk — never fully buffered in RAM. + let content_length = headers + .get("content-length") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + buzz_media::process_video_upload( + &state.media_storage, + &state.config.media, + &auth.tenant, + &auth.auth_event, + replay, + content_length, + attribution, + ) + .await? + } else { + // Non-video path: buffer the body (bounded by the larger of the image + // and generic-file caps), then decide image-vs-generic by sniffed MIME. + // Images go through the thumbnailing pipeline; non-media attachments + // (docs, archives, text, data) take the generic file path and are + // served as downloads. Recognized audio/video cannot fall through it. + let max = state + .config + .media + .max_image_bytes + .max(state.config.media.max_file_bytes); + let bytes = + axum::body::to_bytes(axum::body::Body::from_stream(replay), max as usize) + .await + .map_err(|_| MediaError::FileTooLarge { size: 0, max })?; + + let is_image = matches!( + infer::get(&bytes).map(|t| t.mime_type()), + Some("image/jpeg" | "image/png" | "image/gif" | "image/webp") + ); + + if is_image { + buzz_media::process_upload( + &state.media_storage, + &state.config.media, + &auth.tenant, + &auth.auth_event, + bytes, + attribution, + ) + .await? + } else if auth.route_mode == UploadRouteMode::LegacyMedia { + let mime = infer::get(&bytes) + .map(|kind| kind.mime_type().to_string()) + .unwrap_or_else(|| "application/octet-stream".to_string()); + return Err(MediaError::DisallowedContentType(mime)); + } else { + buzz_media::process_file_upload( + &state.media_storage, + &state.config.media, + &auth.tenant, + &auth.auth_event, + bytes, + attribution, + ) + .await? + } + }) + }) + .await + .map_err(|error| { + if buzz_deletion::ServingWriteGuard::is_lease_lost(&error) { + serving_lease_lost(error) + } else { + match error.downcast::() { + Ok(error) => error, + Err(_) => MediaError::Internal, + } + } + })??; rewrite_descriptor_urls_for_tenant( &mut descriptor, @@ -441,6 +476,7 @@ pub async fn upload_blob( } } + serving_write.finish().await.map_err(serving_lease_lost)?; Ok(Json(descriptor)) } @@ -913,6 +949,20 @@ mod tests { const VALID_HASH: &str = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + #[test] + fn serving_write_error_taxonomy_separates_fence_from_backend_failure() { + let fenced = anyhow::Error::from(buzz_db::DbError::AccessDenied("fenced".to_string())); + assert!(matches!( + serving_write_error(fenced), + MediaError::CommunityWriteFenced + )); + let backend = anyhow::Error::from(buzz_db::DbError::Sqlx(sqlx::Error::PoolTimedOut)); + assert!(matches!( + serving_write_error(backend), + MediaError::ServiceUnavailable + )); + } + #[test] fn upload_routes_distinguish_standard_and_legacy_modes() { assert_eq!( diff --git a/crates/buzz-relay/src/audio/handler.rs b/crates/buzz-relay/src/audio/handler.rs index 16cd56209c7..4c158eab0c4 100644 --- a/crates/buzz-relay/src/audio/handler.rs +++ b/crates/buzz-relay/src/audio/handler.rs @@ -25,7 +25,7 @@ use bytes::Bytes; use futures_util::{SinkExt, StreamExt}; use nostr::{EventBuilder, Kind, Tag}; use serde::Deserialize; -use tokio::sync::{mpsc, OwnedSemaphorePermit, Semaphore}; +use tokio::sync::{mpsc, watch, OwnedSemaphorePermit, Semaphore}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; use uuid::Uuid; @@ -38,7 +38,7 @@ use buzz_core::StoredEvent; use buzz_pubsub::EventTopic; use crate::audio::room::PeerCtrl; -use crate::state::{run_registered_community_connection, AppState}; +use crate::state::{run_registered_community_connection, AppState, CommunityConnectionControl}; /// Maximum binary frame size: 4 KB is generous for a single Opus packet. const MAX_AUDIO_FRAME_BYTES: usize = 4096; @@ -149,6 +149,7 @@ async fn handle_audio_connection( _permit: OwnedSemaphorePermit, ) { let cancel = CancellationToken::new(); + let control = CommunityConnectionControl::new(cancel); let community_id = tenant.community(); let registry = Arc::clone(&state.community_connections); let check_state = Arc::clone(&state); @@ -157,9 +158,11 @@ async fn handle_audio_connection( ®istry, Uuid::new_v4(), community_id, - cancel.clone(), + control, move || async move { check_state.db.is_community_active(community_id).await }, - move || handle_active_audio_connection(socket, run_state, tenant, channel_id, cancel), + move |control| { + handle_active_audio_connection(socket, run_state, tenant, channel_id, control) + }, ) .await; } @@ -169,8 +172,10 @@ async fn handle_active_audio_connection( state: Arc, tenant: TenantContext, channel_id: Uuid, - cancel: CancellationToken, + control: CommunityConnectionControl, ) { + let cancel = control.cancellation_token(); + let disconnect_reason = control.disconnect_reason(); let (mut ws_send, mut ws_recv) = socket.split(); let challenge = generate_challenge(); @@ -660,7 +665,13 @@ async fn handle_active_audio_connection( let (ctrl_tx, ctrl_rx) = mpsc::channel::(8); let send_cancel = cancel.child_token(); - let send_task = tokio::spawn(send_loop(ws_send, data_rx, ctrl_rx, send_cancel)); + let send_task = tokio::spawn(send_loop( + ws_send, + data_rx, + ctrl_rx, + send_cancel, + disconnect_reason, + )); let hb_cancel = cancel.clone(); let hb_missed = Arc::clone(&missed_pongs); @@ -1056,12 +1067,15 @@ async fn recv_loop( /// /// Control frames (Ping, Pong, Close, control JSON) are drained first on every /// iteration, so heartbeat pings are never starved by audio backpressure. -async fn send_loop( - mut ws_send: futures_util::stream::SplitSink, +async fn send_loop( + mut ws_send: S, mut data_rx: mpsc::Receiver, mut ctrl_rx: mpsc::Receiver, cancel: CancellationToken, -) { + disconnect_reason: watch::Receiver>, +) where + S: futures_util::Sink + Unpin, +{ loop { // Priority: drain all pending control frames before data. while let Ok(ctrl_msg) = ctrl_rx.try_recv() { @@ -1073,7 +1087,10 @@ async fn send_loop( tokio::select! { biased; _ = cancel.cancelled() => { - let _ = ws_send.send(WsMessage::Close(None)).await; + let close = disconnect_reason + .borrow() + .map_or(WsMessage::Close(None), |reason| reason.close_message()); + let _ = ws_send.send(close).await; break; } Some(ctrl_msg) = ctrl_rx.recv() => { @@ -1416,6 +1433,74 @@ mod tests { received } + #[tokio::test] + async fn audio_send_loop_sends_policy_close_when_community_is_deleted() { + use futures_util::Sink; + + struct MockSink { + messages: Arc>>, + } + + impl Sink for MockSink { + type Error = std::io::Error; + + fn poll_ready( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn start_send( + self: std::pin::Pin<&mut Self>, + item: WsMessage, + ) -> Result<(), Self::Error> { + self.messages.lock().expect("mock sink poisoned").push(item); + Ok(()) + } + + fn poll_flush( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn poll_close( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.poll_flush(cx) + } + } + + let (_data_tx, data_rx) = mpsc::channel(1); + let (_ctrl_tx, ctrl_rx) = mpsc::channel(1); + let cancel = CancellationToken::new(); + let control = CommunityConnectionControl::new(cancel.clone()); + let disconnect_reason = control.disconnect_reason(); + let registry = crate::state::CommunityConnectionRegistry::new(); + let community = buzz_core::CommunityId::from_uuid(Uuid::new_v4()); + let _guard = registry.register(Uuid::new_v4(), community, control); + assert_eq!(registry.disconnect_community(community), 1); + let messages = Arc::new(Mutex::new(Vec::new())); + let sink = MockSink { + messages: Arc::clone(&messages), + }; + + send_loop(sink, data_rx, ctrl_rx, cancel, disconnect_reason).await; + + let messages = messages.lock().expect("mock sink poisoned"); + assert_eq!(messages.len(), 1); + match &messages[0] { + WsMessage::Close(Some(close)) => { + assert_eq!(close.code, axum::extract::ws::close_code::POLICY); + assert_eq!(close.reason.as_str(), "community deleted"); + } + other => panic!("expected one 1008 deletion close, got {other:?}"), + } + } + #[tokio::test] async fn audio_websocket_parser_rejects_oversized_messages_before_handler_reads_them() { assert!( diff --git a/crates/buzz-relay/src/connection.rs b/crates/buzz-relay/src/connection.rs index 72a7eb91269..c37421e7e80 100644 --- a/crates/buzz-relay/src/connection.rs +++ b/crates/buzz-relay/src/connection.rs @@ -8,7 +8,7 @@ use std::time::Duration; use axum::extract::ws::{Message as WsMessage, WebSocket}; use futures_util::{Sink, SinkExt, StreamExt}; -use tokio::sync::{mpsc, Mutex, RwLock}; +use tokio::sync::{mpsc, watch, Mutex, RwLock}; use tokio_util::sync::CancellationToken; use tracing::Instrument as _; use tracing::{debug, info, trace, warn}; @@ -20,7 +20,10 @@ use nostr::Filter; use crate::handlers; use crate::protocol::{ClientMessage, RelayMessage}; -use crate::state::{run_registered_community_connection, AppState}; +use crate::state::{ + run_registered_community_connection, AppState, CommunityConnectionControl, + CommunityDisconnectReason, +}; use buzz_pubsub::EventTopic; /// Maximum time a new socket may hold a connection slot without completing NIP-42 auth. @@ -128,6 +131,7 @@ pub async fn handle_connection( ) { let conn_id = Uuid::new_v4(); let cancel = CancellationToken::new(); + let control = CommunityConnectionControl::new(cancel); let community_id = tenant.community(); let registry = Arc::clone(&state.community_connections); let check_state = Arc::clone(&state); @@ -136,9 +140,9 @@ pub async fn handle_connection( ®istry, conn_id, community_id, - cancel.clone(), + control, move || async move { check_state.db.is_community_active(community_id).await }, - move || handle_active_connection(socket, run_state, addr, tenant, conn_id, cancel), + move |control| handle_active_connection(socket, run_state, addr, tenant, conn_id, control), ) .await; } @@ -149,8 +153,10 @@ async fn handle_active_connection( addr: SocketAddr, tenant: TenantContext, conn_id: Uuid, - cancel: CancellationToken, + control: CommunityConnectionControl, ) { + let cancel = control.cancellation_token(); + let disconnect_reason = control.disconnect_reason(); let permit = match state.conn_semaphore.clone().try_acquire_owned() { Ok(p) => p, Err(_) => { @@ -226,7 +232,14 @@ async fn handle_active_connection( let (ws_send, ws_recv) = socket.split(); let send_cancel = cancel.child_token(); - let send_task = tokio::spawn(send_loop(ws_send, rx, ctrl_rx, restart_rx, send_cancel)); + let send_task = tokio::spawn(send_loop( + ws_send, + rx, + ctrl_rx, + restart_rx, + send_cancel, + disconnect_reason, + )); let missed_pongs = Arc::new(AtomicU8::new(0)); let heartbeat_cancel = cancel.clone(); @@ -310,8 +323,17 @@ async fn send_loop( ctrl_rx: mpsc::Receiver, restart_rx: mpsc::Receiver, cancel: CancellationToken, + disconnect_reason: watch::Receiver>, ) { - send_loop_inner(ws_send, data_rx, ctrl_rx, restart_rx, cancel).await; + send_loop_inner( + ws_send, + data_rx, + ctrl_rx, + restart_rx, + cancel, + disconnect_reason, + ) + .await; } async fn send_loop_inner( @@ -320,6 +342,7 @@ async fn send_loop_inner( mut ctrl_rx: mpsc::Receiver, mut restart_rx: mpsc::Receiver, cancel: CancellationToken, + disconnect_reason: watch::Receiver>, ) where S: Sink + Unpin, { @@ -359,7 +382,10 @@ async fn send_loop_inner( break; } } - let _ = ws_send.send(WsMessage::Close(None)).await; + let close = disconnect_reason + .borrow() + .map_or(WsMessage::Close(None), |reason| reason.close_message()); + let _ = ws_send.send(close).await; break; } Some(ctrl_msg) = ctrl_rx.recv() => { @@ -787,6 +813,17 @@ mod tests { } } + fn ordinary_disconnect_reason() -> watch::Receiver> { + let (_tx, rx) = watch::channel(None); + rx + } + + fn deleted_community_disconnect_reason() -> watch::Receiver> { + let (tx, rx) = watch::channel(None); + tx.send_replace(Some(CommunityDisconnectReason::CommunityDeleted)); + rx + } + fn text_payloads(messages: &[WsMessage]) -> Vec { messages .iter() @@ -823,7 +860,15 @@ mod tests { let (sink, state) = MockSink::new(Some(1)); let (_restart_tx, restart_rx) = mpsc::channel(1); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, CancellationToken::new()).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + CancellationToken::new(), + ordinary_disconnect_reason(), + ) + .await; let state = state.lock().expect("mock sink poisoned"); assert_eq!(state.flush_count, 1); @@ -844,7 +889,15 @@ mod tests { let (sink, state) = MockSink::new(Some(1)); let (_restart_tx, restart_rx) = mpsc::channel(1); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, CancellationToken::new()).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + CancellationToken::new(), + ordinary_disconnect_reason(), + ) + .await; let state = state.lock().expect("mock sink poisoned"); assert_eq!(state.flush_count, 1); @@ -870,7 +923,15 @@ mod tests { let (sink, state) = MockSink::new(Some(2)); let (_restart_tx, restart_rx) = mpsc::channel(1); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, CancellationToken::new()).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + CancellationToken::new(), + ordinary_disconnect_reason(), + ) + .await; let state = state.lock().expect("mock sink poisoned"); assert_eq!(state.flush_count, 2); @@ -894,7 +955,15 @@ mod tests { .expect("queue restart close"); let (sink, state) = MockSink::new(None); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, CancellationToken::new()).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + CancellationToken::new(), + ordinary_disconnect_reason(), + ) + .await; assert_eq!(flushed_rx.await, Ok(true)); let state = state.lock().expect("mock sink poisoned"); @@ -923,7 +992,15 @@ mod tests { .expect("queue restart close"); let (sink, state) = MockSink::new(Some(1)); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, CancellationToken::new()).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + CancellationToken::new(), + ordinary_disconnect_reason(), + ) + .await; assert_eq!(flushed_rx.await, Ok(false)); let state = state.lock().expect("mock sink poisoned"); @@ -931,6 +1008,59 @@ mod tests { assert_eq!(state.messages.len(), 1, "no fallback close is appended"); } + #[tokio::test] + async fn send_loop_sends_policy_close_when_community_is_deleted() { + let (_data_tx, data_rx) = mpsc::channel(1); + let (_ctrl_tx, ctrl_rx) = mpsc::channel(1); + let (_restart_tx, restart_rx) = mpsc::channel(1); + let cancel = CancellationToken::new(); + cancel.cancel(); + + let (sink, state) = MockSink::new(None); + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + cancel, + deleted_community_disconnect_reason(), + ) + .await; + + let state = state.lock().expect("mock sink poisoned"); + assert_eq!(state.messages.len(), 1); + match &state.messages[0] { + WsMessage::Close(Some(close)) => { + assert_eq!(close.code, axum::extract::ws::close_code::POLICY); + assert_eq!(close.reason.as_str(), "community deleted"); + } + other => panic!("expected one 1008 deletion close, got {other:?}"), + } + } + + #[tokio::test] + async fn send_loop_sends_bare_close_for_ordinary_cancellation() { + let (_data_tx, data_rx) = mpsc::channel(1); + let (_ctrl_tx, ctrl_rx) = mpsc::channel(1); + let (_restart_tx, restart_rx) = mpsc::channel(1); + let cancel = CancellationToken::new(); + cancel.cancel(); + + let (sink, state) = MockSink::new(None); + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + cancel, + ordinary_disconnect_reason(), + ) + .await; + + let state = state.lock().expect("mock sink poisoned"); + assert_eq!(state.messages.as_slice(), [WsMessage::Close(None)]); + } + #[tokio::test] async fn send_loop_flushes_queued_control_before_close_on_cancel() { // A ban disconnect queues its `OK false "blocked: …"` reason frame on @@ -951,7 +1081,15 @@ mod tests { let (sink, state) = MockSink::new(None); let (_restart_tx, restart_rx) = mpsc::channel(1); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, cancel).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + cancel, + ordinary_disconnect_reason(), + ) + .await; let state = state.lock().expect("mock sink poisoned"); assert_eq!( @@ -966,8 +1104,8 @@ mod tests { other => panic!("expected the ban reason frame first, got {other:?}"), } assert!( - matches!(state.messages[1], WsMessage::Close(_)), - "Close is sent only after the reason frame is flushed" + matches!(state.messages[1], WsMessage::Close(None)), + "ordinary cancellation retains the bare Close after the reason frame" ); } } diff --git a/crates/buzz-relay/src/handlers/command_executor.rs b/crates/buzz-relay/src/handlers/command_executor.rs index efdb307e157..abb9bb20665 100644 --- a/crates/buzz-relay/src/handlers/command_executor.rs +++ b/crates/buzz-relay/src/handlers/command_executor.rs @@ -112,6 +112,12 @@ async fn persist_command_event( .begin_transaction() .await .map_err(|e| IngestError::Internal(format!("error: begin transaction: {e}")))?; + buzz_deletion::store(&state.db) + .guard_transaction(&mut tx, tenant.community()) + .await + .map_err(|error| { + IngestError::Rejected(format!("restricted: community writes are fenced: {error}")) + })?; // INSERT with ON CONFLICT DO NOTHING — idempotency guard. let id_bytes = event.id.as_bytes(); diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index a67797385b8..ccba40f3282 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -705,16 +705,49 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc {} + Ok(false) => { + reject("restricted"); + conn.send(RelayMessage::ok( + &event_id_hex, + false, + "restricted: community writes are fenced", + )); + return; + } + Err(error) => { + reject("error"); + tracing::warn!(%error, event_id = %event_id_hex, "failed to check ephemeral-event community lifecycle"); + conn.send(RelayMessage::ok( + &event_id_hex, + false, + "error: internal server error", + )); + return; + } + } + match handle_ephemeral_event( event, conn_id, - &event_id_hex, pubkey_bytes, auth_pubkey, - conn, + Arc::clone(&conn), state, ) - .await; + .await + { + Ok(()) => { + conn.send(RelayMessage::ok(&event_id_hex, true, "")); + } + Err(message) => { + reject("invalid"); + conn.send(RelayMessage::ok(&event_id_hex, false, &message)); + } + } return; } @@ -762,33 +795,19 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc, auth_pubkey: nostr::PublicKey, conn: Arc, state: Arc, -) { +) -> Result<(), String> { let event_clone = event.clone(); + let event_id = event.id.to_hex(); let verify_result = tokio::task::spawn_blocking(move || verify_event(&event_clone)).await; match verify_result { Ok(Ok(())) => {} - Ok(Err(e)) => { - conn.send(RelayMessage::ok( - event_id_hex, - false, - &format!("invalid: {e}"), - )); - return; - } - Err(_) => { - conn.send(RelayMessage::ok( - event_id_hex, - false, - "error: internal error", - )); - return; - } + Ok(Err(e)) => return Err(format!("invalid: {e}")), + Err(_) => return Err("error: internal error".to_string()), } // Special handling for presence events (kind:20001). @@ -829,18 +848,8 @@ async fn handle_ephemeral_event( // Check channel membership before publishing other ephemeral events. if let Some(ch_id) = super::ingest::extract_channel_id(&event) { - if let Err(msg) = super::ingest::check_channel_membership( - &conn.tenant, - &state, - ch_id, - &pubkey_bytes, - None, - ) - .await - { - conn.send(RelayMessage::ok(event_id_hex, false, &msg)); - return; - } + super::ingest::check_channel_membership(&conn.tenant, &state, ch_id, &pubkey_bytes, None) + .await?; // Mark as local before Redis publish to prevent double-delivery when // the event comes back through the Redis subscriber loop. @@ -854,7 +863,7 @@ async fn handle_ephemeral_event( state .local_event_ids .invalidate(&(conn.tenant.community(), event.id.to_bytes())); - warn!(conn_id = %conn_id, event_id = %event_id_hex, "Ephemeral publish failed: {e}"); + warn!(conn_id = %conn_id, event_id = %event_id, "Ephemeral publish failed: {e}"); } // Direct fan-out to local WS subscribers, through the guarded send path @@ -882,7 +891,7 @@ async fn handle_ephemeral_event( state .local_event_ids .invalidate(&(conn.tenant.community(), event.id.to_bytes())); - warn!(conn_id = %conn_id, event_id = %event_id_hex, "Ephemeral global publish failed: {e}"); + warn!(conn_id = %conn_id, event_id = %event_id, "Ephemeral global publish failed: {e}"); } // Direct fan-out to local WS subscribers through the guarded send path. @@ -893,7 +902,7 @@ async fn handle_ephemeral_event( fan_out_event_to_local_subscribers(&state, conn.tenant.community(), &stored_event).await; } - conn.send(RelayMessage::ok(event_id_hex, true, "")); + Ok(()) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index a8651f5c02c..981e6d4ee3d 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -300,6 +300,24 @@ pub enum IngestError { Internal(String), } +/// Map the durable community write-fence lookup onto the ingest error taxonomy. +/// +/// An inactive community is an authorization decision and keeps the exact +/// `restricted:` wire text the ephemeral path uses. A lookup outage is a +/// server fault and fails closed as `error:`/500 — a Postgres blip can +/// neither admit a write past the fence nor read as a client mistake. +fn map_serving_fence_state(active: Result) -> Result<(), IngestError> { + match active { + Ok(true) => Ok(()), + Ok(false) => Err(IngestError::Rejected( + "restricted: community writes are fenced".into(), + )), + Err(error) => Err(IngestError::Internal(format!( + "error: checking community write fence: {error}" + ))), + } +} + fn map_relay_admin_error(error: super::relay_admin::RelayAdminError) -> IngestError { use super::relay_admin::RelayAdminError; match error { @@ -1945,6 +1963,17 @@ async fn ingest_event_inner( let kind_u32 = event_kind_u32(&event); debug!(event_id = %event_id_hex, kind = kind_u32, "ingest_event"); + // Durable community write fence: persistent ingest is a DB write the + // deletion engine cannot exclude via serving-write leases (those cover + // external side effects only), so the shared WS/HTTP seam must refuse + // writes once the community leaves the active lifecycle state. Row churn + // inside the remaining race window is swept by the destructive DB stage. + map_serving_fence_state( + buzz_deletion::store(&state.db) + .is_serving_active(tenant.community()) + .await, + )?; + if kind_u32 == KIND_AUTH { return Err(IngestError::Rejected( "invalid: AUTH events cannot be submitted".into(), @@ -3185,6 +3214,123 @@ mod tests { } } + /// An active community passes the durable write fence untouched. + #[test] + fn serving_fence_active_community_admits_write() { + assert!(map_serving_fence_state(Ok(true)).is_ok()); + } + + /// A fenced/tombstoned/archived community is an authorization decision: + /// `restricted:` and (via `bridge.rs`) HTTP 400 — with the exact wire text + /// the ephemeral WS path uses, so clients see one refusal vocabulary. + #[test] + fn serving_fence_inactive_community_maps_to_restricted() { + match map_serving_fence_state(Ok(false)) { + Err(IngestError::Rejected(msg)) => { + assert_eq!(msg, "restricted: community writes are fenced"); + } + other => panic!("fenced community must map to Rejected, got {other:?}"), + } + } + + /// A fence-lookup outage is a server fault and must fail closed as + /// `error:`/500 — a Postgres blip can neither admit a write past the + /// fence nor be reported to an innocent client as a bad request. + #[test] + fn serving_fence_lookup_outage_fails_closed_as_internal() { + let outage = buzz_db::DbError::Sqlx(sqlx::Error::PoolTimedOut); + match map_serving_fence_state(Err(outage)) { + Err(IngestError::Internal(msg)) => { + assert!( + msg.starts_with("error: "), + "fence outages need the `error:` NIP-01 prefix, got {msg:?}" + ); + } + other => panic!("fence lookup failure must map to Internal, got {other:?}"), + } + } + + /// Production-path regression: the exact predicate `ingest_event_inner` + /// consults must admit writes while a community is active and refuse them + /// once the community deletion lifecycle fences it. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn ingest_write_fence_follows_community_deletion_lifecycle() { + use buzz_db::deletion::{ + FrozenInventory, KeyStreamDigest, PrefixManifest, StorageManifest, + DEFAULT_LEASE_DURATION, + }; + + let url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); // sadscan:disable np.postgres.1 + let pool = sqlx::PgPool::connect(&url).await.expect("connect test DB"); + let db = buzz_db::Db::from_pool(pool); + db.migrate().await.expect("migrate test DB"); + let store = buzz_deletion::store(&db); + + let host = format!("lane3-fence-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("community") + .id; + + assert!( + map_serving_fence_state(store.is_serving_active(community).await).is_ok(), + "active community must admit persistent ingest" + ); + + let submitted = store + .submit( + &host, + "test-operator", + Some("lane3 ingest fence regression"), + ) + .await + .expect("submit"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community) + .await + .expect("schema inventory"), + storage: StorageManifest { + version: 4, + prefixes: buzz_media::tenant_prefixes(*community.as_uuid()) + .into_iter() + .map(|prefix| PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + }) + .collect(), + }, + }; + let request = store + .freeze_inventory(submitted.id, &inventory) + .await + .expect("freeze inventory"); + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + store.fence(&claim.lease).await.expect("fence"); + + match map_serving_fence_state(store.is_serving_active(community).await) { + Err(IngestError::Rejected(msg)) => { + assert_eq!(msg, "restricted: community writes are fenced"); + } + other => panic!("fenced community must refuse persistent ingest, got {other:?}"), + } + } + #[derive(Debug, Default)] struct VecTracer { steps: Mutex>, diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index 34dc2dfcf80..3584e1849d1 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -201,6 +201,12 @@ async fn main() -> anyhow::Result<()> { error!("Failed to ensure partitions: {e}"); } + db.validate_deletion_serving_catalog().await.map_err(|e| { + error!("Community deletion serving-fence validation failed: {e}"); + anyhow::anyhow!("Community deletion serving fence is unsafe: {e}") + })?; + info!("Community deletion serving fences verified"); + // Freshness fence probe: cursor pages route to the replica only for // history the probe has verified as fully replayed. Deliberately AFTER // the migration decision: spawn_fence_probe first verifies the @@ -469,6 +475,7 @@ async fn main() -> anyhow::Result<()> { if let Some(handle) = buzz_relay::mesh_boot::boot_mesh( &state.config, state.redis_pool.clone(), + state.db.clone(), &state.relay_keypair, Arc::clone(&state.shutting_down), ) @@ -1018,6 +1025,24 @@ async fn main() -> anyhow::Result<()> { metrics::gauge!("buzz_redis_pool_size").set(rs.size as f64); metrics::gauge!("buzz_redis_pool_max").set(rs.max_size as f64); metrics::gauge!("buzz_redis_pool_waiting").set(rs.waiting as f64); + + let deletion_store = pool_state.db.deletion_store(); + match deletion_store.reap_expired_serving_write_leases(1000).await { + Ok(reaped) => metrics::counter!("buzz_deletion_serving_leases_reaped_total") + .increment(reaped), + Err(error) => tracing::warn!(%error, "serving-lease reaper failed"), + } + match deletion_store.serving_lease_stats().await { + Ok(stats) => { + metrics::gauge!("buzz_deletion_serving_leases_active") + .set(stats.active as f64); + metrics::gauge!("buzz_deletion_serving_leases_expired") + .set(stats.expired as f64); + metrics::gauge!("buzz_deletion_serving_leases_dead_tuples") + .set(stats.dead_tuples as f64); + } + Err(error) => tracing::warn!(%error, "serving-lease metrics failed"), + } } }); } diff --git a/crates/buzz-relay/src/mesh_boot.rs b/crates/buzz-relay/src/mesh_boot.rs index 20e550aa08a..cd7c427c72e 100644 --- a/crates/buzz-relay/src/mesh_boot.rs +++ b/crates/buzz-relay/src/mesh_boot.rs @@ -411,6 +411,7 @@ fn advertise_addrs(endpoint: &MeshEndpoint) -> Vec { pub async fn boot_mesh( config: &Config, redis_pool: deadpool_redis::Pool, + db: buzz_db::Db, relay_keypair: &nostr::Keys, shutting_down: Arc, ) -> anyhow::Result> { @@ -508,7 +509,7 @@ pub async fn boot_mesh( transport.set_inbound(Box::new(dispatcher.clone())); Ok(Some(MeshHandle { - directory: SessionDirectory::new(redis_pool), + directory: SessionDirectory::with_db(redis_pool, db), transport, membership: membership_arc, local_runtime_id: runtime_id, @@ -535,7 +536,13 @@ mod tests { .create_pool(Some(deadpool_redis::Runtime::Tokio1)) .unwrap(); let keys = nostr::Keys::generate(); - let handle = boot_mesh(&config, pool, &keys, Arc::new(AtomicBool::new(false))) + let db = buzz_db::Db::from_pool( + sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_lazy("postgres://unused:unused@127.0.0.1:1/unused") + .expect("lazy database pool"), + ); + let handle = boot_mesh(&config, pool, db, &keys, Arc::new(AtomicBool::new(false))) .await .expect("off path is never an error"); assert!(handle.is_none()); diff --git a/crates/buzz-relay/src/push_runtime.rs b/crates/buzz-relay/src/push_runtime.rs index 49845067eac..4946b248c65 100644 --- a/crates/buzz-relay/src/push_runtime.rs +++ b/crates/buzz-relay/src/push_runtime.rs @@ -418,6 +418,23 @@ async fn deliver_one( return; } }; + let serving_write = match buzz_deletion::acquire_serving_write( + &state.db, + outcome.community, + "push_delivery", + ) + .await + { + Ok(guard) => guard, + Err(error) => { + warn!(wake=%outcome.id, %error, "push delivery suppressed by community deletion fence"); + let _ = state + .db + .fail_push_wake(outcome.community, outcome.id, outcome.claim_id) + .await; + return; + } + }; let Some(url) = state.config.push_gateway_delivery_url.as_ref() else { return; }; @@ -429,7 +446,20 @@ async fn deliver_one( return; } }; - let response = send_gateway_request(http, url, body, auth).await; + if let Err(error) = serving_write.verify().await { + warn!(wake=%outcome.id, %error, "push serving lease lost before delivery"); + return; + } + let response = match serving_write + .protect(send_gateway_request(http, url, body, auth)) + .await + { + Ok(response) => response, + Err(error) => { + warn!(wake=%outcome.id, %error, "push serving lease lost during delivery"); + return; + } + }; match response { Ok(r) if r.status().is_success() => match r.json::().await { Ok(DeliveryResponse::Accepted) => { @@ -502,6 +532,9 @@ async fn deliver_one( .await; } } + if let Err(error) = serving_write.finish().await { + warn!(wake=%outcome.id, %error, "failed to release community serving lease after push delivery"); + } } fn delivery_body(endpoint_grant: &str, request_id: uuid::Uuid, expires_at: i64) -> Vec { diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index 400ed1dfe34..82ad9938a2f 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -376,22 +376,30 @@ async fn readiness_handler(State(state): State>) -> impl IntoRespo } let check = async { - let (pg_ok, redis_ok) = tokio::join!(state.db.ping(), async { - state.redis_pool.get().await.is_ok() - },); - (pg_ok, redis_ok) + let (pg_ok, redis_ok, deletion_catalog_ok) = tokio::join!( + state.db.ping(), + async { state.redis_pool.get().await.is_ok() }, + async { state.db.validate_deletion_serving_catalog().await.is_ok() }, + ); + (pg_ok, redis_ok, deletion_catalog_ok) }; - let (pg_ok, redis_ok) = tokio::time::timeout(Duration::from_secs(2), check) - .await - .unwrap_or((false, false)); + let (pg_ok, redis_ok, deletion_catalog_ok) = + tokio::time::timeout(Duration::from_secs(2), check) + .await + .unwrap_or((false, false, false)); - if pg_ok && redis_ok { + if pg_ok && redis_ok && deletion_catalog_ok { (StatusCode::OK, Json(json!({"status": "ready"}))).into_response() } else { ( StatusCode::SERVICE_UNAVAILABLE, - Json(json!({"status": "not_ready", "postgres": pg_ok, "redis": redis_ok})), + Json(json!({ + "status": "not_ready", + "postgres": pg_ok, + "redis": redis_ok, + "deletion_catalog": deletion_catalog_ok + })), ) .into_response() } diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index 84c8911e477..a7c97be94a4 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -10,8 +10,7 @@ use axum::body::Bytes; use axum::extract::ws::{Message as WsMessage, Utf8Bytes as WsUtf8Bytes}; use dashmap::DashMap; use futures_util::future::join_all; -use tokio::sync::mpsc; -use tokio::sync::Semaphore; +use tokio::sync::{mpsc, watch, Semaphore}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use uuid::Uuid; @@ -37,6 +36,55 @@ use crate::subscription::SubscriptionRegistry; pub(crate) type ScopedPubkeyKey = (CommunityId, [u8; 32]); +/// Why a community-bound socket is being asked to stop. +/// +/// Only deletion is externally attributed today. Ordinary lifecycle exits keep +/// using cancellation alone and therefore retain the existing bare-close +/// behavior. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CommunityDisconnectReason { + CommunityDeleted, +} + +impl CommunityDisconnectReason { + pub(crate) fn close_message(self) -> WsMessage { + match self { + Self::CommunityDeleted => WsMessage::Close(Some(axum::extract::ws::CloseFrame { + code: axum::extract::ws::close_code::POLICY, + reason: WsUtf8Bytes::from_static("community deleted"), + })), + } + } +} + +/// Per-socket lifecycle controls shared by the registry and the writer. +#[derive(Clone)] +pub(crate) struct CommunityConnectionControl { + cancel: CancellationToken, + reason_tx: watch::Sender>, +} + +impl CommunityConnectionControl { + pub(crate) fn new(cancel: CancellationToken) -> Self { + let (reason_tx, _reason_rx) = watch::channel(None); + Self { cancel, reason_tx } + } + + pub(crate) fn cancellation_token(&self) -> CancellationToken { + self.cancel.clone() + } + + pub(crate) fn disconnect_reason(&self) -> watch::Receiver> { + self.reason_tx.subscribe() + } + + fn disconnect_community(&self) { + self.reason_tx + .send_replace(Some(CommunityDisconnectReason::CommunityDeleted)); + self.cancel.cancel(); + } +} + /// Leaves headroom under the process-wide drain deadline for a stalled writer. const RESTART_CLOSE_ACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); type SlidingWindowCounter = (u32, Instant); @@ -68,7 +116,7 @@ struct ConnEntry { /// registration cancels the token; archival before registration is observed by /// the revalidation. The returned guard removes the entry on every exit path. pub struct CommunityConnectionRegistry { - connections: Arc>, + connections: Arc>, } impl Default for CommunityConnectionRegistry { @@ -86,26 +134,27 @@ impl CommunityConnectionRegistry { } /// Registers one socket and returns a guard that deregisters it on drop. - pub fn register( + pub(crate) fn register( &self, connection_id: Uuid, community_id: CommunityId, - cancel: CancellationToken, + control: CommunityConnectionControl, ) -> CommunityConnectionGuard { self.connections - .insert(connection_id, (community_id, cancel)); + .insert(connection_id, (community_id, control)); CommunityConnectionGuard { connection_id, connections: Arc::clone(&self.connections), } } - /// Cancels every socket type currently bound to `community_id`. + /// Disconnects every socket type currently bound to `community_id` and + /// attributes the close to community deletion. pub fn disconnect_community(&self, community_id: CommunityId) -> usize { let mut closed = 0; for entry in self.connections.iter() { if entry.value().0 == community_id { - entry.value().1.cancel(); + entry.value().1.disconnect_community(); closed += 1; } } @@ -124,7 +173,7 @@ impl CommunityConnectionRegistry { /// Removes a socket lifecycle registration on every handler exit path. pub struct CommunityConnectionGuard { connection_id: Uuid, - connections: Arc>, + connections: Arc>, } impl Drop for CommunityConnectionGuard { @@ -137,20 +186,21 @@ impl Drop for CommunityConnectionGuard { /// /// The ordering is the archival admission invariant: archive-before-query is /// observed by the query, while archive-after-registration sees the token. -pub async fn run_registered_community_connection( +pub(crate) async fn run_registered_community_connection( registry: &CommunityConnectionRegistry, connection_id: Uuid, community_id: CommunityId, - cancel: CancellationToken, + control: CommunityConnectionControl, check_active: Check, run: Run, ) where Check: FnOnce() -> CheckFuture, CheckFuture: Future>, - Run: FnOnce() -> RunFuture, + Run: FnOnce(CommunityConnectionControl) -> RunFuture, RunFuture: Future, { - let _guard = registry.register(connection_id, community_id, cancel.clone()); + let cancel = control.cancel.clone(); + let _guard = registry.register(connection_id, community_id, control.clone()); if !matches!(check_active().await, Ok(true)) { cancel.cancel(); return; @@ -158,7 +208,7 @@ pub async fn run_registered_community_connection, } /// Active session ownership lease read from Redis. @@ -179,6 +180,9 @@ pub enum DirectoryError { /// Lease TTL cannot be represented in Redis milliseconds. #[error("lease ttl must be at least 1ms and fit in i64 milliseconds")] InvalidLeaseTtl, + /// Durable community deletion fence rejected a Redis mutation. + #[error("community write fenced: {0}")] + CommunityWriteFenced(String), } impl SessionDirectory { @@ -187,9 +191,36 @@ impl SessionDirectory { Self::with_lease_ttl(pool, DEFAULT_LEASE_TTL) } + /// Create a serving directory whose Redis mutations use durable, + /// heartbeat-backed community write leases. + pub fn with_db(pool: deadpool_redis::Pool, db: buzz_db::Db) -> Self { + Self { + pool, + lease_ttl: DEFAULT_LEASE_TTL, + db: Some(db), + } + } + /// Create a directory backed by `pool` with an explicit lease TTL. pub fn with_lease_ttl(pool: deadpool_redis::Pool, lease_ttl: Duration) -> Self { - Self { pool, lease_ttl } + Self { + pool, + lease_ttl, + db: None, + } + } + + async fn begin_serving_write( + &self, + community_id: CommunityId, + ) -> Result, DirectoryError> { + match &self.db { + Some(db) => buzz_deletion::acquire_serving_write(db, community_id, "session_directory") + .await + .map(Some) + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string())), + None => Ok(None), + } } /// Attempt to create/take over the session lease. @@ -204,10 +235,11 @@ impl SessionDirectory { owner_runtime_id: RuntimeId, profile: Profile, ) -> Result { + let serving_write = self.begin_serving_write(community_id).await?; let keys = SessionKeys::new(community_id, session_id); let ttl_ms = ttl_ms(self.lease_ttl)?; let mut conn = self.pool.get().await?; - let (status, value, _known_generation): (String, String, String) = + let mutation = async { Script::new(ACQUIRE_SCRIPT) .key(&keys.lease) .key(&keys.generation) @@ -215,8 +247,22 @@ impl SessionDirectory { .arg(profile.as_wire_str()) .arg(ttl_ms) .invoke_async(&mut *conn) - .await?; + .await + }; + let (status, value, _known_generation): (String, String, String) = match &serving_write { + Some(guard) => guard + .protect(mutation) + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))??, + None => mutation.await?, + }; let lease = parse_lease(community_id, session_id, &value)?; + if let Some(guard) = serving_write { + guard + .finish() + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))?; + } match status.as_str() { "acquired" => Ok(AcquireResult::Acquired(lease)), "exists" => Ok(AcquireResult::Exists(lease)), @@ -244,18 +290,34 @@ impl SessionDirectory { /// Renew a lease only if the current Redis value exactly matches the /// caller's owner runtime and generation. pub async fn renew(&self, lease: &SessionLease) -> Result { + let serving_write = self.begin_serving_write(lease.community_id).await?; let keys = SessionKeys::new(lease.community_id, lease.session_id); let ttl_ms = ttl_ms(self.lease_ttl)?; let mut conn = self.pool.get().await?; - let (status, value, known_generation): (String, String, String) = Script::new(RENEW_SCRIPT) - .key(&keys.lease) - .key(&keys.generation) - .arg(lease.owner_runtime_id.to_hex()) - .arg(lease.generation) - .arg(ttl_ms) - .invoke_async(&mut *conn) - .await?; + let mutation = async { + Script::new(RENEW_SCRIPT) + .key(&keys.lease) + .key(&keys.generation) + .arg(lease.owner_runtime_id.to_hex()) + .arg(lease.generation) + .arg(ttl_ms) + .invoke_async(&mut *conn) + .await + }; + let (status, value, known_generation): (String, String, String) = match &serving_write { + Some(guard) => guard + .protect(mutation) + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))??, + None => mutation.await?, + }; let current = parse_optional_lease(lease.community_id, lease.session_id, &value)?; + if let Some(guard) = serving_write { + guard + .finish() + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))?; + } match status.as_str() { "renewed" => Ok(RenewResult::Renewed( current.expect("renewed returns lease"), @@ -275,17 +337,32 @@ impl SessionDirectory { /// Release a lease only if the current Redis value exactly matches the /// caller's owner runtime and generation. pub async fn release(&self, lease: &SessionLease) -> Result { + let serving_write = self.begin_serving_write(lease.community_id).await?; let keys = SessionKeys::new(lease.community_id, lease.session_id); let mut conn = self.pool.get().await?; - let (status, value, known_generation): (String, String, String) = + let mutation = async { Script::new(RELEASE_SCRIPT) .key(&keys.lease) .key(&keys.generation) .arg(lease.owner_runtime_id.to_hex()) .arg(lease.generation) .invoke_async(&mut *conn) - .await?; + .await + }; + let (status, value, known_generation): (String, String, String) = match &serving_write { + Some(guard) => guard + .protect(mutation) + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))??, + None => mutation.await?, + }; let current = parse_optional_lease(lease.community_id, lease.session_id, &value)?; + if let Some(guard) = serving_write { + guard + .finish() + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))?; + } match status.as_str() { "released" => Ok(ReleaseResult::Released( current.expect("released returns lease"), diff --git a/crates/buzz-test-client/tests/e2e_relay.rs b/crates/buzz-test-client/tests/e2e_relay.rs index 2013875d9ab..5d5ad8916c3 100644 --- a/crates/buzz-test-client/tests/e2e_relay.rs +++ b/crates/buzz-test-client/tests/e2e_relay.rs @@ -120,7 +120,12 @@ async fn seed_relay_member(host: &str, keys: &Keys, role: &str) { } async fn seed_relay_owner(keys: &Keys) { - seed_relay_member("localhost:3000", keys, "owner").await; + seed_relay_member(&relay_authority(), keys, "owner").await; +} + +fn relay_authority() -> String { + let url = url::Url::parse(&relay_http_url()).expect("relay HTTP URL"); + url[url::Position::BeforeHost..url::Position::AfterPort].to_string() } fn http_origin_for_host(host: &str) -> String { @@ -315,7 +320,7 @@ async fn test_invite_claim_rejects_invalid_code() { #[ignore] async fn test_invite_mint_requires_owner_or_admin() { let member = Keys::generate(); - seed_relay_member("localhost:3000", &member, "member").await; + seed_relay_member(&relay_authority(), &member, "member").await; let response = invite_post(&member, "/api/invites", "{}").await; assert_eq!(response.status(), reqwest::StatusCode::FORBIDDEN); @@ -791,10 +796,10 @@ async fn test_auth_event_kind_rejected() { /// NIP-11 max_subscriptions must be enforced; (limit+1)th REQ gets CLOSED. /// -/// The relay's MAX_SUBSCRIPTIONS is 1024. Opening 1024 subs in a test is slow, -/// so we open a smaller batch and verify the NIP-11 advertised limit matches -/// the actual enforcement constant. The full-limit test is covered by the -/// NIP-11 assertion below (which verifies the advertised value is 1024). +/// This is a protocol-cap test, not an admission-throughput test. Open one REQ +/// at a time and wait out any shared fixed-window quota before retrying a REQ +/// rejected specifically as `rate-limited`, so production admission remains +/// enabled while the test deterministically reaches the independent 1024 cap. #[tokio::test] #[ignore] async fn test_subscription_limit_enforced() { @@ -802,60 +807,75 @@ async fn test_subscription_limit_enforced() { let keys = Keys::generate(); let mut client = BuzzTestClient::connect(&url, &keys).await.expect("connect"); - // Open 1024 subscriptions (the relay's MAX_SUBSCRIPTIONS). for i in 0..1024 { let sid = format!("limit-sub-{i}"); - let filter = Filter::new().kind(Kind::Custom(9)); - client - .subscribe(&sid, vec![filter]) - .await - .expect("subscribe"); - // Drain EOSE to avoid buffer buildup. - client - .collect_until_eose(&sid, Duration::from_secs(5)) - .await - .expect("EOSE"); + let filter = Filter::new().kind(Kind::Custom(49_999)); + subscribe_until_eose(&mut client, &sid, filter).await; } let overflow_sid = sub_id("overflow"); - // Use a kind that no other test writes, so we don't receive stale events. - let filter = Filter::new().kind(Kind::Custom(49999)); - client - .subscribe(&overflow_sid, vec![filter]) - .await - .expect("send REQ"); - - // Drain EOSE and stale events from the 100 earlier subscriptions - // until we receive the CLOSED for the overflow subscription. - let msg = loop { - let m = client - .recv_event(Duration::from_secs(5)) + let filter = Filter::new().kind(Kind::Custom(49_999)); + loop { + client + .subscribe(&overflow_sid, vec![filter.clone()]) .await - .expect("recv CLOSED (or timeout)"); - match &m { - RelayMessage::Eose { .. } => continue, - RelayMessage::Event { .. } => continue, // stale event from earlier subs - _ => break m, - } - }; + .expect("send overflow REQ"); - match msg { - RelayMessage::Closed { - subscription_id, - message, - } => { - assert_eq!(subscription_id, overflow_sid); - assert!( - message.to_lowercase().contains("too many"), - "Expected 'too many' in CLOSED message, got: {message}" - ); + match client + .recv_event(Duration::from_secs(6)) + .await + .expect("recv overflow CLOSED") + { + RelayMessage::Closed { + subscription_id, + message, + } if subscription_id == overflow_sid && message.starts_with("rate-limited:") => { + tokio::time::sleep(Duration::from_secs(5)).await; + } + RelayMessage::Closed { + subscription_id, + message, + } => { + assert_eq!(subscription_id, overflow_sid); + assert!( + message.to_lowercase().contains("too many"), + "Expected 'too many' in CLOSED message, got: {message}" + ); + break; + } + other => panic!("Expected CLOSED for overflow subscription, got {other:?}"), } - other => panic!("Expected CLOSED for overflow subscription, got {other:?}"), } client.disconnect().await.expect("disconnect"); } +async fn subscribe_until_eose(client: &mut BuzzTestClient, sid: &str, filter: Filter) { + loop { + client + .subscribe(sid, vec![filter.clone()]) + .await + .expect("subscribe"); + match client + .recv_event(Duration::from_secs(6)) + .await + .expect("EOSE or rate-limit CLOSED") + { + RelayMessage::Eose { subscription_id } => { + assert_eq!(subscription_id, sid); + return; + } + RelayMessage::Closed { + subscription_id, + message, + } if subscription_id == sid && message.starts_with("rate-limited:") => { + tokio::time::sleep(Duration::from_secs(5)).await; + } + other => panic!("unexpected response while opening {sid}: {other:?}"), + } + } +} + #[tokio::test] #[ignore] async fn test_nip11_relay_info() { diff --git a/crates/buzz-workflow/Cargo.toml b/crates/buzz-workflow/Cargo.toml index d4813e56d42..7d361b1477a 100644 --- a/crates/buzz-workflow/Cargo.toml +++ b/crates/buzz-workflow/Cargo.toml @@ -10,6 +10,7 @@ description = "YAML-as-code workflow engine for Buzz" [dependencies] buzz-core = { workspace = true } buzz-db = { workspace = true } +buzz-deletion = { workspace = true } hex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/buzz-workflow/src/executor.rs b/crates/buzz-workflow/src/executor.rs index e30541377e4..dffa4927168 100644 --- a/crates/buzz-workflow/src/executor.rs +++ b/crates/buzz-workflow/src/executor.rs @@ -526,165 +526,202 @@ pub async fn dispatch_action( ) -> Result { use ActionDef::*; - match action { - SendMessage { text, channel } => { - // Look up workflow metadata for destination validation and - // attribution, scoped to the run's community — the same run/workflow - // UUID may exist in another community, so a bare-id lookup could - // load the wrong row and drive a side effect under it. - let wf_run = engine - .db - .get_workflow_run(community_id, run_id) - .await - .map_err(|e| { - WorkflowError::WebhookError(format!( - "SendMessage: failed to load workflow run {run_id}: {e}" - )) - })?; - let workflow = engine - .db - .get_workflow(community_id, wf_run.workflow_id) - .await - .map_err(|e| { - WorkflowError::WebhookError(format!( - "SendMessage: failed to load workflow {}: {e}", - wf_run.workflow_id - )) - })?; - let channel_id = resolve_send_message_channel( - channel.as_deref(), - &trigger_ctx.channel_id, - workflow.channel_id, - )?; - let owner_pubkey_hex = hex::encode(&workflow.owner_pubkey); - - info!( - run_id = %run_id, - step = step_id, - channel = %channel_id, - "SendMessage → {channel_id}: {text}" - ); - - let event_id = engine - .action_sink()? - .send_message(community_id, &channel_id, text, &owner_pubkey_hex) - .await - .map_err(WorkflowError::from)?; - - Ok(StepResult::Completed(serde_json::json!({ - "sent": true, - "event_id": event_id, - }))) - } - - SendDm { to, text: _ } => { - warn!(run_id = %run_id, step = step_id, "SendDm not yet implemented (to={to})"); - // TODO (WF-07): emit DM event. - Err(WorkflowError::NotImplemented("SendDm".into())) - } - - SetChannelTopic { topic: _ } => { - warn!(run_id = %run_id, step = step_id, "SetChannelTopic not yet implemented"); - // TODO (WF-07): update channel topic via DB. - Err(WorkflowError::NotImplemented("SetChannelTopic".into())) - } - - AddReaction { emoji } => { - info!(run_id = %run_id, step = step_id, "AddReaction → :{emoji}:"); - if trigger_ctx.message_id.is_empty() { - return Err(WorkflowError::InvalidDefinition( - "AddReaction: no trigger.message_id available".into(), - )); - } - - #[cfg(feature = "reqwest")] - { - let result = add_reaction_impl(&trigger_ctx.message_id, emoji).await?; - Ok(StepResult::Completed(result)) - } - - #[cfg(not(feature = "reqwest"))] - { - warn!( - run_id = %run_id, - step = step_id, - "AddReaction: reqwest feature not enabled, skipping HTTP call" - ); - Ok(StepResult::Completed( - serde_json::json!({ "added": false, "skipped": true }), + // The workflow engine can outlive the serving request that spawned it. + // Revalidate the durable community fence immediately before every external + // side effect (message publish, webhook, delay/resume). A storage failure is + // a denial, never permission to continue. + let serving_write = + buzz_deletion::acquire_serving_write(&engine.db, community_id, "workflow_action") + .await + .map_err(|error| { + WorkflowError::WebhookError(format!( + "community write fence rejected workflow side effect: {error}" )) - } - } + })?; - CallWebhook { - url, - method, - headers, - body, - } => { - let method_str = method.as_deref().unwrap_or("POST"); - info!(run_id = %run_id, step = step_id, "CallWebhook → {method_str} {url}"); + serving_write.verify().await.map_err(|error| { + WorkflowError::WebhookError(format!("community write lease lost: {error}")) + })?; - #[cfg(feature = "reqwest")] - { - let result = call_webhook_impl(url, method_str, headers, body).await?; - Ok(StepResult::Completed(result)) - } + let result = serving_write + .protect(async { + match action { + SendMessage { text, channel } => { + // Look up workflow metadata for destination validation and + // attribution, scoped to the run's community — the same run/workflow + // UUID may exist in another community, so a bare-id lookup could + // load the wrong row and drive a side effect under it. + let wf_run = engine + .db + .get_workflow_run(community_id, run_id) + .await + .map_err(|e| { + WorkflowError::WebhookError(format!( + "SendMessage: failed to load workflow run {run_id}: {e}" + )) + })?; + let workflow = engine + .db + .get_workflow(community_id, wf_run.workflow_id) + .await + .map_err(|e| { + WorkflowError::WebhookError(format!( + "SendMessage: failed to load workflow {}: {e}", + wf_run.workflow_id + )) + })?; + let channel_id = resolve_send_message_channel( + channel.as_deref(), + &trigger_ctx.channel_id, + workflow.channel_id, + )?; + let owner_pubkey_hex = hex::encode(&workflow.owner_pubkey); + + info!( + run_id = %run_id, + step = step_id, + channel = %channel_id, + "SendMessage → {channel_id}: {text}" + ); + + let event_id = engine + .action_sink()? + .send_message(community_id, &channel_id, text, &owner_pubkey_hex) + .await + .map_err(WorkflowError::from)?; + + Ok(StepResult::Completed(serde_json::json!({ + "sent": true, + "event_id": event_id, + }))) + } - #[cfg(not(feature = "reqwest"))] - { - // reqwest not enabled — log and return placeholder. - warn!( - run_id = %run_id, step = step_id, - "CallWebhook: reqwest feature not enabled, skipping HTTP call" - ); - let _ = (headers, body); // suppress unused warnings - Ok(StepResult::Completed(serde_json::json!({ - "status": 0, - "body": null, - "skipped": true - }))) - } - } + SendDm { to, text: _ } => { + warn!(run_id = %run_id, step = step_id, "SendDm not yet implemented (to={to})"); + // TODO (WF-07): emit DM event. + Err(WorkflowError::NotImplemented("SendDm".into())) + } - RequestApproval { - from, - message, - timeout, - } => { - let timeout_str = timeout.as_deref().unwrap_or("24h"); - info!( - run_id = %run_id, step = step_id, - "RequestApproval from={from} timeout={timeout_str}: {message}" - ); + SetChannelTopic { topic: _ } => { + warn!(run_id = %run_id, step = step_id, "SetChannelTopic not yet implemented"); + // TODO (WF-07): update channel topic via DB. + Err(WorkflowError::NotImplemented("SetChannelTopic".into())) + } + + AddReaction { emoji } => { + info!(run_id = %run_id, step = step_id, "AddReaction → :{emoji}:"); + if trigger_ctx.message_id.is_empty() { + Err(WorkflowError::InvalidDefinition( + "AddReaction: no trigger.message_id available".into(), + )) + } else { + #[cfg(feature = "reqwest")] + { + let result = add_reaction_impl(&trigger_ctx.message_id, emoji).await?; + Ok(StepResult::Completed(result)) + } + + #[cfg(not(feature = "reqwest"))] + { + warn!( + run_id = %run_id, + step = step_id, + "AddReaction: reqwest feature not enabled, skipping HTTP call" + ); + Ok(StepResult::Completed( + serde_json::json!({ "added": false, "skipped": true }), + )) + } + } + } - let token = generate_approval_token(run_id, step_id); + CallWebhook { + url, + method, + headers, + body, + } => { + let method_str = method.as_deref().unwrap_or("POST"); + info!(run_id = %run_id, step = step_id, "CallWebhook → {method_str} {url}"); + + #[cfg(feature = "reqwest")] + { + let result = call_webhook_impl(url, method_str, headers, body).await?; + Ok(StepResult::Completed(result)) + } - // TODO (WF-08): create approval record in DB, emit kind:46010. - // For now, return Suspended with the token so the caller can persist state. + #[cfg(not(feature = "reqwest"))] + { + // reqwest not enabled — log and return placeholder. + warn!( + run_id = %run_id, step = step_id, + "CallWebhook: reqwest feature not enabled, skipping HTTP call" + ); + let _ = (headers, body); // suppress unused warnings + Ok(StepResult::Completed(serde_json::json!({ + "status": 0, + "body": null, + "skipped": true + }))) + } + } - Ok(StepResult::Suspended { - approval_token: token, - }) - } + RequestApproval { + from, + message, + timeout, + } => { + let timeout_str = timeout.as_deref().unwrap_or("24h"); + info!( + run_id = %run_id, step = step_id, + "RequestApproval from={from} timeout={timeout_str}: {message}" + ); + + let token = generate_approval_token(run_id, step_id); + + // TODO (WF-08): create approval record in DB, emit kind:46010. + // For now, return Suspended with the token so the caller can persist state. + + Ok(StepResult::Suspended { + approval_token: token, + }) + } - Delay { duration } => { - let secs = parse_duration_secs(duration)?; - // Cap delay at 270 seconds (4.5 minutes) — must be less than default_timeout_secs (300s) - // to avoid non-deterministic StepTimeout. Long delays (hours/days) - // should use the scheduled resume pattern (future work: WF-09). - const MAX_DELAY_SECS: u64 = 270; - if secs > MAX_DELAY_SECS { - return Err(WorkflowError::InvalidDefinition(format!( - "delay exceeds maximum of {MAX_DELAY_SECS} seconds (got {secs}s); \ + Delay { duration } => { + let secs = parse_duration_secs(duration)?; + // Cap delay at 270 seconds (4.5 minutes) — must be less than default_timeout_secs (300s) + // to avoid non-deterministic StepTimeout. Long delays (hours/days) + // should use the scheduled resume pattern (future work: WF-09). + const MAX_DELAY_SECS: u64 = 270; + if secs > MAX_DELAY_SECS { + return Err(WorkflowError::InvalidDefinition(format!( + "delay exceeds maximum of {MAX_DELAY_SECS} seconds (got {secs}s); \ use the scheduled resume pattern for long delays" - ))); + ))); + } + info!(run_id = %run_id, step = step_id, "Delay {duration} ({secs}s)"); + tokio::time::sleep(std::time::Duration::from_secs(secs)).await; + Ok(StepResult::Completed( + serde_json::json!({ "slept_secs": secs }), + )) + } } - info!(run_id = %run_id, step = step_id, "Delay {duration} ({secs}s)"); - tokio::time::sleep(std::time::Duration::from_secs(secs)).await; - Ok(StepResult::Completed( - serde_json::json!({ "slept_secs": secs }), - )) + }) + .await + .map_err(|error| { + WorkflowError::WebhookError(format!("community write lease lost: {error}")) + })?; + let release = serving_write.finish().await.map_err(|error| { + WorkflowError::WebhookError(format!("community write lease release failed: {error}")) + }); + match result { + Ok(value) => { + release?; + Ok(value) + } + Err(error) => { + let _ = release; + Err(error) } } } diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index b2778df28b5..86989676604 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -62,9 +62,15 @@ Buzz uses one URL style for both media and Git/CAS object-store requests: | `virtual` | `https://bucket.endpoint/key` | AWS-style providers and new Railway Storage Buckets | The chart always renders `s3.addressingStyle` as -`BUZZ_S3_ADDRESSING_STYLE`. It renders `s3.region` as `BUZZ_S3_REGION` only -when explicitly set, preserving the relay's existing `AWS_REGION` fallback for -upgrades. Only `path` and `virtual` addressing styles are accepted; invalid +`BUZZ_S3_ADDRESSING_STYLE` and `s3.region` as `BUZZ_S3_REGION`. The region +defaults to `us-east-1`, keeping bundled MinIO and the in-pod +`buzz-admin deletions` workflow operable without an ambient `AWS_REGION`. +Production providers must set their credential region explicitly when it +differs. Existing releases that previously omitted `s3.region` will begin +rendering `BUZZ_S3_REGION=us-east-1` after upgrade, even if an image or +`relay.extraEnv` entry supplied `AWS_REGION`; set `s3.region` to the provider's +actual credential region before upgrading. Only `path` and `virtual` addressing +styles are accepted; invalid values fail chart rendering and relay startup. The bundled MinIO quickstart deliberately keeps `path` because its Service DNS resolves one endpoint hostname, not arbitrary `.` names. diff --git a/deploy/charts/buzz/templates/deployment.yaml b/deploy/charts/buzz/templates/deployment.yaml index 0ad41ac4611..451ebb1cded 100644 --- a/deploy/charts/buzz/templates/deployment.yaml +++ b/deploy/charts/buzz/templates/deployment.yaml @@ -170,9 +170,7 @@ spec: - { name: BUZZ_S3_ENDPOINT, value: {{ $s3Endpoint | quote }} } {{- end }} - { name: BUZZ_S3_BUCKET, value: {{ .Values.s3.bucket | quote }} } - {{- if .Values.s3.region }} - { name: BUZZ_S3_REGION, value: {{ .Values.s3.region | quote }} } - {{- end }} - { name: BUZZ_S3_ADDRESSING_STYLE, value: {{ .Values.s3.addressingStyle | quote }} } # ── Secrets (from chart-managed or existing) ───────────── diff --git a/deploy/charts/buzz/tests/render_test.yaml b/deploy/charts/buzz/tests/render_test.yaml index 196a4a53032..10a1a34d1fd 100644 --- a/deploy/charts/buzz/tests/render_test.yaml +++ b/deploy/charts/buzz/tests/render_test.yaml @@ -30,11 +30,11 @@ tests: path: kind value: Service template: templates/service.yaml - - notContains: + - contains: path: spec.template.spec.containers[0].env content: name: BUZZ_S3_REGION - any: true + value: "us-east-1" template: templates/deployment.yaml - contains: path: spec.template.spec.containers[0].env diff --git a/deploy/charts/buzz/values.schema.json b/deploy/charts/buzz/values.schema.json index d3670595b5b..94d369c8903 100644 --- a/deploy/charts/buzz/values.schema.json +++ b/deploy/charts/buzz/values.schema.json @@ -200,7 +200,8 @@ "bucket": { "type": "string", "minLength": 1 }, "region": { "type": "string", - "description": "Optional S3 region used for SigV4 signing. When empty, BUZZ_S3_REGION is omitted so the relay can use AWS_REGION or its own default." + "minLength": 1, + "description": "S3 region used for SigV4 signing by the relay and deletion operator. Defaults to us-east-1 for bundled MinIO/local deployments; set the provider region explicitly when it differs." }, "addressingStyle": { "type": "string", diff --git a/deploy/charts/buzz/values.yaml b/deploy/charts/buzz/values.yaml index 8131aef4321..ca3403a633f 100644 --- a/deploy/charts/buzz/values.yaml +++ b/deploy/charts/buzz/values.yaml @@ -342,9 +342,10 @@ externalRedis: s3: endpoint: "" bucket: "buzz-media" - # Optional SigV4 signing region. Leave empty to preserve the relay's - # AWS_REGION fallback; set the provider's credential value when needed. - region: "" + # SigV4 signing region shared by the relay and `buzz-admin deletions`. + # Keep the MinIO/local default operable; production providers should set + # their credential region explicitly when it differs. + region: "us-east-1" # path: https://endpoint/bucket/key (bundled MinIO-compatible default) # virtual: https://bucket.endpoint/key (standard S3; required by new Railway buckets) addressingStyle: path diff --git a/desktop/package.json b/desktop/package.json index 0c1fb137339..3601f25185e 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -1,7 +1,7 @@ { "name": "buzz", "private": true, - "version": "0.5.10", + "version": "0.5.11", "type": "module", "scripts": { "dev": "vite", diff --git a/desktop/playwright.config.ts b/desktop/playwright.config.ts index a9ec17a249f..e930f0ef612 100644 --- a/desktop/playwright.config.ts +++ b/desktop/playwright.config.ts @@ -77,6 +77,7 @@ export default defineConfig({ "**/relay-connectivity.spec.ts", "**/unread-pill.spec.ts", "**/sidebar-more-unread-overlap.spec.ts", + "**/sidebar-snapshot.spec.ts", "**/home-collapsed-top-chrome.spec.ts", "**/top-chrome-zoom-clearance.spec.ts", "**/thread-unread.spec.ts", diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 9833847ece5..6c206264679 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -210,7 +210,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -221,7 +221,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1084,7 +1084,7 @@ dependencies = [ [[package]] name = "buzz-desktop" -version = "0.5.10" +version = "0.5.11" dependencies = [ "anyhow", "arboard", @@ -1627,7 +1627,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -2305,7 +2305,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" dependencies = [ "data-encoding", - "syn 2.0.118", + "syn 1.0.109", ] [[package]] @@ -2483,7 +2483,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2764,7 +2764,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3282,7 +3282,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows-link 0.2.1", + "windows-link 0.1.3", "windows-result 0.4.1", ] @@ -5777,7 +5777,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "536bfad37a309d62069485248eeaba1e8d9853aaf951caaeaed0585a95346f08" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5936,7 +5936,7 @@ dependencies = [ "png 0.18.1", "serde", "thiserror 2.0.18", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6449,7 +6449,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7153,7 +7153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.45.0", ] [[package]] @@ -8179,7 +8179,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8920,7 +8920,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8990,7 +8990,7 @@ dependencies = [ "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9257,7 +9257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9897,7 +9897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -10991,10 +10991,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -11016,7 +11016,7 @@ dependencies = [ "parking_lot", "rustix 1.1.4", "signal-hook 0.3.18", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -11758,7 +11758,7 @@ dependencies = [ "png 0.18.1", "serde", "thiserror 2.0.18", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -11859,7 +11859,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -12470,15 +12470,15 @@ dependencies = [ [[package]] name = "webbrowser" -version = "1.2.1" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc95580916af1e68ff6a7be07446fc5db73ebf71cf092de939bbf5f7e189f72" +checksum = "62c35be770821a214dbc362fc26908c853e776c0004294d0b10b8a6bad582f94" dependencies = [ - "core-foundation 0.10.1", "jni 0.22.4", "log", "ndk-context", "objc2", + "objc2-app-kit", "objc2-foundation", "url", "web-sys", @@ -12710,7 +12710,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 31092de99a9..54676458737 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -7,7 +7,7 @@ members = ["crates/buzz-terminal"] [package] name = "buzz-desktop" -version = "0.5.10" +version = "0.5.11" description = "Buzz desktop app" authors = ["you"] edition = "2021" diff --git a/desktop/src-tauri/src/commands/agent_models_databricks.rs b/desktop/src-tauri/src/commands/agent_models_databricks.rs index 63b4564e61d..4b6e512c059 100644 --- a/desktop/src-tauri/src/commands/agent_models_databricks.rs +++ b/desktop/src-tauri/src/commands/agent_models_databricks.rs @@ -1,7 +1,8 @@ //! Databricks v1/v2 model discovery and interactive reauthentication. -use std::collections::BTreeMap; -use std::sync::LazyLock; +use std::collections::{BTreeMap, HashMap}; +use std::sync::{LazyLock, Mutex, MutexGuard}; +use std::time::{Duration, Instant}; use crate::commands::agent_models_env::{ env_or_process_value, redaction_env_with_value, DiscoveryProvider, @@ -13,6 +14,77 @@ use crate::managed_agents::AgentModelsResponse; // callback listener/browser flow for the process-wide OAuth cache. static AUTH_GATE: LazyLock> = LazyLock::new(|| tokio::sync::Mutex::new(())); +// Hard cap on the interactive browser flow launched from a discovery surface. +// An abandoned SSO tab must fail discovery cleanly rather than wedge the +// dropdown forever. (`authenticate_databricks` has its own 60s callback wait; +// this outer bound also covers endpoint discovery and token exchange.) +const AUTH_FLOW_TIMEOUT: Duration = Duration::from_secs(150); + +// How long a failed/cancelled interactive sign-in suppresses re-launching the +// browser from passive surfaces. +pub(super) const AUTH_COOLDOWN: Duration = Duration::from_secs(5 * 60); + +/// Per-host record of a recently failed, cancelled, or timed-out interactive +/// sign-in. +/// +/// Passive discovery surfaces fire on every form-state change, so without this +/// a cancelled SSO page would re-pop the browser on the very next keystroke. +/// Entries expire so a genuine later retry still launches; the saved-model +/// picker bypasses the cooldown and a success clears it. +#[derive(Default)] +pub(super) struct AuthCooldown { + until: Mutex>, +} + +impl AuthCooldown { + fn map(&self) -> MutexGuard<'_, HashMap> { + // The critical sections below are panic-free map ops, so recover from a + // poisoned lock rather than wedge every future sign-in on one panic. + self.until + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub(super) fn is_active(&self, host: &str, now: Instant) -> bool { + let mut map = self.map(); + match map.get(host) { + Some(&expiry) if now < expiry => true, + Some(_) => { + map.remove(host); + false + } + None => false, + } + } + + pub(super) fn record(&self, host: &str, now: Instant) { + self.map().insert(host.to_string(), now + AUTH_COOLDOWN); + } + + pub(super) fn clear(&self, host: &str) { + self.map().remove(host); + } + + /// Whether the interactive browser flow may launch now under `auth_intent`. + /// Passive surfaces are suppressed while a per-host cooldown is active; the + /// explicit picker path always launches and clears any stale suppression. + pub(super) fn permits_launch( + &self, + auth_intent: DatabricksAuthIntent, + host: &str, + now: Instant, + ) -> bool { + if auth_intent.respects_cooldown() { + !self.is_active(host, now) + } else { + self.clear(host); + true + } + } +} + +static AUTH_COOLDOWNS: LazyLock = LazyLock::new(AuthCooldown::default); + pub(super) fn is_databricks_provider(provider: Option<&str>) -> bool { matches!( provider @@ -50,8 +122,14 @@ pub(super) enum DatabricksAuthIntent { } impl DatabricksAuthIntent { - fn allows_interactive_auth(self) -> bool { - matches!(self, Self::InteractiveModelPicker) + /// Passive draft discovery honors (and, on failure, writes) the per-host + /// cooldown so a cancelled SSO page does not re-pop on the next form + /// keystroke. The saved-model picker is an explicit user action, so it + /// bypasses the cooldown and clears it before launching. Both surfaces + /// launch the browser flow (Phase 2 goose-parity); this predicate is the + /// only behavioral difference between them. + fn respects_cooldown(self) -> bool { + matches!(self, Self::PassiveDraftDiscovery) } } @@ -60,11 +138,16 @@ pub(super) fn databricks_sign_in_required_error() -> String { .to_string() } -pub(super) fn should_start_interactive_auth( - api_key: &str, - auth_intent: DatabricksAuthIntent, -) -> bool { - api_key.is_empty() && auth_intent.allows_interactive_auth() +pub(super) fn databricks_sign_in_timed_out_error() -> String { + "Databricks sign-in timed out; open the model picker to retry, or run `buzz-agent auth databricks`" + .to_string() +} + +pub(super) fn should_start_interactive_auth(api_key: &str) -> bool { + // Phase 2: both discovery surfaces launch the browser flow when no static + // token is configured. Which surface is allowed to actually pop the browser + // (vs. respect a cooldown) is decided via `AuthCooldown::permits_launch`. + api_key.is_empty() } pub(super) async fn discover_databricks_models( @@ -93,22 +176,26 @@ pub(super) async fn discover_databricks_models( let entries = match buzz_agent_pkg::discover_databricks_models(&config).await { Ok(entries) => entries, - Err(buzz_agent_pkg::AgentError::LlmAuth(_)) - if should_start_interactive_auth(&api_key, auth_intent) => - { + Err(buzz_agent_pkg::AgentError::LlmAuth(_)) if should_start_interactive_auth(&api_key) => { let _auth = AUTH_GATE.lock().await; match buzz_agent_pkg::discover_databricks_models(&config).await { + // A peer sign-in under the gate already succeeded. Ok(entries) => entries, Err(buzz_agent_pkg::AgentError::LlmAuth(_)) => { - buzz_agent_pkg::authenticate_databricks(&host) - .await - .map_err(|error| { - format_redacted_error( - "Databricks sign-in failed", - &error, - &redaction_env, - ) - })?; + // Passive surfaces suppress the browser while a recent + // failure/cancel is cooling down; the explicit picker path + // always launches (and clears any stale cooldown). + if !AUTH_COOLDOWNS.permits_launch(auth_intent, &host, Instant::now()) { + return Err(databricks_sign_in_required_error()); + } + run_interactive_databricks_auth( + buzz_agent_pkg::authenticate_databricks(&host), + AUTH_FLOW_TIMEOUT, + &AUTH_COOLDOWNS, + &host, + &redaction_env, + ) + .await?; buzz_agent_pkg::discover_databricks_models(&config) .await .map_err(|error| { @@ -172,3 +259,43 @@ fn format_redacted_error( let message = crate::managed_agents::redact_env_values_in(&error.to_string(), redaction_env); format!("{context}: {message}") } + +/// Run the interactive browser OAuth flow under a hard timeout and maintain the +/// per-host cooldown. Success clears the cooldown; a failure, cancel, or +/// timeout records it so passive surfaces stop re-launching the browser on the +/// next form keystroke. `timeout` is injected (production passes +/// [`AUTH_FLOW_TIMEOUT`]) so the timeout/cooldown policy is unit-testable +/// without a live browser. +pub(super) async fn run_interactive_databricks_auth( + auth: Fut, + timeout: Duration, + cooldowns: &AuthCooldown, + host: &str, + redaction_env: &BTreeMap, +) -> Result<(), String> +where + Fut: std::future::Future>, +{ + match tokio::time::timeout(timeout, auth).await { + Ok(Ok(())) => { + cooldowns.clear(host); + Ok(()) + } + Ok(Err(error)) => { + cooldowns.record(host, Instant::now()); + Err(format_redacted_error( + "Databricks sign-in failed", + &error, + redaction_env, + )) + } + Err(_elapsed) => { + cooldowns.record(host, Instant::now()); + Err(databricks_sign_in_timed_out_error()) + } + } +} + +#[cfg(test)] +#[path = "agent_models_databricks_tests.rs"] +mod tests; diff --git a/desktop/src-tauri/src/commands/agent_models_databricks_tests.rs b/desktop/src-tauri/src/commands/agent_models_databricks_tests.rs new file mode 100644 index 00000000000..cb530ec59bd --- /dev/null +++ b/desktop/src-tauri/src/commands/agent_models_databricks_tests.rs @@ -0,0 +1,109 @@ +//! Cooldown and interactive-auth policy tests for Databricks discovery. +//! +//! Housed as a child of `agent_models_databricks` (not the shared +//! `agent_models_tests`) so the async timeout/cooldown cases sit next to the +//! code they exercise and reach its `pub(super)` items directly via +//! `use super::*` — and so the shared test file stays under its size ratchet. + +use super::*; + +#[test] +fn databricks_cooldown_suppresses_passive_relaunch_but_never_the_picker() { + let cooldowns = AuthCooldown::default(); + let host = "https://example.cloud.databricks.com"; + let now = Instant::now(); + + // A fresh host permits either surface to launch. + assert!(cooldowns.permits_launch(DatabricksAuthIntent::PassiveDraftDiscovery, host, now)); + assert!(cooldowns.permits_launch(DatabricksAuthIntent::InteractiveModelPicker, host, now)); + + // After a failed/cancelled attempt, passive discovery must NOT re-pop the + // browser while the window is active... + cooldowns.record(host, now); + assert!(!cooldowns.permits_launch(DatabricksAuthIntent::PassiveDraftDiscovery, host, now)); + + // ...but an explicit picker click always launches, and clears the window so + // a later passive read is unblocked too. + assert!(cooldowns.permits_launch(DatabricksAuthIntent::InteractiveModelPicker, host, now)); + assert!(cooldowns.permits_launch(DatabricksAuthIntent::PassiveDraftDiscovery, host, now)); +} + +#[test] +fn databricks_cooldown_expires_after_its_window_and_is_host_scoped() { + let cooldowns = AuthCooldown::default(); + let host = "https://a.cloud.databricks.com"; + let other = "https://b.cloud.databricks.com"; + let now = Instant::now(); + + cooldowns.record(host, now); + // A cooldown on one host never suppresses another. + assert!(!cooldowns.is_active(other, now)); + assert!(cooldowns.is_active(host, now)); + + // The window is closed the instant it elapses, so a genuine later retry + // launches again. + let after = now + AUTH_COOLDOWN; + assert!(!cooldowns.is_active(host, after)); +} + +#[tokio::test] +async fn databricks_interactive_auth_success_clears_a_prior_cooldown() { + let cooldowns = AuthCooldown::default(); + let host = "https://example.cloud.databricks.com"; + let redaction = BTreeMap::new(); + cooldowns.record(host, Instant::now()); + + let result = run_interactive_databricks_auth( + async { Ok(()) }, + Duration::from_secs(150), + &cooldowns, + host, + &redaction, + ) + .await; + + assert!(result.is_ok()); + assert!(!cooldowns.is_active(host, Instant::now())); +} + +#[tokio::test] +async fn databricks_interactive_auth_failure_records_a_cooldown() { + let cooldowns = AuthCooldown::default(); + let host = "https://example.cloud.databricks.com"; + let redaction = BTreeMap::new(); + + let result = run_interactive_databricks_auth( + async { Err(buzz_agent_pkg::AgentError::LlmAuth("closed the tab".into())) }, + Duration::from_secs(150), + &cooldowns, + host, + &redaction, + ) + .await; + + let error = result.expect_err("a failed sign-in must surface an error"); + assert!(error.contains("Databricks sign-in failed")); + assert!(cooldowns.is_active(host, Instant::now())); +} + +#[tokio::test(start_paused = true)] +async fn databricks_interactive_auth_timeout_records_cooldown_and_returns_timeout_copy() { + let cooldowns = AuthCooldown::default(); + let host = "https://example.cloud.databricks.com"; + let redaction = BTreeMap::new(); + + // An abandoned SSO tab: the flow never resolves. Under the paused clock the + // injected timeout fires deterministically without real waiting. + let result = run_interactive_databricks_auth( + std::future::pending::>(), + Duration::from_secs(150), + &cooldowns, + host, + &redaction, + ) + .await; + + let error = result.expect_err("a timed-out sign-in must surface an error"); + assert_eq!(error, databricks_sign_in_timed_out_error()); + assert!(cooldowns.is_active(host, Instant::now())); +} diff --git a/desktop/src-tauri/src/commands/agent_models_tests.rs b/desktop/src-tauri/src/commands/agent_models_tests.rs index e7d0e70fd0b..6226acfd964 100644 --- a/desktop/src-tauri/src/commands/agent_models_tests.rs +++ b/desktop/src-tauri/src/commands/agent_models_tests.rs @@ -577,19 +577,12 @@ fn is_databricks_provider_matches_both_variants() { } #[test] -fn databricks_interactive_auth_requires_explicit_intent_and_no_static_token() { - assert!(should_start_interactive_auth( - "", - DatabricksAuthIntent::InteractiveModelPicker - )); - assert!(!should_start_interactive_auth( - "", - DatabricksAuthIntent::PassiveDraftDiscovery - )); - assert!(!should_start_interactive_auth( - "static-token", - DatabricksAuthIntent::InteractiveModelPicker - )); +fn databricks_interactive_auth_launches_only_without_a_static_token() { + // Phase 2: both surfaces launch the browser flow when the token is empty; + // the surface distinction is now cooldown-only (asserted separately). A + // configured static token still short-circuits interactive auth entirely. + assert!(should_start_interactive_auth("")); + assert!(!should_start_interactive_auth("static-token")); } #[test] diff --git a/desktop/src-tauri/src/managed_agents/agent_env.rs b/desktop/src-tauri/src/managed_agents/agent_env.rs index 05979e76cbf..59b300d9d17 100644 --- a/desktop/src-tauri/src/managed_agents/agent_env.rs +++ b/desktop/src-tauri/src/managed_agents/agent_env.rs @@ -8,6 +8,25 @@ use std::collections::BTreeMap; use base64::Engine as _; +/// Seconds a woken lazy harness stays warm before it releases its worker +/// subprocesses back to the empty-slot state (via `BUZZ_ACP_IDLE_POOL_SLEEP`). +/// The next accepted event re-wakes it through the same lazy path. Matches the +/// harness's own 15-minute per-turn idle window so a warm pool survives a +/// normal back-and-forth but a truly quiet harness stops paying for workers. +const IDLE_POOL_SLEEP_SECS: &str = "900"; + +/// Value for `BUZZ_ACP_IDLE_POOL_SLEEP`. Idle re-sleep is only meaningful for +/// lazy harnesses (the harness ignores it otherwise); gate to `lazy` here so +/// the env reads inert (`"0"` = disabled) for eager harnesses. This is a +/// desktop-owned lifetime policy (reserved key), not user-tunable. +pub(super) fn idle_pool_sleep_env(lazy: bool) -> &'static str { + if lazy { + IDLE_POOL_SLEEP_SECS + } else { + "0" + } +} + /// Return the baked-in build-time env pairs as a map. /// /// Internal builds (buzz-releases) bake provider/model defaults and arbitrary diff --git a/desktop/src-tauri/src/managed_agents/env_vars/tests.rs b/desktop/src-tauri/src/managed_agents/env_vars/tests.rs index 34cdfede2c2..f3de11ad242 100644 --- a/desktop/src-tauri/src/managed_agents/env_vars/tests.rs +++ b/desktop/src-tauri/src/managed_agents/env_vars/tests.rs @@ -164,7 +164,11 @@ fn reserved_keys_include_respond_to_gate() { #[test] fn reserved_keys_include_remote_lifetime_policy() { - for key in ["BUZZ_ACP_EXIT_AFTER_INACTIVITY", "BUZZ_ACP_NO_PRESENCE"] { + for key in [ + "BUZZ_ACP_EXIT_AFTER_INACTIVITY", + "BUZZ_ACP_IDLE_POOL_SLEEP", + "BUZZ_ACP_NO_PRESENCE", + ] { assert!(is_reserved_env_key(key), "{key} should be reserved"); let agent = map(&[(key, "0")]); assert!(merged_user_env(&BTreeMap::new(), &agent).is_empty()); diff --git a/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs b/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs index 8698d3a51d1..afaaa2b4eb3 100644 --- a/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs +++ b/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs @@ -59,6 +59,9 @@ pub(crate) const RESERVED_ENV_KEYS: &[&str] = &[ // Remote lifetime/presence policy: user env must not disable the // desktop/provider-owned bounds while the saved record still promises them. "BUZZ_ACP_EXIT_AFTER_INACTIVITY", + // Desktop-owned pool lifetime policy: user env must not disable or reset + // the idle worker-reclamation window while the desktop launcher sets it. + "BUZZ_ACP_IDLE_POOL_SLEEP", "BUZZ_ACP_NO_PRESENCE", // Readiness handoff: desktop is the ONLY readiness source. A saved or // ambient env var must not be able to forge setup mode (NotReady) on a diff --git a/desktop/src-tauri/src/managed_agents/runtime.rs b/desktop/src-tauri/src/managed_agents/runtime.rs index d95c5954177..b1c342e9955 100644 --- a/desktop/src-tauri/src/managed_agents/runtime.rs +++ b/desktop/src-tauri/src/managed_agents/runtime.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use tauri::AppHandle; -use super::agent_env::build_buzz_agent_provider_defaults; +use super::agent_env::{build_buzz_agent_provider_defaults, idle_pool_sleep_env}; use crate::{ managed_agents::{ @@ -531,6 +531,7 @@ pub fn spawn_agent_child( command.env("BUZZ_PRIVATE_KEY", &record.private_key_nsec); command.env("BUZZ_RELAY_URL", &effective_relay_url); command.env("BUZZ_ACP_LAZY_POOL", if lazy { "true" } else { "false" }); + command.env("BUZZ_ACP_IDLE_POOL_SLEEP", idle_pool_sleep_env(lazy)); command.env("BUZZ_ACP_AGENT_COMMAND", &resolved_agent_command); command.env("BUZZ_ACP_AGENT_ARGS", agent_args.join(",")); match &resolved_mcp_command { diff --git a/desktop/src-tauri/tauri.conf.json b/desktop/src-tauri/tauri.conf.json index 856a2dbde73..643c69c2dd3 100644 --- a/desktop/src-tauri/tauri.conf.json +++ b/desktop/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "BitcoinMarkets", - "version": "0.5.10", + "version": "0.5.11", "identifier": "app.bitcoinmarkets.desktop", "build": { "beforeDevCommand": { diff --git a/desktop/src/app/AppShell.tsx b/desktop/src/app/AppShell.tsx index 73e40e8cd89..db10c12dc95 100644 --- a/desktop/src/app/AppShell.tsx +++ b/desktop/src/app/AppShell.tsx @@ -939,6 +939,7 @@ export function AppShell() { onSelectChannel={(channelId) => { void goChannel(channelId); }} + relayUrl={communitiesHook.activeCommunity?.relayUrl} /> { return { default: module.ChannelManagementSheet }; }); +const MembersSidebar = React.lazy(async () => { + const module = await import("@/features/channels/ui/MembersSidebar"); + return { default: module.MembersSidebar }; +}); + export type BrowseDialogType = "stream" | "forum" | null; type AppShellOverlaysProps = { @@ -30,6 +35,7 @@ type AppShellOverlaysProps = { onChannelManagementOpenChange: (open: boolean) => void; onDeleteActiveChannel: () => void; onSelectChannel: (channelId: string) => void; + relayUrl?: string; }; export function AppShellOverlays({ @@ -45,7 +51,11 @@ export function AppShellOverlays({ onChannelManagementOpenChange, onDeleteActiveChannel, onSelectChannel, + relayUrl, }: AppShellOverlaysProps) { + const [membersChannel, setMembersChannel] = React.useState( + null, + ); const [visibleBrowseDialogType, setVisibleBrowseDialogType] = React.useState(null); const { cancelDeferredModalOpen, openNextFrame: openModalNextFrame } = @@ -89,11 +99,28 @@ export function AppShellOverlays({ channel={activeChannel} currentPubkey={currentPubkey} onDeleted={onDeleteActiveChannel} + onOpenMembers={() => setMembersChannel(activeChannel)} onOpenChange={onChannelManagementOpenChange} open={true} /> ) : null} + + {membersChannel ? ( + + { + if (!nextOpen) { + setMembersChannel(null); + } + }} + open={true} + relayUrl={relayUrl} + /> + + ) : null} ); } diff --git a/desktop/src/app/useAppShellLifecycleEffects.ts b/desktop/src/app/useAppShellLifecycleEffects.ts index 2d56d24cb37..ae62ae3e224 100644 --- a/desktop/src/app/useAppShellLifecycleEffects.ts +++ b/desktop/src/app/useAppShellLifecycleEffects.ts @@ -1,6 +1,7 @@ import * as React from "react"; import { setDesktopAppBadge } from "@/features/notifications/lib/desktop"; +import { useForegroundQueryRefresh } from "@/features/workflows/hooks"; import { relayClient } from "@/shared/api/relayClient"; import { useRelayResumeTriggers } from "@/shared/api/useRelayResumeTriggers"; @@ -20,6 +21,7 @@ export function useAppShellLifecycleEffects({ // Event-driven reconnect: network online / focus / visibility short-circuit // the backoff timer when the relay session is degraded (CMD+R gap G1). useRelayResumeTriggers(); + useForegroundQueryRefresh(); // Prevent webview file:/// navigation on file drop outside the composer. // Scoped to file drags only (text drag-and-drop into inputs still works). diff --git a/desktop/src/features/agent-memory/ui/MemorySection.tsx b/desktop/src/features/agent-memory/ui/MemorySection.tsx index cfc5ed7bd92..d63ceeb3cd9 100644 --- a/desktop/src/features/agent-memory/ui/MemorySection.tsx +++ b/desktop/src/features/agent-memory/ui/MemorySection.tsx @@ -10,6 +10,7 @@ import { Skeleton } from "@/shared/ui/skeleton"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip"; const MEMORY_LIST_PREVIEW_LIMIT = 3; +type MemorySectionVariant = "cards" | "grouped"; const MEMORY_TRUNCATED_TOOLTIP = "This list may be incomplete — the relay returned the maximum number of memories."; @@ -41,15 +42,17 @@ const MEMORY_DANGLING_REF_TOOLTIP = */ export function MemorySection({ agentPubkey, + variant = "cards", viewerIsOwner, }: { agentPubkey: string; + variant?: MemorySectionVariant; viewerIsOwner: boolean; }): React.ReactElement | null { // Hide entirely for non-owners. if (!viewerIsOwner) return null; - return ; + return ; } export function MemoryRefreshButton({ @@ -92,7 +95,13 @@ export function MemoryRefreshButton({ ); } -function MemorySectionForOwner({ agentPubkey }: { agentPubkey: string }) { +function MemorySectionForOwner({ + agentPubkey, + variant, +}: { + agentPubkey: string; + variant: MemorySectionVariant; +}) { const { query, graph } = useAgentMemoryGraph(agentPubkey); // Order matters here. We want: @@ -107,13 +116,14 @@ function MemorySectionForOwner({ agentPubkey }: { agentPubkey: string }) { return (
- {showInitialSkeleton ? : null} + {showInitialSkeleton ? : null} {showInitialError ? ( query.refetch()} retrying={query.isFetching} + variant={variant} /> ) : null} @@ -123,10 +133,17 @@ function MemorySectionForOwner({ agentPubkey }: { agentPubkey: string }) { still have prior data on screen. Distinct from the initial error state above. */} {query.isError && !query.isFetching ? ( - query.refetch()} /> + query.refetch()} + variant={variant} + /> ) : null} - + ) : null}
@@ -135,11 +152,11 @@ function MemorySectionForOwner({ agentPubkey }: { agentPubkey: string }) { // ── Subviews ──────────────────────────────────────────────────────────────── -function MemorySkeleton() { +function MemorySkeleton({ variant }: { variant: MemorySectionVariant }) { return (
@@ -154,16 +171,21 @@ function MemoryErrorState({ error, onRetry, retrying, + variant, }: { error: unknown; onRetry: () => void; retrying: boolean; + variant: MemorySectionVariant; }) { const message = error instanceof Error ? error.message : String(error ?? "unknown error"); return (
@@ -189,10 +211,19 @@ function MemoryErrorState({ ); } -function MemoryStaleErrorBanner({ onRetry }: { onRetry: () => void }) { +function MemoryStaleErrorBanner({ + onRetry, + variant, +}: { + onRetry: () => void; + variant: MemorySectionVariant; +}) { return (
@@ -211,9 +242,11 @@ function MemoryStaleErrorBanner({ onRetry }: { onRetry: () => void }) { function MemoryGraphView({ graph, truncated, + variant, }: { graph: NonNullable["graph"]>; truncated: boolean; + variant: MemorySectionVariant; }) { const { rootedTree, orphans, dangling } = graph; const [showAllEntries, setShowAllEntries] = React.useState(false); @@ -250,10 +283,13 @@ function MemoryGraphView({ : entries.slice(0, MEMORY_LIST_PREVIEW_LIMIT); return ( -
+
{!core && memories.length > 0 ? (

No core memory yet — agent @@ -261,12 +297,18 @@ function MemoryGraphView({

) : null} -
+
{visibleEntries.map((entry) => ( ))}
@@ -276,14 +318,22 @@ function MemoryGraphView({ count={entries.length} onClick={() => setShowAllEntries(true)} truncated={truncated} + variant={variant} /> ) : null} - {truncated && !hasMoreEntries ? : null} + {truncated && !hasMoreEntries ? ( + + ) : null} {hasMoreEntries && showAllEntries ? ( + ) : channel.channelType !== "dm" ? ( +
+

+ {channel.name} +

+ {description ? ( +

+ {description} +

+ ) : null} +
+ ) : null}
); } -export function ChannelQuickAction({ - active, - disabled, - icon: Icon, - label, - onClick, +export function FieldGroup({ + children, + description, testId, + title, }: { - active?: boolean; - disabled?: boolean; - icon: LucideIcon; - label: string; - onClick: () => void; + children: React.ReactNode; + description?: React.ReactNode; testId?: string; + title?: React.ReactNode; }) { return ( - - ); -} - -export function FieldGroup({ children }: { children: React.ReactNode }) { - return ( -
{children}
+ + {children} + ); } @@ -114,26 +137,51 @@ export function getMarkdownPreviewText(content: string) { .join(" "); } +function truncateIdentifier(value: string) { + if (value.length <= 12) return value; + return `${value.slice(0, 8)}…${value.slice(-4)}`; +} + export function CopyFieldRow({ icon: Icon, label, value, testId, }: { - icon: LucideIcon; + icon?: LucideIcon; label: string; value: string; testId?: string; }) { + const [copied, setCopied] = React.useState(false); + const resetTimerRef = React.useRef(null); + + React.useEffect( + () => () => { + if (resetTimerRef.current !== null) { + window.clearTimeout(resetTimerRef.current); + } + }, + [], + ); + async function handleCopy() { await writeTextToClipboard(value); + setCopied(true); + if (resetTimerRef.current !== null) { + window.clearTimeout(resetTimerRef.current); + } + resetTimerRef.current = window.setTimeout(() => { + setCopied(false); + resetTimerRef.current = null; + }, 1_500); toast.success(`Copied ${label.toLowerCase()}`); } return ( ); } @@ -160,73 +235,203 @@ export function CopyFieldRow({ export function InfoFieldRow({ icon: Icon, label, + multiline = false, + onClick, + trailing, value, testId, }: { - icon: LucideIcon; + icon?: LucideIcon; label: string; + multiline?: boolean; + onClick?: () => void; + trailing?: React.ReactNode; value: string; testId?: string; }) { - return ( -
- - - + const content = ( + <> + {Icon ? ( + + ) : null} - + {label} - + {value} -
+ {trailing} + ); -} -export function NarrativeGroup({ children }: { children: React.ReactNode }) { + if (onClick) { + return ( + + ); + } + return ( -
{children}
+
+ {content} +
); } -export function NarrativeField({ +export function EditableInfoFieldRow({ + editTestId, icon: Icon, label, + multiline = false, + onEdit, value, testId, }: { - icon: LucideIcon; + editTestId?: string; + icon?: LucideIcon; label: string; + multiline?: boolean; + onEdit?: () => void; value: string; testId: string; }) { - return ( -
- - - - - + const content = ( + <> + {Icon ? ( + + ) : null} + + {label} - + {value} + {onEdit ? ( + + ) : null} + + ); + + if (onEdit) { + return ( + + ); + } + + return ( +
+ {content}
); } +type ActionFieldRowProps = { + destructive?: boolean; + description?: string; + disabled?: boolean; + icon: LucideIcon; + label: string; + onClick?: () => void; + testId: string; +}; + +export const ActionFieldRow = React.forwardRef< + HTMLButtonElement, + ActionFieldRowProps +>(function ActionFieldRow( + { + destructive = false, + description, + disabled, + icon: Icon, + label, + onClick, + testId, + ...triggerProps + }, + ref, +) { + return ( + + ); +}); + export function IngressRow({ description, + helpText, icon: Icon, label, onClick, @@ -234,6 +439,7 @@ export function IngressRow({ trailing, }: { description?: string; + helpText?: string; icon: LucideIcon; label: string; onClick: () => void; @@ -241,29 +447,51 @@ export function IngressRow({ trailing?: string; }) { return ( - + + + {helpText} + + + ) : null} +
+ {description ? ( + + {description} + + ) : null}
- {description ? ( - - {description} + {trailing ? ( + + {trailing} ) : null} - - {trailing ? ( - {trailing} - ) : null} - - + +
+ ); } diff --git a/desktop/src/features/channels/ui/ChannelMemberAvatarStack.tsx b/desktop/src/features/channels/ui/ChannelMemberAvatarStack.tsx new file mode 100644 index 00000000000..a9662bbf047 --- /dev/null +++ b/desktop/src/features/channels/ui/ChannelMemberAvatarStack.tsx @@ -0,0 +1,75 @@ +import * as React from "react"; + +import { useUsersBatchQuery } from "@/features/profile/hooks"; +import { resolveUserLabel } from "@/features/profile/lib/identity"; +import type { ChannelMember } from "@/shared/api/types"; +import { normalizePubkey } from "@/shared/lib/pubkey"; +import { UserAvatar } from "@/shared/ui/UserAvatar"; + +const MAX_VISIBLE_AVATARS = 3; + +export function ChannelMemberAvatarStack({ + currentPubkey, + members, +}: { + currentPubkey?: string; + members: ChannelMember[]; +}) { + const visibleMembers = members.slice(0, MAX_VISIBLE_AVATARS); + const visiblePubkeys = React.useMemo( + () => members.slice(0, MAX_VISIBLE_AVATARS).map((member) => member.pubkey), + [members], + ); + const profilesQuery = useUsersBatchQuery(visiblePubkeys); + const profiles = profilesQuery.data?.profiles; + const overflowCount = members.length - visibleMembers.length; + const stackItemCount = visibleMembers.length + (overflowCount > 0 ? 1 : 0); + + if (members.length === 0) { + return null; + } + + return ( +
+ {visibleMembers.map((member, index) => { + const normalizedPubkey = normalizePubkey(member.pubkey); + const profile = profiles?.[normalizedPubkey]; + const label = resolveUserLabel({ + currentPubkey, + fallbackName: member.displayName, + profiles, + pubkey: member.pubkey, + }); + + return ( + 0 ? "-ml-2" : ""} + data-testid="channel-management-member-avatar" + key={normalizedPubkey} + style={{ zIndex: index + 1 }} + > + + + ); + })} + {overflowCount > 0 ? ( + + +{overflowCount} + + ) : null} +
+ ); +} diff --git a/desktop/src/features/channels/ui/ChannelPane.tsx b/desktop/src/features/channels/ui/ChannelPane.tsx index 28f3f8aae7a..8fc7cfaf51a 100644 --- a/desktop/src/features/channels/ui/ChannelPane.tsx +++ b/desktop/src/features/channels/ui/ChannelPane.tsx @@ -762,6 +762,7 @@ export const ChannelPane = React.memo(function ChannelPane({ key="channel-management-panel" onChannelManagementDeleted={onChannelManagementDeleted} onCloseChannelManagement={onCloseChannelManagement} + onOpenMembers={onOpenMembers} onResetThreadPanelWidth={onResetThreadPanelWidth} onThreadPanelResizeStart={onThreadPanelResizeStart} threadPanelWidthPx={threadPanelWidthPx} diff --git a/desktop/src/features/channels/ui/channelFormStyles.ts b/desktop/src/features/channels/ui/channelFormStyles.ts index 60e4053901a..df07001293c 100644 --- a/desktop/src/features/channels/ui/channelFormStyles.ts +++ b/desktop/src/features/channels/ui/channelFormStyles.ts @@ -2,4 +2,4 @@ export const CHANNEL_FORM_FIELD_SHELL_CLASS = "rounded-xl border border-input bg-muted/40 transition-colors duration-150 ease-out hover:border-muted-foreground/40 focus-within:border-muted-foreground/50"; export const CHANNEL_FORM_FIELD_CONTROL_CLASS = - "border-0 bg-transparent text-muted-foreground/55 shadow-none outline-none ring-0 transition-colors duration-150 ease-out placeholder:text-muted-foreground/55 focus:bg-transparent focus:text-foreground focus:outline-hidden focus-visible:ring-0"; + "border-0 bg-transparent text-foreground shadow-none outline-none ring-0 transition-colors duration-150 ease-out placeholder:text-muted-foreground/55 focus:bg-transparent focus:text-foreground focus:outline-hidden focus-visible:ring-0"; diff --git a/desktop/src/features/channels/useThreadActivityPersistence.test.mjs b/desktop/src/features/channels/useThreadActivityPersistence.test.mjs new file mode 100644 index 00000000000..686a08fcc42 --- /dev/null +++ b/desktop/src/features/channels/useThreadActivityPersistence.test.mjs @@ -0,0 +1,276 @@ +/** + * Integration tests for useThreadActivityPersistence. + * + * These mount the REAL production hook via createRoot + act to exercise the + * actual lifecycle: pagehide flush, visibilitychange→hidden flush, unmount + * cleanup, scope-switch hydration (flush-before-reseed), legacy-key cleanup, + * and the read-live-buffer-at-flush-time contract that distinguishes this + * scheduler from a snapshot-at-schedule one. Debounce timing is covered by + * fake timers in threadActivityWriteScheduler.test.mjs. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + installDOMShim, + installFreshStorage, +} from "./observedUnreadTestHarness.mjs"; + +installDOMShim(); +installFreshStorage(); + +import React from "react"; +import { act } from "react"; +import { createRoot } from "react-dom/client"; + +import { + activityStorageKey, + readActivityFromStorage, + writeActivityToStorage, +} from "./threadActivityStorage.ts"; +import { useThreadActivityPersistence } from "./useThreadActivityPersistence.ts"; + +const RELAY = "wss://relay.example.com"; + +// Mount the hook with a caller-owned buffer ref (mirrors useUnreadChannels, +// which owns threadActivityRef and passes it in). +async function mountHook(itemsRef, props) { + const apiRef = { current: null }; + + function Harness({ pubkey, relay }) { + apiRef.current = useThreadActivityPersistence(pubkey, relay, itemsRef); + return null; + } + + const root = createRoot(document.createElement("div")); + const render = async (p) => { + await act(async () => { + root.render(React.createElement(Harness, p)); + }); + }; + await render(props); + + return { + get api() { + return apiRef.current; + }, + render, + unmount: async () => { + await act(async () => root.unmount()); + }, + }; +} + +// ── flush contract: read the live buffer at flush time ─────────────────────── + +test("pagehide flush persists the live buffer's final state, not a schedule-time snapshot", async () => { + installFreshStorage(); + const itemsRef = { current: [] }; + const harness = await mountHook(itemsRef, { pubkey: "pk1", relay: RELAY }); + + // A reply lands, the writer buffers it in place and arms a coalesced write. + itemsRef.current = [{ id: "early" }]; + harness.api.schedule(harness.api.currentScope); + // A second reply lands within the debounce window — buffer grows in place. + itemsRef.current = [{ id: "early" }, { id: "late" }]; + + await act(async () => { + globalThis.dispatchEvent({ type: "pagehide" }); + }); + + assert.deepEqual( + readActivityFromStorage("pk1", RELAY).map((item) => item.id), + ["early", "late"], + "flush must persist the buffer's final state including the late reply", + ); + + await harness.unmount(); +}); + +test("visibilitychange to hidden flushes a pending write", async () => { + installFreshStorage(); + const itemsRef = { current: [] }; + const harness = await mountHook(itemsRef, { pubkey: "pk1", relay: RELAY }); + + itemsRef.current = [{ id: "hidden-flush" }]; + harness.api.schedule(harness.api.currentScope); + + await act(async () => { + globalThis.document.visibilityState = "hidden"; + globalThis.document.dispatchEvent({ type: "visibilitychange" }); + }); + + assert.deepEqual( + readActivityFromStorage("pk1", RELAY).map((item) => item.id), + ["hidden-flush"], + "backgrounding the webview must persist the pending buffer", + ); + + await harness.unmount(); +}); + +test("visibilitychange to visible does not write", async () => { + installFreshStorage(); + const itemsRef = { current: [] }; + const harness = await mountHook(itemsRef, { pubkey: "pk1", relay: RELAY }); + + itemsRef.current = [{ id: "still-buffered" }]; + harness.api.schedule(harness.api.currentScope); + + await act(async () => { + globalThis.document.visibilityState = "visible"; + globalThis.document.dispatchEvent({ type: "visibilitychange" }); + }); + + assert.deepEqual( + readActivityFromStorage("pk1", RELAY), + [], + "a visible transition must not flush", + ); + + await harness.unmount(); +}); + +test("unmount with a pending write flushes before teardown", async () => { + installFreshStorage(); + const itemsRef = { current: [] }; + const harness = await mountHook(itemsRef, { pubkey: "pk1", relay: RELAY }); + + itemsRef.current = [{ id: "unmount-flush" }]; + harness.api.schedule(harness.api.currentScope); + + await harness.unmount(); + + assert.deepEqual( + readActivityFromStorage("pk1", RELAY).map((item) => item.id), + ["unmount-flush"], + "unmount cleanup must flush the pending write", + ); +}); + +// ── scope switch: flush-before-reseed under the OLD key ────────────────────── + +test("scope switch flushes A synchronously under A's key and does not leak A into B", async () => { + installFreshStorage(); + + const pkA = "pkA"; + const relayA = "wss://relay-a.example.com"; + const pkB = "pkB"; + const relayB = "wss://relay-b.example.com"; + + const itemsRef = { current: [] }; + const harness = await mountHook(itemsRef, { pubkey: pkA, relay: relayA }); + + // A accumulates a buffered reply with a pending coalesced write. + itemsRef.current = [{ id: "a-only" }]; + harness.api.schedule(harness.api.currentScope); + + // Switch identity to B: the hydration effect must flush A first. + await harness.render({ pubkey: pkB, relay: relayB }); + + assert.deepEqual( + readActivityFromStorage(pkA, relayA).map((item) => item.id), + ["a-only"], + "A's pending write must land under A's key on scope switch", + ); + assert.deepEqual( + readActivityFromStorage(pkB, relayB), + [], + "B's bucket must not contain A's rows", + ); + assert.ok( + harness.api.currentScope.includes(pkB), + "currentScope must reflect B after the switch", + ); + + await harness.unmount(); +}); + +test("scope switch hydrates B's buffer from B's persisted bucket", async () => { + installFreshStorage(); + + const pkA = "pkA"; + const relayA = "wss://relay-a.example.com"; + const pkB = "pkB"; + const relayB = "wss://relay-b.example.com"; + writeActivityToStorage(pkB, relayB, [{ id: "b-persisted" }]); + + const itemsRef = { current: [] }; + const harness = await mountHook(itemsRef, { pubkey: pkA, relay: relayA }); + + itemsRef.current = [{ id: "a-only" }]; + await harness.render({ pubkey: pkB, relay: relayB }); + + assert.deepEqual( + itemsRef.current.map((item) => item.id), + ["b-persisted"], + "the buffer must be reseeded from B's bucket, dropping A's rows", + ); + + await harness.unmount(); +}); + +// ── legacy key cleanup ─────────────────────────────────────────────────────── + +test("mounting removes the orphaned legacy pubkey-only key", async () => { + const ls = installFreshStorage(); + const pubkey = "pk-legacy"; + const legacyKey = `buzz-thread-activity.v1:${pubkey}`; + ls.setItem(legacyKey, JSON.stringify([{ id: "stale" }])); + + const itemsRef = { current: [] }; + const harness = await mountHook(itemsRef, { pubkey, relay: RELAY }); + + assert.equal( + ls.getItem(legacyKey), + null, + "hydration must drop the orphaned legacy key", + ); + + await harness.unmount(); +}); + +// ── scope fence: never write before a valid scope is loaded ────────────────── + +test("isScopeLoaded is false without an identity and true after hydration", async () => { + installFreshStorage(); + const itemsRef = { current: [] }; + const harness = await mountHook(itemsRef, { pubkey: null, relay: RELAY }); + + assert.equal( + harness.api.isScopeLoaded(), + false, + "an absent pubkey must never pass the scope fence", + ); + + await harness.render({ pubkey: "pk1", relay: RELAY }); + assert.equal( + harness.api.isScopeLoaded(), + true, + "a valid scope must pass once its hydration effect commits", + ); + + await harness.unmount(); +}); + +test("schedule under an empty scope never writes", async () => { + const ls = installFreshStorage(); + const itemsRef = { current: [] }; + const harness = await mountHook(itemsRef, { pubkey: null, relay: RELAY }); + + itemsRef.current = [{ id: "orphan" }]; + harness.api.schedule(harness.api.currentScope); + + await act(async () => { + globalThis.dispatchEvent({ type: "pagehide" }); + }); + + assert.equal( + ls.getItem(activityStorageKey("", RELAY)), + null, + "no write may land when identity is unknown", + ); + + await harness.unmount(); +}); diff --git a/desktop/src/features/channels/useThreadActivityPersistence.ts b/desktop/src/features/channels/useThreadActivityPersistence.ts new file mode 100644 index 00000000000..025a82467e5 --- /dev/null +++ b/desktop/src/features/channels/useThreadActivityPersistence.ts @@ -0,0 +1,113 @@ +import * as React from "react"; +import { + activityScopeKey, + flushThreadActivityWrite, + readActivityFromStorage, + removeLegacyThreadActivityKey, + scheduleThreadActivityWrite, + type ThreadActivityItem, + type ThreadActivityRefs, +} from "@/features/channels/threadActivityStorage"; + +export type ThreadActivityPersistence = { + /** Scope key loaded into the buffer ("" until identity is known). */ + scopeLoadedRef: React.MutableRefObject; + /** Current scope derived from normalized pubkey + relay. */ + currentScope: string; + /** + * True only when the hydration effect has committed for the current scope + * AND that scope is non-empty. Reads the ref at call time so it is never a + * stale snapshot. Use as the write/merge guard: an empty scope must never + * pass, or a writer could fire before the first valid scope is seeded. + */ + isScopeLoaded: () => boolean; + /** Arm a coalesced write after mutating the buffer for `scope`. */ + schedule: (scope: string) => void; +}; + +/** + * Manages the thread-activity localStorage persistence layer for + * useUnreadChannels: owns the loaded-scope ref, the coalescing timer, the + * pagehide/visibility flush, and hydration on identity/relay change. The buffer + * itself (`itemsRef`) is owned by the parent and merged in place by its writers; + * this hook only decides when the buffer is durably persisted. + * + * Sibling of useObservedUnreadPersistence — same scope-fence and flush shape, + * minus marker-prune/removeChannel/clearAll, which thread activity has no + * analog for. + */ +export function useThreadActivityPersistence( + normalizedPubkey: string | null, + normalizedRelayUrl: string, + itemsRef: React.MutableRefObject, +): ThreadActivityPersistence { + const currentScope = activityScopeKey(normalizedPubkey, normalizedRelayUrl); + + const scopeLoadedRef = React.useRef(""); + const timerRef = React.useRef | null>(null); + + const persistRefs = React.useRef({ + itemsRef, + scopeLoadedRef, + timerRef, + }); + persistRefs.current.itemsRef = itemsRef; + + // pagehide + visibilitychange→hidden: synchronously persist any pending write + // before the webview unloads or is backgrounded. Cmd+R and #5588's idle + // reload both tear the webview down within the coalescing window; without + // these flushes the last burst of replies would be lost. + React.useEffect(() => { + const refs = persistRefs.current; + const flush = () => flushThreadActivityWrite(refs); + const onVisibility = () => { + if (document.visibilityState === "hidden") flush(); + }; + window.addEventListener("pagehide", flush); + document.addEventListener("visibilitychange", onVisibility); + return () => { + window.removeEventListener("pagehide", flush); + document.removeEventListener("visibilitychange", onVisibility); + }; + }, []); + + // Hydrate the buffer whenever identity/relay changes. Flush the OLD scope + // first so an in-flight coalesced write lands under the old key before the + // buffer is clobbered, then drop the orphaned legacy key for this pubkey. + // biome-ignore lint/correctness/useExhaustiveDependencies: normalizedRelayUrl is an intentional reset signal alongside normalizedPubkey + React.useEffect(() => { + flushThreadActivityWrite(persistRefs.current); + + if (normalizedPubkey && normalizedRelayUrl) { + removeLegacyThreadActivityKey(normalizedPubkey); + itemsRef.current = readActivityFromStorage( + normalizedPubkey, + normalizedRelayUrl, + ); + } else { + itemsRef.current = []; + } + scopeLoadedRef.current = currentScope; + + // Flush the current scope on unmount / before the next run so a pending + // write is never dropped when refs are clobbered. + return () => { + flushThreadActivityWrite(persistRefs.current); + }; + }, [normalizedPubkey, normalizedRelayUrl]); + + const schedule = React.useCallback( + (scope: string) => scheduleThreadActivityWrite(scope, persistRefs.current), + [], + ); + + const isScopeLoaded = React.useCallback( + () => currentScope !== "" && scopeLoadedRef.current === currentScope, + [currentScope], + ); + + return React.useMemo( + () => ({ scopeLoadedRef, currentScope, isScopeLoaded, schedule }), + [currentScope, isScopeLoaded, schedule], + ); +} diff --git a/desktop/src/features/channels/useUnreadChannels.ts b/desktop/src/features/channels/useUnreadChannels.ts index d46cc71cdbe..0464a00fe0b 100644 --- a/desktop/src/features/channels/useUnreadChannels.ts +++ b/desktop/src/features/channels/useUnreadChannels.ts @@ -38,11 +38,8 @@ import { useStableMap, useStableSet } from "@/shared/hooks/useStableReference"; import { normalizeRelayUrl } from "@/features/profile/lib/selfProfileStorage"; import { DM_NOTIFIABLE_EVENT_KINDS } from "./isDmNotifiableKind"; import { - activityScopeKey, addThreadActivityItems, projectActivityForScope, - readActivityFromStorage, - writeActivityToStorage, type ThreadActivityItem, } from "@/features/channels/threadActivityStorage"; export type { ThreadActivityItem } from "@/features/channels/threadActivityStorage"; @@ -55,6 +52,7 @@ export { writeActivityToStorage, } from "@/features/channels/threadActivityStorage"; import { useObservedUnreadPersistence } from "@/features/channels/useObservedUnreadPersistence"; +import { useThreadActivityPersistence } from "@/features/channels/useThreadActivityPersistence"; type UseUnreadChannelsOptions = UseLiveChannelUpdatesOptions & { pubkey?: string; @@ -147,14 +145,6 @@ export function useUnreadChannels( const normalizedRelayUrl = relayUrlOption ? normalizeRelayUrl(relayUrlOption) : ""; - // Single identity for the in-memory thread-activity buffer — computed once - // per render and used at reset, both writers, and the return fence. The - // helper returns "" when either value is absent, which never matches a valid - // loaded scope, so the fence returns [] until the buffer is seeded. - const currentActivityScope = activityScopeKey( - normalizedPubkey, - normalizedRelayUrl, - ); const { getEffectiveTimestamp, @@ -227,12 +217,10 @@ export function useUnreadChannels( mutedChannelIdsRef.current = mutedChannelIdsOption ?? new Set(); // Thread reply events that triggered notifications — surfaced in the Home - // activity feed as synthetic FeedItems. + // activity feed as synthetic FeedItems. The buffer is the source of truth + // between coalesced writes; useThreadActivityPersistence owns the loaded + // scope, the write timer, flush, and hydration. const threadActivityRef = React.useRef([]); - // Tracks the (pubkey:relayUrl) scope currently loaded into threadActivityRef. - // Writers guard against this before merging so in-flight writes from a prior - // scope cannot corrupt the new one; renders return [] until it matches. - const threadActivityScopeRef = React.useRef(""); // Tracks which channels we've already issued a catch-up REQ for this // session. Prevents re-fetching on every channels-list refetch, while still @@ -266,6 +254,15 @@ export function useUnreadChannels( { onPruned: bumpLatestVersion }, ); + // Thread-activity persistence: coalesced writes, pagehide/visibility flush, + // hydration + legacy-key cleanup. Owns the loaded scope for the buffer above. + const activityPersistence = useThreadActivityPersistence( + normalizedPubkey, + normalizedRelayUrl, + threadActivityRef, + ); + const currentActivityScope = activityPersistence.currentScope; + // Reset all in-session state when the identity or relay changes. In-memory // caches are cleared; persisted stores are loaded for the new pubkey (so // forced-unread, participation, etc. are correct for the new identity). @@ -285,11 +282,6 @@ export function useUnreadChannels( ? mentionedStore.read(pubkey) : new Set(); mutedRootIdsRef.current = pubkey ? mutedStore.read(pubkey) : new Set(); - threadActivityRef.current = - normalizedPubkey && normalizedRelayUrl - ? readActivityFromStorage(normalizedPubkey, normalizedRelayUrl) - : []; - threadActivityScopeRef.current = currentActivityScope; bumpLatestVersion(); bumpMembershipVersion(); }, [pubkey, relayClient, normalizedRelayUrl]); @@ -490,15 +482,10 @@ export function useUnreadChannels( const handleThreadReplyNotification = React.useCallback( (channelId: string, event: RelayEvent) => { - // Guard: don't merge into a ref whose scope has drifted from the current - // identity. Also reject an empty scope — activityScopeKey() returns "" - // when pubkey or relay is absent, and "" !== "" is false, so without this - // guard a writer could fire before the first valid scope is established. - if ( - !currentActivityScope || - threadActivityScopeRef.current !== currentActivityScope - ) - return; + // Guard: don't merge into a buffer whose scope has drifted from the + // current identity. isScopeLoaded() also rejects an empty scope, so a + // writer can never fire before the first valid scope is seeded. + if (!activityPersistence.isScopeLoaded()) return; const channelName = channels.find((ch) => ch.id === channelId)?.name ?? ""; @@ -516,25 +503,13 @@ export function useUnreadChannels( if (!added.didAdd) return; const didRecordMentionedRoot = recordMentionedRoot(event); threadActivityRef.current = added.items; - if (normalizedPubkey !== null && normalizedRelayUrl) { - writeActivityToStorage( - normalizedPubkey, - normalizedRelayUrl, - added.items, - ); - } + activityPersistence.schedule(currentActivityScope); if (didRecordMentionedRoot) { bumpMembershipVersion(); } bumpLatestVersion(); }, - [ - channels, - currentActivityScope, - normalizedPubkey, - normalizedRelayUrl, - recordMentionedRoot, - ], + [channels, currentActivityScope, activityPersistence, recordMentionedRoot], ); const muteThread = React.useCallback( @@ -785,13 +760,7 @@ export function useUnreadChannels( ); if (added.didAdd) { threadActivityRef.current = added.items; - if (normalizedPubkey && normalizedRelayUrl) { - writeActivityToStorage( - normalizedPubkey, - normalizedRelayUrl, - added.items, - ); - } + activityPersistence.schedule(currentActivityScope); didAdvance = true; } } @@ -1009,7 +978,7 @@ export function useUnreadChannels( mentionedRootIds, recordThreadInteraction, threadActivityItems: projectActivityForScope( - threadActivityScopeRef.current, + activityPersistence.scopeLoadedRef.current, currentActivityScope, threadActivityRef.current, ), diff --git a/desktop/src/features/custom-emoji/focusRefetchPolicy.test.mjs b/desktop/src/features/custom-emoji/focusRefetchPolicy.test.mjs index 95aa6311d6f..73f8df42cc9 100644 --- a/desktop/src/features/custom-emoji/focusRefetchPolicy.test.mjs +++ b/desktop/src/features/custom-emoji/focusRefetchPolicy.test.mjs @@ -54,12 +54,12 @@ test("custom-emoji: skips fresh focus refetch", async () => { ); }); -test("custom-emoji: refetches genuinely stale data on focus", async () => { +test("custom-emoji: does not refetch stale data on focus", async () => { assert.equal( await focusRefetchCount({ ageMs: customEmojiFocusRefetchPolicy.staleTime + 1, policy: customEmojiFocusRefetchPolicy, }), - 1, + 0, ); }); diff --git a/desktop/src/features/custom-emoji/hooks.ts b/desktop/src/features/custom-emoji/hooks.ts index e730e950fb6..22016496f2f 100644 --- a/desktop/src/features/custom-emoji/hooks.ts +++ b/desktop/src/features/custom-emoji/hooks.ts @@ -32,7 +32,7 @@ export const CUSTOM_EMOJI_FOCUS_STALE_TIME_MS = 5 * 60_000; /** Focus-refetch policy for the custom emoji query; consumed by focusRefetchPolicy.test.mjs. */ export const customEmojiFocusRefetchPolicy = { staleTime: CUSTOM_EMOJI_FOCUS_STALE_TIME_MS, - refetchOnWindowFocus: true, + refetchOnWindowFocus: false, } as const; export const customEmojiQueryKey = ["custom-emoji"] as const; diff --git a/desktop/src/features/forum/focusRefetchPolicy.test.mjs b/desktop/src/features/forum/focusRefetchPolicy.test.mjs index 9a87e1fe392..50e1dd21e11 100644 --- a/desktop/src/features/forum/focusRefetchPolicy.test.mjs +++ b/desktop/src/features/forum/focusRefetchPolicy.test.mjs @@ -54,12 +54,12 @@ test("forum: skips fresh focus refetch", async () => { ); }); -test("forum: refetches genuinely stale data on focus", async () => { +test("forum: does not refetch stale data on focus", async () => { assert.equal( await focusRefetchCount({ ageMs: forumFocusRefetchPolicy.staleTime + 1, policy: forumFocusRefetchPolicy, }), - 1, + 0, ); }); diff --git a/desktop/src/features/forum/hooks.ts b/desktop/src/features/forum/hooks.ts index e773dd07a64..2918a694fd0 100644 --- a/desktop/src/features/forum/hooks.ts +++ b/desktop/src/features/forum/hooks.ts @@ -22,7 +22,7 @@ export const FORUM_FOCUS_STALE_TIME_MS = 5 * 60_000; /** Focus-refetch policy shared by forum-posts and forum-thread queries; consumed by focusRefetchPolicy.test.mjs. */ export const forumFocusRefetchPolicy = { staleTime: FORUM_FOCUS_STALE_TIME_MS, - refetchOnWindowFocus: true, + refetchOnWindowFocus: false, } as const; export function forumPostsQueryKey(channelId: string) { diff --git a/desktop/src/features/home/focusRefetchPolicy.test.mjs b/desktop/src/features/home/focusRefetchPolicy.test.mjs index 41250dfe8ea..1b9da3922a4 100644 --- a/desktop/src/features/home/focusRefetchPolicy.test.mjs +++ b/desktop/src/features/home/focusRefetchPolicy.test.mjs @@ -77,13 +77,13 @@ for (const entry of [ ); }); - test(`${entry.name} refetches genuinely stale data on focus`, async () => { + test(`${entry.name} does not refetch stale data on focus`, async () => { assert.equal( await focusRefetchCount({ ageMs: entry.focusPolicy.staleTime + 1, policy: entry.focusPolicy, }), - 1, + 0, ); }); } diff --git a/desktop/src/features/home/hooks.ts b/desktop/src/features/home/hooks.ts index 3b64f49069d..ce4a0a3056a 100644 --- a/desktop/src/features/home/hooks.ts +++ b/desktop/src/features/home/hooks.ts @@ -12,7 +12,7 @@ export const HOME_FEED_FOCUS_STALE_TIME_MS = 5 * 60_000; /** Focus-refetch policy for the home feed query; consumed by focusRefetchPolicy.test.mjs. */ export const homeFeedFocusRefetchPolicy = { staleTime: HOME_FEED_FOCUS_STALE_TIME_MS, - refetchOnWindowFocus: true, + refetchOnWindowFocus: false, } as const; export function useHomeFeedQuery() { diff --git a/desktop/src/features/home/ui/HomeMembersSidebarOverlay.tsx b/desktop/src/features/home/ui/HomeMembersSidebarOverlay.tsx new file mode 100644 index 00000000000..88df346af43 --- /dev/null +++ b/desktop/src/features/home/ui/HomeMembersSidebarOverlay.tsx @@ -0,0 +1,37 @@ +import * as React from "react"; + +import { useCommunities } from "@/features/communities/useCommunities"; +import type { Channel } from "@/shared/api/types"; + +const MembersSidebar = React.lazy(async () => { + const module = await import("@/features/channels/ui/MembersSidebar"); + return { default: module.MembersSidebar }; +}); + +export function HomeMembersSidebarOverlay({ + channel, + currentPubkey, + onClose, +}: { + channel: Channel | null; + currentPubkey?: string; + onClose: () => void; +}) { + const { activeCommunity } = useCommunities(); + + if (!channel) return null; + + return ( + + { + if (!nextOpen) onClose(); + }} + open={true} + relayUrl={activeCommunity?.relayUrl} + /> + + ); +} diff --git a/desktop/src/features/home/ui/HomeView.tsx b/desktop/src/features/home/ui/HomeView.tsx index 4ffbbaddc53..3b7bff44ea0 100644 --- a/desktop/src/features/home/ui/HomeView.tsx +++ b/desktop/src/features/home/ui/HomeView.tsx @@ -61,7 +61,7 @@ import { useRelaySelfQuery } from "@/features/moderation/hooks"; import { resolveUserLabel } from "@/features/profile/lib/identity"; import { useRemindLater } from "@/features/reminders/ui/RemindMeLaterProvider"; import { deleteMessage, sendChannelMessage } from "@/shared/api/tauri"; -import type { HomeFeedResponse } from "@/shared/api/types"; +import type { Channel, HomeFeedResponse } from "@/shared/api/types"; import { KIND_REACTION } from "@/shared/constants/kinds"; import { topChromeInset } from "@/shared/layout/chromeLayout"; import { cn } from "@/shared/lib/cn"; @@ -72,6 +72,7 @@ import { AUXILIARY_PANEL_SINGLE_COLUMN_BREAKPOINT_PX } from "@/shared/layout/Aux import { useHistorySearchState } from "@/shared/hooks/useHistorySearchState"; import { ProfilePanelProvider } from "@/shared/context/ProfilePanelContext"; import { Button } from "@/shared/ui/button"; +import { HomeMembersSidebarOverlay } from "./HomeMembersSidebarOverlay"; const INBOX_SEARCH_KEYS = [ "item", @@ -167,6 +168,9 @@ export function HomeView({ const [managedChannelId, setManagedChannelId] = React.useState( null, ); + const [membersChannel, setMembersChannel] = React.useState( + null, + ); const { goChannel } = useAppNavigation(); const openDmMutation = useOpenDmMutation(); const openDm = openDmMutation.mutateAsync; @@ -972,6 +976,7 @@ export function HomeView({ channel={managedChannel} currentPubkey={currentPubkey} layout="split" + onOpenMembers={() => setMembersChannel(managedChannel)} onOpenChange={(nextOpen) => { if (!nextOpen) { setManagedChannelId(null); @@ -983,6 +988,11 @@ export function HomeView({ ) : null}
+ setMembersChannel(null)} + /> ); } diff --git a/desktop/src/features/messages/hooks.ts b/desktop/src/features/messages/hooks.ts index 9091121d0bf..e59cd72a919 100644 --- a/desktop/src/features/messages/hooks.ts +++ b/desktop/src/features/messages/hooks.ts @@ -1,5 +1,10 @@ import { useEffect, useEffectEvent } from "react"; -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { + type QueryClient, + useMutation, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; import { toast } from "sonner"; import { @@ -17,7 +22,6 @@ import { import { projectChannelWindowMessages, refreshChannelWindowMessages, - shouldRefreshChannelWindowAfterSubscribe, } from "@/features/messages/lib/projectChannelWindow"; import { reconcileChannelWindowMessages } from "@/features/messages/lib/channelWindowReconciliation"; import { @@ -235,26 +239,46 @@ export function useChannelWindowQuery(channel: Channel | null) { }); } +export function reconcileFetchedChannelWindow( + queryClient: QueryClient, + channelId: string, + events: Awaited>, + previousMessages: RelayEvent[], + signal: AbortSignal, +): RelayEvent[] { + // Tauri invokes cannot be canceled after dispatch. A replacement refetch can + // therefore win while this older request is still in flight. Never let that + // canceled request commit its stale page into the authoritative window. + signal.throwIfAborted(); + const windowKey = channelWindowKey(channelId); + const page = parseChannelWindowResponse(events, channelId, null); + const current = + queryClient.getQueryData(windowKey) ?? + emptyChannelWindowStore(); + const next = replaceNewestChannelWindow(current, page); + queryClient.setQueryData(windowKey, next); + return reconcileChannelWindowMessages(next, previousMessages); +} + export function useChannelMessagesQuery(channel: Channel | null) { const queryClient = useQueryClient(); const queryKey = channelMessagesKey(channel?.id ?? "none"); - const windowKey = channelWindowKey(channel?.id ?? "none"); return useQuery({ enabled: channel !== null && channel.channelType !== "forum", queryKey, - queryFn: async () => { + queryFn: async ({ signal }) => { if (!channel) throw new Error("No channel selected."); const previousMessages = queryClient.getQueryData(queryKey) ?? []; const events = await getChannelWindowEvents(channel.id); - const page = parseChannelWindowResponse(events, channel.id, null); - const current = - queryClient.getQueryData(windowKey) ?? - emptyChannelWindowStore(); - const next = replaceNewestChannelWindow(current, page); - queryClient.setQueryData(windowKey, next); - return reconcileChannelWindowMessages(next, previousMessages); + return reconcileFetchedChannelWindow( + queryClient, + channel.id, + events, + previousMessages, + signal, + ); }, staleTime: 5 * 60 * 1_000, gcTime: 60 * 60 * 1_000, @@ -382,9 +406,10 @@ export function useChannelSubscription(channel: Channel | null) { } cleanup = dispose; - if (!shouldRefreshChannelWindowAfterSubscribe(queryClient, channelId)) { - return; - } + // The live subscription starts at "now", so it cannot close the gap + // between the last page snapshot and subscription establishment. Always + // refresh after the subscription is active; freshness alone is not a + // proof that no relay events landed in that interval. void refreshNewestWindow().catch((error) => { if (!isDisposed) { console.error( @@ -406,7 +431,7 @@ export function useChannelSubscription(channel: Channel | null) { void cleanup(); } }; - }, [channelId, channelType, queryClient]); + }, [channelId, channelType]); } export function useSendMessageMutation( diff --git a/desktop/src/features/messages/lib/channelWindowReconciliation.ts b/desktop/src/features/messages/lib/channelWindowReconciliation.ts index cc2c0f034c8..f6a7e4df21f 100644 --- a/desktop/src/features/messages/lib/channelWindowReconciliation.ts +++ b/desktop/src/features/messages/lib/channelWindowReconciliation.ts @@ -28,6 +28,18 @@ export function reconcileChannelWindowMessages( messages: RelayEvent[], ) { const windowEvents = flattenChannelWindowEvents(window); + if (window.pages.length === 0) { + // A pageless window is unresolved, not authoritative. This state can exist + // briefly when the companion window query mounts beside an already-cached + // rendered timeline. Preserve that cache while admitting live events; + // otherwise the first live event projects a one-row overlay over the + // entire conversation until reload refetches page zero. + let merged = messages; + for (const event of windowEvents) { + merged = reconcileIncomingMessage(merged, event); + } + return [...merged].sort((left, right) => compareRelayOrder(right, left)); + } const authoritativeIds = new Set(windowEvents.map((event) => event.id)); const retained = retainRefetchReconciliationEvents(messages).filter( (event) => !authoritativeIds.has(event.id), diff --git a/desktop/src/features/messages/lib/projectChannelWindow.test.mjs b/desktop/src/features/messages/lib/projectChannelWindow.test.mjs index 2ca2354271e..14ec110addf 100644 --- a/desktop/src/features/messages/lib/projectChannelWindow.test.mjs +++ b/desktop/src/features/messages/lib/projectChannelWindow.test.mjs @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import { QueryClient, QueryObserver } from "@tanstack/react-query"; +import { reconcileFetchedChannelWindow } from "../hooks.ts"; import { channelMessagesKey, channelWindowKey } from "./messageQueryKeys.ts"; import { appendOlderChannelWindow, @@ -11,10 +12,8 @@ import { replaceNewestChannelWindow, } from "./channelWindowStore.ts"; import { - CHANNEL_WINDOW_FRESH_MS, projectChannelWindowMessages, refreshChannelWindowMessages, - shouldRefreshChannelWindowAfterSubscribe, } from "./projectChannelWindow.ts"; import { reconcileChannelWindowMessages } from "./channelWindowReconciliation.ts"; @@ -30,6 +29,18 @@ function event(id, createdAt) { }; } +function wirePage(rows) { + return [ + ...rows, + { + ...event("bounds", 0), + kind: 39006, + tags: [["d", "channel:head"]], + content: JSON.stringify({ has_more: false, next_cursor: null }), + }, + ]; +} + function newestPage(rows) { return { startCursor: null, @@ -276,92 +287,79 @@ test("test_live_projection_retains_pending_send_and_non_broadcast_thread_reply", ]); }); -test("test_subscribe_refresh_skips_fresh_populated_window", () => { +test("test_canceled_stale_fetch_cannot_overwrite_catch_up_window", async () => { const harness = createHarness(); - const updatedAt = harness.client.getQueryState( - harness.windowKey, - ).dataUpdatedAt; - - assert.equal( - shouldRefreshChannelWindowAfterSubscribe( - harness.client, - harness.channelId, - updatedAt + CHANNEL_WINDOW_FRESH_MS - 1, - ), - false, - ); -}); - -test("test_subscribe_refresh_runs_for_stale_window", () => { - const harness = createHarness(); - const updatedAt = harness.client.getQueryState( - harness.windowKey, - ).dataUpdatedAt; + const requests = []; + let resolveRequestStarted; + let requestStarted = new Promise((resolve) => { + resolveRequestStarted = resolve; + }); + const observer = new QueryObserver(harness.client, { + queryKey: harness.messagesKey, + queryFn: async ({ signal }) => { + const previousMessages = harness.client.getQueryData(harness.messagesKey); + let resolveFetch; + const fetch = new Promise((resolve) => { + resolveFetch = resolve; + }); + requests.push({ resolveFetch, signal }); + resolveRequestStarted(); + const events = await fetch; + return reconcileFetchedChannelWindow( + harness.client, + harness.channelId, + events, + previousMessages, + signal, + ); + }, + }); + const unsubscribe = observer.subscribe(() => {}); - assert.equal( - shouldRefreshChannelWindowAfterSubscribe( - harness.client, - harness.channelId, - updatedAt + CHANNEL_WINDOW_FRESH_MS, - ), - true, + await requestStarted; + requestStarted = new Promise((resolve) => { + resolveRequestStarted = resolve; + }); + const catchUp = refreshChannelWindowMessages( + harness.client, + harness.channelId, ); -}); + await requestStarted; -test("test_live_cache_merge_does_not_extend_window_freshness", () => { - const harness = createHarness(); - const windowUpdatedAt = harness.client.getQueryState( - harness.windowKey, - ).dataUpdatedAt; - - harness.client.setQueryData(harness.messagesKey, (messages) => [ - ...messages, - event("live-cache-only", 110), - ]); - - assert.equal( - shouldRefreshChannelWindowAfterSubscribe( - harness.client, - harness.channelId, - windowUpdatedAt + CHANNEL_WINDOW_FRESH_MS, - ), - true, + assert.equal(requests[0].signal.aborted, true); + requests[1].resolveFetch( + wirePage([event("gap", 110), event("initial", 100)]), ); -}); + await catchUp; + assert.deepEqual(contents(harness), ["initial", "gap"]); -test("test_subscribe_refresh_runs_without_a_message_query", () => { - const client = new QueryClient({ - defaultOptions: { queries: { retry: false } }, - }); + requests[0].resolveFetch(wirePage([event("initial", 100)])); + await new Promise((resolve) => setImmediate(resolve)); + appendLiveEvent(harness, event("live", 120)); - assert.equal( - shouldRefreshChannelWindowAfterSubscribe(client, "missing-channel"), - true, + assert.deepEqual(contents(harness), ["initial", "gap", "live"]); + assert.deepEqual( + flattenChannelWindowEvents( + harness.client.getQueryData(harness.windowKey), + ).map((item) => item.content), + ["initial", "gap", "live"], ); + unsubscribe(); }); -test("test_subscribe_refresh_does_not_duplicate_inflight_initial_fetch", async () => { - const client = new QueryClient({ - defaultOptions: { queries: { retry: false } }, - }); - const channelId = "pending-channel"; - const queryKey = channelMessagesKey(channelId); - let resolveFetch; - const observer = new QueryObserver(client, { - queryKey, - queryFn: () => - new Promise((resolve) => { - resolveFetch = resolve; - }), - }); - const unsubscribe = observer.subscribe(() => {}); +test("test_pageless_live_projection_preserves_cached_timeline", () => { + const harness = createHarness(); + const cached = harness.client.getQueryData(harness.messagesKey); + const pageless = emptyChannelWindowStore(); + harness.client.setQueryData(harness.windowKey, pageless); - assert.equal( - shouldRefreshChannelWindowAfterSubscribe(client, channelId), - false, + const next = mergeLiveChannelWindowEvent( + harness.client.getQueryData(harness.windowKey), + event("live", 110), ); + harness.client.setQueryData(harness.windowKey, next); + projectChannelWindowMessages(harness.client, harness.channelId); - resolveFetch([]); - await client.getQueryCache().find({ queryKey })?.promise; - unsubscribe(); + assert.deepEqual(contents(harness), ["initial", "live"]); + assert.equal(harness.client.getQueryData(harness.messagesKey)[0], cached[0]); }); diff --git a/desktop/src/features/messages/lib/projectChannelWindow.ts b/desktop/src/features/messages/lib/projectChannelWindow.ts index 2d56c096b6c..81ef3de42d0 100644 --- a/desktop/src/features/messages/lib/projectChannelWindow.ts +++ b/desktop/src/features/messages/lib/projectChannelWindow.ts @@ -8,34 +8,6 @@ import { } from "./channelWindowStore"; import { reconcileChannelWindowMessages } from "./channelWindowReconciliation"; -export const CHANNEL_WINDOW_FRESH_MS = 5 * 60_000; - -/** - * Subscription setup closes the gap between the initial page and live events, - * but revisiting a channel with a fresh page has no gap to close. Reconnects - * still refresh unconditionally at their call site. - */ -export function shouldRefreshChannelWindowAfterSubscribe( - queryClient: QueryClient, - channelId: string, - now = Date.now(), -): boolean { - const messagesState = queryClient.getQueryState( - channelMessagesKey(channelId), - ); - if (!messagesState) return true; - if (messagesState.fetchStatus === "fetching") return false; - const windowState = queryClient.getQueryState(channelWindowKey(channelId)); - if ( - messagesState.status !== "success" || - windowState?.status !== "success" || - windowState.dataUpdatedAt === 0 - ) { - return true; - } - return now - windowState.dataUpdatedAt >= CHANNEL_WINDOW_FRESH_MS; -} - /** Keep the rendered timeline cache aligned with its authoritative window. */ export function projectChannelWindowMessages( queryClient: QueryClient, diff --git a/desktop/src/features/messages/lib/useRichTextEditor.ts b/desktop/src/features/messages/lib/useRichTextEditor.ts index e800787ca7a..aeb7198b4c5 100644 --- a/desktop/src/features/messages/lib/useRichTextEditor.ts +++ b/desktop/src/features/messages/lib/useRichTextEditor.ts @@ -742,17 +742,26 @@ export function useRichTextEditor({ [editor], ); - const setContentAndFocusEnd = React.useCallback( - (markdown: string) => { + /** + * Replace the editor document with literal plain text and focus its end. + * + * Unlike markdown `setContent`, this preserves trailing whitespace. The + * transaction is marked as programmatic so authored-update observers do not + * reconcile against the intermediate post-send restoration. + */ + const restorePlainTextAndFocusEnd = React.useCallback( + (text: string) => { if (!editor) return; - // The caller already synchronizes composer state. Keep this programmatic - // restoration out of user-edit observers (autocomplete/reconciliation), - // then move selection in the same command chain. - editor - .chain() - .setContent(markdown, { emitUpdate: false }) - .focus("end") - .run(); + const paragraph = editor.schema.nodes.paragraph.create( + null, + text ? editor.schema.text(text) : undefined, + ); + const tr = editor.state.tr + .replaceWith(0, editor.state.doc.content.size, paragraph) + .setMeta("preventUpdate", true); + tr.setSelection(TextSelection.atEnd(tr.doc)); + editor.view.dispatch(tr); + editor.view.focus(); }, [editor], ); @@ -950,7 +959,7 @@ export function useRichTextEditor({ isEmpty, clearContent, setContent, - setContentAndFocusEnd, + restorePlainTextAndFocusEnd, focus, focusEnd, focusPreserve, diff --git a/desktop/src/features/messages/ui/useMentionSendFlow.ts b/desktop/src/features/messages/ui/useMentionSendFlow.ts index cc51c733453..4a721dcfed9 100644 --- a/desktop/src/features/messages/ui/useMentionSendFlow.ts +++ b/desktop/src/features/messages/ui/useMentionSendFlow.ts @@ -73,7 +73,7 @@ type UseMentionSendFlowOptions = { >; richText: Pick< UseRichTextEditorResult, - "clearContent" | "setContent" | "setContentAndFocusEnd" + "clearContent" | "setContent" | "restorePlainTextAndFocusEnd" >; setContent: (content: string) => void; setIsEmojiPickerOpen: React.Dispatch>; @@ -334,7 +334,7 @@ export function useMentionSendFlow({ setContent(postSendContent); contentRef.current = postSendContent; if (postSendContent) { - richText.setContentAndFocusEnd(postSendContent); + richText.restorePlainTextAndFocusEnd(postSendContent); mentions.cancelMentionAutocomplete(); } else richText.clearContent(); setPendingImeta([]); @@ -352,7 +352,7 @@ export function useMentionSendFlow({ mentions.cancelMentionAutocomplete, mentions.clearMentions, richText.clearContent, - richText.setContentAndFocusEnd, + richText.restorePlainTextAndFocusEnd, setContent, setIsEmojiPickerOpen, setPendingImeta, diff --git a/desktop/src/features/notifications/hooks.ts b/desktop/src/features/notifications/hooks.ts index d70ac60b220..1a2cb4a9a53 100644 --- a/desktop/src/features/notifications/hooks.ts +++ b/desktop/src/features/notifications/hooks.ts @@ -4,6 +4,7 @@ import { useHomeFeedQuery } from "@/features/home/hooks"; import { useUsersBatchQuery } from "@/features/profile/hooks"; import type { UserProfileLookup } from "@/features/profile/lib/identity"; import type { Channel, FeedItem, HomeFeedResponse } from "@/shared/api/types"; +import { scheduleAfterForegroundReady } from "@/shared/lib/foregroundReady"; import { getDesktopNotificationPermissionState, requestDesktopNotificationAccess, @@ -210,14 +211,23 @@ export function useNotificationSettings(pubkey?: string) { }, [normalizedPubkey]); React.useEffect(() => { + let cancelPendingRefresh: (() => void) | null = null; const refreshWhenVisible = () => { - if (document.visibilityState === "visible") { - void refreshPermission(); + if (document.visibilityState !== "visible") { + cancelPendingRefresh?.(); + cancelPendingRefresh = null; + return; } + if (cancelPendingRefresh) return; + cancelPendingRefresh = scheduleAfterForegroundReady(() => { + cancelPendingRefresh = null; + if (document.visibilityState === "visible") void refreshPermission(); + }); }; document.addEventListener("visibilitychange", refreshWhenVisible); window.addEventListener("focus", refreshWhenVisible); return () => { + cancelPendingRefresh?.(); document.removeEventListener("visibilitychange", refreshWhenVisible); window.removeEventListener("focus", refreshWhenVisible); }; diff --git a/desktop/src/features/presence/focusRefetchPolicy.test.mjs b/desktop/src/features/presence/focusRefetchPolicy.test.mjs index f83a61d64bb..122ef43cc1b 100644 --- a/desktop/src/features/presence/focusRefetchPolicy.test.mjs +++ b/desktop/src/features/presence/focusRefetchPolicy.test.mjs @@ -54,12 +54,12 @@ test("presence: skips fresh focus refetch", async () => { ); }); -test("presence: refetches genuinely stale data on focus", async () => { +test("presence: does not refetch stale data on focus", async () => { assert.equal( await focusRefetchCount({ ageMs: presenceFocusRefetchPolicy.staleTime + 1, policy: presenceFocusRefetchPolicy, }), - 1, + 0, ); }); diff --git a/desktop/src/features/presence/hooks.ts b/desktop/src/features/presence/hooks.ts index 6da0a6215a4..b1acacafbc5 100644 --- a/desktop/src/features/presence/hooks.ts +++ b/desktop/src/features/presence/hooks.ts @@ -29,7 +29,7 @@ export const PRESENCE_FOCUS_STALE_TIME_MS = 5 * 60_000; /** Focus-refetch policy for the presence query; consumed by focusRefetchPolicy.test.mjs. */ export const presenceFocusRefetchPolicy = { staleTime: PRESENCE_FOCUS_STALE_TIME_MS, - refetchOnWindowFocus: true, + refetchOnWindowFocus: false, } as const; const PRESENCE_ACTIVITY_THROTTLE_MS = 1_000; const PRESENCE_PREFERENCE_STORAGE_KEY = "buzz-presence-preference"; diff --git a/desktop/src/features/profile/ui/ProfileTabContentTransition.tsx b/desktop/src/features/profile/ui/ProfileTabContentTransition.tsx new file mode 100644 index 00000000000..b5174251dc2 --- /dev/null +++ b/desktop/src/features/profile/ui/ProfileTabContentTransition.tsx @@ -0,0 +1,116 @@ +import * as React from "react"; +import { + AnimatePresence, + motion, + type Variants, + useReducedMotion, +} from "motion/react"; + +import type { ProfilePanelTab } from "@/features/profile/ui/UserProfilePanelUtils"; +import { cn } from "@/shared/lib/cn"; + +type TabTransitionDirection = -1 | 0 | 1; + +type TabTransitionContext = { + direction: TabTransitionDirection; + reduceMotion: boolean; +}; + +const TAB_CONTENT_OFFSET_PX = 28; + +const tabContentVariants: Variants = { + enter: ({ direction, reduceMotion }: TabTransitionContext) => ({ + opacity: reduceMotion ? 1 : 0.72, + pointerEvents: "auto", + transform: reduceMotion + ? "translateX(0px)" + : `translateX(${direction * TAB_CONTENT_OFFSET_PX}px)`, + }), + center: { + opacity: 1, + pointerEvents: "auto", + transform: "translateX(0px)", + }, + exit: ({ direction, reduceMotion }: TabTransitionContext) => ({ + opacity: reduceMotion ? 1 : 0, + pointerEvents: "none", + transform: reduceMotion + ? "translateX(0px)" + : `translateX(${-direction * TAB_CONTENT_OFFSET_PX}px)`, + }), +}; + +export function ProfileTabContentTransition({ + activeTab, + children, + className, + tabs, +}: { + activeTab: ProfilePanelTab; + children: React.ReactNode; + className?: string; + tabs: ProfilePanelTab[]; +}) { + const reduceMotion = useReducedMotion() ?? false; + const [lastTransition, setLastTransition] = React.useState<{ + direction: TabTransitionDirection; + tab: ProfilePanelTab; + }>({ direction: 0, tab: activeTab }); + const previousIndex = tabs.indexOf(lastTransition.tab); + const activeIndex = tabs.indexOf(activeTab); + const direction: TabTransitionDirection = + lastTransition.tab === activeTab + ? lastTransition.direction + : previousIndex < 0 || activeIndex < 0 || previousIndex === activeIndex + ? 0 + : activeIndex > previousIndex + ? 1 + : -1; + const transitionContext: TabTransitionContext = { + direction, + reduceMotion, + }; + + React.useLayoutEffect(() => { + if (lastTransition.tab !== activeTab) { + setLastTransition({ direction, tab: activeTab }); + } + }, [activeTab, direction, lastTransition.tab]); + + return ( +
0 ? "forward" : "backward" + } + > + + + {children} + + +
+ ); +} diff --git a/desktop/src/features/profile/ui/UserProfileAgentActions.tsx b/desktop/src/features/profile/ui/UserProfileAgentActions.tsx index 81db3405b21..5919a16e29b 100644 --- a/desktop/src/features/profile/ui/UserProfileAgentActions.tsx +++ b/desktop/src/features/profile/ui/UserProfileAgentActions.tsx @@ -6,23 +6,12 @@ import { Download, Power, Settings, - Trash2, } from "lucide-react"; import type { IdentityArchiveActions } from "@/features/identity-archive/hooks"; import { ArchiveConfirmDialog } from "@/features/profile/ui/ArchiveConfirmDialog"; import type { ManagedAgent } from "@/shared/api/types"; -import { - AlertDialog, - AlertDialogAction, - AlertDialogCancel, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, -} from "@/shared/ui/alert-dialog"; -import { Button, buttonVariants } from "@/shared/ui/button"; +import { Button } from "@/shared/ui/button"; import { DropdownMenu, DropdownMenuContent, @@ -37,7 +26,6 @@ export function UserProfileAgentSettingsMenu({ isPending, isBot = false, managedAgent, - onDelete, onDuplicatePersona, onExportPersona, onToggleAutoStart, @@ -47,14 +35,12 @@ export function UserProfileAgentSettingsMenu({ isPending: boolean; isBot?: boolean; managedAgent?: ManagedAgent; - onDelete?: () => void; onDuplicatePersona?: () => void; onExportPersona?: () => void; onToggleAutoStart?: () => void; personaActionKey?: string; }) { const [archiveConfirmOpen, setArchiveConfirmOpen] = React.useState(false); - const [deleteConfirmOpen, setDeleteConfirmOpen] = React.useState(false); const actionKey = managedAgent?.pubkey ?? "persona-draft"; const personaKey = personaActionKey ?? actionKey; const canToggleAutoStart = @@ -66,11 +52,8 @@ export function UserProfileAgentSettingsMenu({ const hasArchiveAction = archiveActions?.canArchive === true && archiveActions.isArchived !== undefined; - const shouldConfirmAgentDelete = - managedAgent !== undefined && onDelete !== undefined; - const hasManageActions = hasArchiveAction || Boolean(onDelete); const hasActions = - canToggleAutoStart || hasPrimaryActions || hasManageActions; + canToggleAutoStart || hasPrimaryActions || hasArchiveAction; if (!hasActions) { return null; @@ -142,7 +125,7 @@ export function UserProfileAgentSettingsMenu({ Export ) : null} - {hasManageActions && (canToggleAutoStart || hasPrimaryActions) ? ( + {hasArchiveAction && (canToggleAutoStart || hasPrimaryActions) ? ( ) : null} {hasArchiveAction && archiveActions ? ( @@ -166,24 +149,6 @@ export function UserProfileAgentSettingsMenu({ ) ) : null} - {onDelete && hasArchiveAction ? : null} - {onDelete ? ( - { - if (shouldConfirmAgentDelete) { - setDeleteConfirmOpen(true); - return; - } - onDelete(); - }} - > - - Delete agent - - ) : null} {hasArchiveAction && archiveActions ? ( @@ -198,32 +163,17 @@ export function UserProfileAgentSettingsMenu({ open={archiveConfirmOpen} /> ) : null} - {shouldConfirmAgentDelete ? ( - { - setDeleteConfirmOpen(false); - onDelete(); - }} - onOpenChange={setDeleteConfirmOpen} - open={deleteConfirmOpen} - /> - ) : null} ); } export function UserProfileAgentSettingsMenuSlot({ archiveActions, - canDeletePersona, canInstantiateAgent, canManagePersona, isAgentActionPending, isBot, managedAgent, - onDeleteAgent, - onDeletePersona, onDuplicatePersona, onExportPersona, onToggleAutoStart, @@ -231,14 +181,11 @@ export function UserProfileAgentSettingsMenuSlot({ viewerIsOwner, }: { archiveActions: IdentityArchiveActions; - canDeletePersona: boolean; canInstantiateAgent: boolean; canManagePersona: boolean; isAgentActionPending: boolean; isBot: boolean; managedAgent?: ManagedAgent; - onDeleteAgent: () => void; - onDeletePersona: () => void; onDuplicatePersona: () => void; onExportPersona: () => void; onToggleAutoStart: () => void; @@ -250,11 +197,12 @@ export function UserProfileAgentSettingsMenuSlot({ const settingsActionPending = isAgentActionPending || archiveActions.isPending; const sharedProps = { - archiveActions: canShowArchiveAction ? archiveActions : undefined, + archiveActions: !isBot && canShowArchiveAction ? archiveActions : undefined, isBot, isPending: settingsActionPending, - onDuplicatePersona: canManagePersona ? onDuplicatePersona : undefined, - onExportPersona: canManagePersona ? onExportPersona : undefined, + onDuplicatePersona: + !isBot && canManagePersona ? onDuplicatePersona : undefined, + onExportPersona: !isBot && canManagePersona ? onExportPersona : undefined, personaActionKey, }; @@ -263,22 +211,16 @@ export function UserProfileAgentSettingsMenuSlot({ ); } if (canInstantiateAgent) { - return ( - - ); + return ; } - if (canShowArchiveAction) { + if (canShowArchiveAction && !isBot) { return ( void; - onOpenChange: (open: boolean) => void; - open: boolean; -}) { - const isProviderAgent = agent.backend.type === "provider"; - - return ( - - - - Delete this agent? - - Deleting this agent stops and removes the agent from this community. - - -
    -
  • Removes the local management record and saved agent key
  • -
  • Removes the agent from every channel it belongs to
  • -
  • - Archives the agent's identity on the relay so it no longer - appears in member lists or mention suggestions -
  • -
  • - {isProviderAgent - ? "Requests remote deletion; if it is online, Buzz first sends a shutdown command when possible. If the deployment cannot be reached through a channel, the remote process may keep running without local management." - : "Stops any local agent process before deleting the record"} -
  • -
-

- You can also archive this agent from the profile settings menu if you - want to hide the agent instead of removing it. -

- - - - - - {isPending ? "Deleting..." : "Delete agent"} - - -
-
- ); -} diff --git a/desktop/src/features/profile/ui/UserProfileAgentManagementRows.tsx b/desktop/src/features/profile/ui/UserProfileAgentManagementRows.tsx new file mode 100644 index 00000000000..8c6b4138cd7 --- /dev/null +++ b/desktop/src/features/profile/ui/UserProfileAgentManagementRows.tsx @@ -0,0 +1,283 @@ +import * as React from "react"; +import { + Archive, + ArchiveRestore, + CopyPlus, + Download, + Trash2, + type LucideIcon, +} from "lucide-react"; + +import type { IdentityArchiveActions } from "@/features/identity-archive/hooks"; +import { ArchiveConfirmDialog } from "@/features/profile/ui/ArchiveConfirmDialog"; +import type { ManagedAgent } from "@/shared/api/types"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/shared/ui/alert-dialog"; +import { Button, buttonVariants } from "@/shared/ui/button"; +import { PanelSectionGroup } from "@/shared/ui/PanelSectionGroup"; + +export function UserProfileAgentManagementRows({ + archiveActions, + canArchiveAgent, + canDeleteAgent, + isDeletePending, + managedAgent, + onDeleteAgent, + onDuplicateAgent, + onExportAgent, +}: { + archiveActions: IdentityArchiveActions; + canArchiveAgent: boolean; + canDeleteAgent: boolean; + isDeletePending: boolean; + managedAgent?: ManagedAgent; + onDeleteAgent: () => void; + onDuplicateAgent?: () => void; + onExportAgent?: () => void; +}) { + if ( + !onDuplicateAgent && + !onExportAgent && + !canArchiveAgent && + !canDeleteAgent + ) { + return null; + } + + return ( + + {onDuplicateAgent ? ( + + ) : null} + {onExportAgent ? ( + + ) : null} + {canArchiveAgent ? ( + + ) : null} + {canDeleteAgent ? ( + + ) : null} + + ); +} + +function ProfileAgentActionRow({ + destructive = false, + disabled = false, + icon: Icon, + label, + onClick, + testId, +}: { + destructive?: boolean; + disabled?: boolean; + icon: LucideIcon; + label: string; + onClick: () => void; + testId: string; +}) { + return ( + + ); +} + +function ProfileArchiveAgentRow({ + archiveActions, +}: { + archiveActions: IdentityArchiveActions; +}) { + const [confirmOpen, setConfirmOpen] = React.useState(false); + const isArchived = archiveActions.isArchived === true; + const Icon = isArchived ? ArchiveRestore : Archive; + const label = archiveActions.isPending + ? isArchived + ? "Unarchiving…" + : "Archiving…" + : isArchived + ? "Unarchive agent" + : "Archive agent"; + + return ( + <> + { + if (isArchived) { + archiveActions.unarchive(); + return; + } + setConfirmOpen(true); + }} + testId={ + isArchived + ? "user-profile-unarchive-agent-row" + : "user-profile-archive-agent-row" + } + /> + { + archiveActions.archive(); + setConfirmOpen(false); + }} + onOpenChange={setConfirmOpen} + open={confirmOpen} + /> + + ); +} + +function ProfileDeleteAgentRow({ + isPending, + managedAgent, + onDelete, +}: { + isPending: boolean; + managedAgent?: ManagedAgent; + onDelete: () => void; +}) { + const [confirmOpen, setConfirmOpen] = React.useState(false); + + return ( + <> + { + if (managedAgent) { + setConfirmOpen(true); + return; + } + onDelete(); + }} + testId="user-profile-delete-agent-row" + /> + {managedAgent ? ( + { + setConfirmOpen(false); + onDelete(); + }} + onOpenChange={setConfirmOpen} + open={confirmOpen} + /> + ) : null} + + ); +} + +function AgentDeleteConfirmDialog({ + agent, + isPending, + onConfirm, + onOpenChange, + open, +}: { + agent: ManagedAgent; + isPending: boolean; + onConfirm: () => void; + onOpenChange: (open: boolean) => void; + open: boolean; +}) { + const isProviderAgent = agent.backend.type === "provider"; + + return ( + + + + Delete this agent? + + Deleting this agent stops and removes the agent from this community. + + +
    +
  • Removes the local management record and saved agent key
  • +
  • Removes the agent from every channel it belongs to
  • +
  • + Archives the agent's identity on the relay so it no longer + appears in member lists or mention suggestions +
  • +
  • + {isProviderAgent + ? "Requests remote deletion; if it is online, Buzz first sends a shutdown command when possible. If the deployment cannot be reached through a channel, the remote process may keep running without local management." + : "Stops any local agent process before deleting the record"} +
  • +
+

+ Archive this agent if you want to hide it instead of removing it. +

+ + + + + + {isPending ? "Deleting…" : "Delete agent"} + + +
+
+ ); +} diff --git a/desktop/src/features/profile/ui/UserProfileEditAgentDialog.tsx b/desktop/src/features/profile/ui/UserProfileEditAgentDialog.tsx new file mode 100644 index 00000000000..b1f4f54eedf --- /dev/null +++ b/desktop/src/features/profile/ui/UserProfileEditAgentDialog.tsx @@ -0,0 +1,34 @@ +import { AgentDialog } from "@/features/agents/ui/AgentDialog"; +import type { EditAgentFocusTarget } from "@/features/agents/openEditAgentEvent"; +import type { ManagedAgent } from "@/shared/api/types"; + +export function UserProfileEditAgentDialog({ + agent, + canEdit, + initialFocus, + onEditLinkedPersona, + onOpenChange, + open, +}: { + agent: ManagedAgent | undefined; + canEdit: boolean; + initialFocus: EditAgentFocusTarget | undefined; + onEditLinkedPersona: (() => void) | undefined; + onOpenChange: (open: boolean) => void; + open: boolean; +}) { + if (!canEdit || !agent) { + return null; + } + + return ( + + ); +} diff --git a/desktop/src/features/profile/ui/UserProfilePanel.tsx b/desktop/src/features/profile/ui/UserProfilePanel.tsx index cb188dd0089..91040a28e24 100644 --- a/desktop/src/features/profile/ui/UserProfilePanel.tsx +++ b/desktop/src/features/profile/ui/UserProfilePanel.tsx @@ -2,10 +2,7 @@ import * as React from "react"; import { toast } from "sonner"; import { useAppNavigation } from "@/app/navigation/useAppNavigation"; -import { - useAgentMemoryQuery, - useIsManagedAgent, -} from "@/features/agent-memory/hooks"; +import { useIsManagedAgent } from "@/features/agent-memory/hooks"; import { type AttachManagedAgentToChannelResult, useAcpRuntimesQuery, @@ -33,13 +30,7 @@ import { resolveStartRuntimeForDefinition, } from "@/features/agents/lib/instanceInputForDefinition"; import { describeLogFile } from "@/features/agents/ui/agentUi"; -import { AgentDialog } from "@/features/agents/ui/AgentDialog"; import { useAgentLifecycleActions } from "@/features/profile/ui/useAgentLifecycleActions"; -import { - consumePendingOpenEditAgent, - type EditAgentFocusTarget, - subscribeOpenEditAgent, -} from "@/features/agents/openEditAgentEvent"; import { duplicatePersonaDialogState, editPersonaDialogState, @@ -59,13 +50,15 @@ import { import { ownsAuthorAgent } from "@/features/profile/lib/identity"; import { resolveProfileActivityAgent } from "@/features/profile/lib/profileActivityAgent"; import { - AgentInfoFocusedView, AgentInstructionsFocusedView, + ProfileSummaryView, +} from "@/features/profile/ui/UserProfilePanelSections"; +import { + AgentInfoFocusedView, ChannelsFocusedView, DiagnosticsFocusedView, MemoryFocusedView, - ProfileSummaryView, -} from "@/features/profile/ui/UserProfilePanelSections"; +} from "@/features/profile/ui/UserProfilePanelFocusedViews"; import { AgentConfigurationFocusedView } from "@/features/profile/ui/UserProfilePanelAgentDetails"; import { UserProfileAgentSettingsMenuSlot } from "@/features/profile/ui/UserProfileAgentActions"; import { useProfileAgentDeletion } from "@/features/profile/ui/UserProfilePanelDeletion"; @@ -86,7 +79,7 @@ import { type UserProfilePanelProps, useRetainedPersona, } from "@/features/profile/ui/UserProfilePanelUtils"; -import { useProfileDmAction } from "@/features/profile/ui/useProfileDmAction"; +import { useProfileInteractionActions } from "@/features/profile/ui/useProfileInteractionActions"; import { useUserStatusQuery } from "@/features/user-status/hooks"; import { useOpenAgentActivity } from "@/features/agents/useOpenAgentActivity"; import { useEscapeKey } from "@/shared/hooks/useEscapeKey"; @@ -101,6 +94,8 @@ import type { } from "@/shared/api/types"; import { UserProfilePanelFrame } from "@/features/profile/ui/UserProfilePanelFrame"; import { getUserProfilePanelHeaderContent } from "@/features/profile/ui/UserProfilePanelHeaderContent"; +import { UserProfileEditAgentDialog } from "@/features/profile/ui/UserProfileEditAgentDialog"; +import { useProfileEditAgentRequest } from "@/features/profile/ui/useProfileEditAgentRequest"; export type { ProfilePanelTab, ProfilePanelView }; export function UserProfilePanel({ @@ -154,28 +149,27 @@ export function UserProfilePanel({ }, [onTabChange], ); - const [editAgentOpen, setEditAgentOpen] = React.useState(false); - const [editAgentFocus, setEditAgentFocus] = React.useState< - EditAgentFocusTarget | undefined - >(undefined); - - // Open the Edit Agent dialog when `requestOpenEditAgent(pubkey)` fires from - // a card or other non-panel surface (e.g. `ConfigNudgeCard`). Mirrors the - // `subscribeOpenCreateAgent` pattern in AgentsView. - React.useEffect(() => { - if (!pubkey) return; - // Consume any pending request that arrived before this panel mounted. - const pending = consumePendingOpenEditAgent(pubkey); - if (pending !== false) { - setEditAgentFocus(pending === true ? undefined : pending); - setEditAgentOpen(true); - } - // Subscribe for events that arrive while the panel is mounted. - return subscribeOpenEditAgent(pubkey, (focus) => { - setEditAgentFocus(focus); - setEditAgentOpen(true); - }); - }, [pubkey]); + const [stickyChrome, setStickyChrome] = React.useState({ + active: false, + height: 0, + }); + const handleStickyChromeChange = React.useCallback( + (nextState: { active: boolean; height: number }) => { + setStickyChrome((currentState) => + currentState.active === nextState.active && + currentState.height === nextState.height + ? currentState + : nextState, + ); + }, + [], + ); + const { + focus: editAgentFocus, + open: editAgentOpen, + setFocus: setEditAgentFocus, + setOpen: setEditAgentOpen, + } = useProfileEditAgentRequest(pubkey); const [addToChannelOpen, setAddToChannelOpen] = React.useState(false); const [personaDialogState, setPersonaDialogState] = React.useState(null); @@ -325,15 +319,8 @@ export function UserProfilePanel({ }), [effectivePubkey, isBot, managedAgent, profile, relayAgent, viewerIsOwner], ); - // Observer ingestion (frame decryption + derived active-turn liveness) is - // owner-global — mounted once in AppShell via useAgentObserverIngestion — - // covering both locally managed agents and declared-owned relay agents. - const canEditAgent = - isOwner === true && - (managedAgent !== undefined || resolvedPersona !== undefined); - const memoryQuery = useAgentMemoryQuery(effectivePubkey, { - enabled: viewerIsOwner && Boolean(effectivePubkey), - }); + // Observer ingestion is owner-global across local and declared-owned agents. + const canEditAgent = Boolean(isOwner && (managedAgent ?? resolvedPersona)); const isSelf = currentPubkey !== undefined && pubkeyLower.length > 0 && @@ -395,10 +382,22 @@ export function UserProfilePanel({ setView("summary", { replace: true }); setTab("info", { replace: true }); }, [setTab, setView, targetKey]); - const { handleMessage, isOpeningDm } = useProfileDmAction({ + const { + canHuddle, + canMessage, + canWave, + handleHuddle, + handleMessage, + handleWave, + isStartingHuddle, + pendingAction, + } = useProfileInteractionActions({ effectivePubkey, + enabled: onOpenDm !== undefined, + isBot, + isSelf, onClose, - onOpenDm, + viewerIsOwner, }); const handleEditAgent = React.useCallback(() => { @@ -407,7 +406,7 @@ export function UserProfilePanel({ return; } setEditAgentOpen(true); - }, [resolvedPersona]); + }, [resolvedPersona, setEditAgentOpen]); const { deleteManagedAgentRecord, deleteManagedAgentsForPersona } = useProfileAgentDeletion({ @@ -707,31 +706,27 @@ export function UserProfilePanel({ : null; const ownerProfilePubkey = ownerPubkey ?? (isOwner === true ? (currentPubkey ?? null) : null); - const ownerAvatarProfile = ownerPubkey - ? ownerProfileQuery.data - : currentProfileQuery.data; - const memoryCount = - memoryQuery.data && - (memoryQuery.data.core ? 1 : 0) + memoryQuery.data.memories.length; const agentInstruction = resolveAgentInstruction( managedAgent, resolvedPersona, ); const canManagePersona = isOwner === true && resolvedPersona !== undefined; - const canEditPersona = canManagePersona; const canDeletePersona = canManagePersona && !resolvedPersona?.sourceTeam; + const canDeleteProfileAgent = + isBot && + ((viewerIsOwner && managedAgent !== undefined) || + (canInstantiateAgent && canDeletePersona)); + const handleDeleteProfileAgent = + viewerIsOwner && managedAgent ? handleDeleteAgent : handleDeletePersona; const archiveActions = useIdentityArchive(effectivePubkey); - const agentSettingsMenu = ( + const agentSettingsMenu = isBot ? null : ( setView("summary"), + onEditAgent: canEditAgent ? handleEditAgent : undefined, view, viewerIsOwner, }, @@ -783,10 +778,12 @@ export function UserProfilePanel({ ? "flex flex-col overflow-hidden" : "overflow-y-auto", )} + data-testid="user-profile-scroll-body" > {view === "summary" ? ( setAddToChannelOpen(true)} + onDeleteAgent={handleDeleteProfileAgent} + onDuplicateAgent={ + isBot && canManagePersona ? handleDuplicatePersona : undefined + } + onExportAgent={ + isBot && canManagePersona ? handleExportPersona : undefined + } onOpenInstance={(instancePubkey) => onOpenProfile?.(instancePubkey)} onOpenActivity={handleOpenActivity} onOpenChannel={handleOpenChannel} onOpenDiagnostics={() => setView("diagnostics")} - onOpenInstructions={() => setView("instructions")} + onStickyChromeChange={handleStickyChromeChange} onTabChange={setTab} - onOpenDm={onOpenDm} - onCreateCard={ - canManagePersona && resolvedPersona - ? () => - setCardMintTarget({ - // Prefer the live instance pubkey; fall back to the - // persona/definition id (same resolution as export). - id: managedAgent?.pubkey ?? resolvedPersona.id, - name: resolvedPersona.displayName, - // Locking needs an instance keypair to encrypt to. - canLock: Boolean(managedAgent?.pubkey), - }) - : undefined - } presenceStatus={presenceStatus} profile={profile} pubkey={effectivePubkey} @@ -905,28 +899,27 @@ export function UserProfilePanel({ ) : null} ); - const editAgentDialog = - canEditAgent && managedAgent ? ( - { - setEditAgentOpen(false); - setEditAgentFocus(undefined); - setPersonaDialogState(editPersonaDialogState(resolvedPersona)); - } - : undefined - } - onOpenChange={(next) => { - setEditAgentOpen(next); - if (!next) setEditAgentFocus(undefined); - }} - open={editAgentOpen} - /> - ) : null; + const editAgentDialog = ( + { + setEditAgentOpen(false); + setEditAgentFocus(undefined); + setPersonaDialogState(editPersonaDialogState(resolvedPersona)); + } + : undefined + } + onOpenChange={(next) => { + setEditAgentOpen(next); + if (!next) setEditAgentFocus(undefined); + }} + open={editAgentOpen} + /> + ); const addAgentToChannelDialog = managedAgent ? ( diff --git a/desktop/src/features/profile/ui/UserProfilePanelAgentDetails.tsx b/desktop/src/features/profile/ui/UserProfilePanelAgentDetails.tsx index 844de528e23..1da2e597dd8 100644 --- a/desktop/src/features/profile/ui/UserProfilePanelAgentDetails.tsx +++ b/desktop/src/features/profile/ui/UserProfilePanelAgentDetails.tsx @@ -116,11 +116,11 @@ export function AgentInstructionRow({ trimmedInstruction.length > 0 && onOpenInstructions !== undefined; const rowContent = ( <> - - - +
-
Instructions
+
+ Agent instructions +
{trimmedInstruction ? ( canOpenInstructions ? ( void; testId?: string; @@ -82,7 +83,6 @@ export function useProfileFieldBuckets({ isOwner, managedAgent, onOpenProfile, - ownerAvatarUrl, ownerDisplayName, ownerHandle, ownerProfilePubkey, @@ -98,7 +98,6 @@ export function useProfileFieldBuckets({ isOwner: boolean | undefined; managedAgent: ManagedAgent | undefined; onOpenProfile?: (pubkey: string) => void; - ownerAvatarUrl: string | null; ownerDisplayName: string | null; ownerHandle: string | null; ownerProfilePubkey: string | null; @@ -118,7 +117,6 @@ export function useProfileFieldBuckets({ includeOperationalFields: isOwner === true, managedAgent, onOpenProfile, - ownerAvatarUrl, ownerDisplayName, ownerHandle, ownerProfilePubkey, @@ -136,7 +134,6 @@ export function useProfileFieldBuckets({ isOwner, managedAgent, onOpenProfile, - ownerAvatarUrl, ownerDisplayName, ownerHandle, ownerProfilePubkey, @@ -167,10 +164,17 @@ export function buildPublicFields({ if (pubkey) { fields.push({ + copyValue: pubkey, displayValue: truncatePubkey(pubkey), - displayNode: , - icon: Fingerprint, + displayNode: ( + + ), label: "Public key", + testId: "user-profile-public-key", }); } @@ -220,7 +224,6 @@ export function buildOwnerFields({ includeOperationalFields, managedAgent, onOpenProfile, - ownerAvatarUrl, ownerDisplayName, ownerHandle, ownerProfilePubkey, @@ -233,7 +236,6 @@ export function buildOwnerFields({ includeOperationalFields: boolean; managedAgent: ManagedAgent | undefined; onOpenProfile?: (pubkey: string) => void; - ownerAvatarUrl: string | null; ownerDisplayName: string | null; ownerHandle: string | null; ownerProfilePubkey: string | null; @@ -256,18 +258,6 @@ export function buildOwnerFields({ : null; const ownerClickable = Boolean(onOpenProfile && ownerProfilePubkey); - const ownerContent = ( - <> - - {ownerDisplayName} - - ); if (ownerDisplayName) { fields.push({ @@ -275,12 +265,7 @@ export function buildOwnerFields({ ? undefined : (ownerProfilePubkey ?? ownerPubkey ?? ownerHandle ?? undefined), displayValue: ownerDisplayName, - displayNode: ( - - {ownerContent} - - ), - icon: UserRound, + displayNode: {ownerDisplayName}, label: "Managed by", onClick: ownerClickable && ownerProfilePubkey @@ -335,8 +320,10 @@ export function buildOwnerFields({ .replace(/\b\w/g, (char: string) => char.toUpperCase()), displayNode: ( ), @@ -439,52 +426,114 @@ function orderProfileFields(fields: ProfileField[]) { ]; } -export function ProfileFieldRows({ fields }: { fields: ProfileField[] }) { +export function ProfileFieldRows({ + fields, + variant = "default", +}: { + fields: ProfileField[]; + variant?: "default" | "runtime"; +}) { return ( <> {orderProfileFields(fields).map((field) => ( - + ))} ); } -export function ProfileFieldGroup({ fields }: { fields: ProfileField[] }) { +export function ProfileSectionGroup({ + children, + headerAction, + testId, + title, +}: { + children: React.ReactNode; + headerAction?: React.ReactNode; + testId?: string; + title?: string; +}) { + return ( + +
{children}
+
+ ); +} + +export function ProfileFieldGroup({ + fields, + title, +}: { + fields: ProfileField[]; + title?: string; +}) { return ( -
-
- -
-
+ + + ); } -function ProfileFieldRow({ field }: { field: ProfileField }) { +function ProfileFieldRow({ + field, + variant, +}: { + field: ProfileField; + variant: "default" | "runtime"; +}) { const Icon = field.icon; const isCopyable = Boolean(field.copyValue); const isActionable = Boolean(field.onClick); + const isTrailingDisplay = + variant === "runtime" && field.label === "Status" && field.displayNode; + const { copied, copy } = useCopyFeedback({ + label: field.label, + value: field.copyValue ?? "", + }); const content = ( <> - - - + {variant === "default" && Icon ? ( + + ) : null} - + {field.label} - - {field.displayNode ?? field.displayValue} - + {!isTrailingDisplay ? ( + + {field.displayNode ?? field.displayValue} + + ) : null} + {isTrailingDisplay ? field.displayNode : null} {field.trailingNode} {isActionable ? ( - + ) : isCopyable ? ( - + ) : null} ); @@ -493,7 +542,7 @@ function ProfileFieldRow({ field }: { field: ProfileField }) { return ( + + ))} + + )} + +
+ ); +} + +export function AgentInfoFocusedView({ + metadataFields, +}: { + metadataFields: ProfileField[]; +}) { + if (metadataFields.length === 0) { + return null; + } + + return ( +
+ +
+ ); +} + +export function DiagnosticsFocusedView({ + canOpenAgentLogs, + fields, + logContent, + logError, + logLoading, + managedAgent, +}: { + canOpenAgentLogs: boolean; + fields: ProfileField[]; + logContent: string | null; + logError: Error | null; + logLoading: boolean; + managedAgent: ManagedAgent | undefined; +}) { + const hasLog = canOpenAgentLogs && managedAgent !== undefined; + const lastErrorField = fields.find((field) => field.label === "Last error"); + const detailFields = fields.filter( + (field) => field.label !== "Last error" && field.label !== "Status", + ); + + if (!lastErrorField && detailFields.length === 0 && !hasLog) { + return null; + } + + return ( +
+ {lastErrorField ? ( + + +
+ Last error + + {lastErrorField.displayValue} + +
+
+ ) : null} + {detailFields.length > 0 ? ( + + ) : null} + {hasLog ? ( +
+ +
+ ) : null} +
+ ); +} diff --git a/desktop/src/features/profile/ui/UserProfilePanelFrame.tsx b/desktop/src/features/profile/ui/UserProfilePanelFrame.tsx index 81d1860448c..11a744ee8a4 100644 --- a/desktop/src/features/profile/ui/UserProfilePanelFrame.tsx +++ b/desktop/src/features/profile/ui/UserProfilePanelFrame.tsx @@ -1,7 +1,11 @@ import type * as React from "react"; -import { AuxiliaryPanel } from "@/shared/layout/AuxiliaryPanel"; -import { AuxiliaryPanelHeader } from "@/shared/layout/AuxiliaryPanel"; +import { + AUXILIARY_PANEL_DEFAULT_SURFACE_CLASS, + AuxiliaryPanel, + AuxiliaryPanelHeader, +} from "@/shared/layout/AuxiliaryPanel"; +import { cn } from "@/shared/lib/cn"; type UserProfilePanelFrameProps = { addAgentToChannelDialog: React.ReactNode; @@ -18,6 +22,9 @@ type UserProfilePanelFrameProps = { personaDialogs: React.ReactNode; profileBody: React.ReactNode; splitPaneClamp: boolean; + stickyChromeActive: boolean; + stickyChromeEnabled: boolean; + stickyChromeHeight: number; widthPx: number; transparentChrome?: boolean; }; @@ -37,12 +44,16 @@ export function UserProfilePanelFrame({ personaDialogs, profileBody, splitPaneClamp, + stickyChromeActive, + stickyChromeEnabled, + stickyChromeHeight, widthPx, transparentChrome = false, }: UserProfilePanelFrameProps) { return ( - {headerLeftContent} - {headerActions} - + <> +