From 97fd9e0b497704b6cf087d4f9fc2d82b673c0a93 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 01:58:13 -0700 Subject: [PATCH 1/4] cmux-tui: cmux link seed push|clone, seed a Cloud project from GitHub with a copy fallback; agent move --sync seed Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-3bab30503f0b9a7059e5 --- cmux-tui/crates/cmux-tui-core/src/git_seed.rs | 18 +- .../crates/cmux-tui-core/src/git_seed/job.rs | 21 + .../crates/cmux-tui/src/link/agent_cli.rs | 62 +- cmux-tui/crates/cmux-tui/src/link/mod.rs | 6 +- cmux-tui/crates/cmux-tui/src/link/seed_cli.rs | 622 ++++++++++++++++++ .../cmux-tui/src/link/seed_cli/record.rs | 92 +++ cmux-tui/crates/cmux-tui/src/link/sync_cli.rs | 43 ++ .../cmux-tui/tests/cloud_move_journey.rs | 45 +- cmux-tui/crates/cmux-tui/tests/link_seed.rs | 367 +++++++++++ 9 files changed, 1244 insertions(+), 32 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui/src/link/seed_cli.rs create mode 100644 cmux-tui/crates/cmux-tui/src/link/seed_cli/record.rs create mode 100644 cmux-tui/crates/cmux-tui/tests/link_seed.rs diff --git a/cmux-tui/crates/cmux-tui-core/src/git_seed.rs b/cmux-tui/crates/cmux-tui-core/src/git_seed.rs index 24372601868c..0b6d6ad7e764 100644 --- a/cmux-tui/crates/cmux-tui-core/src/git_seed.rs +++ b/cmux-tui/crates/cmux-tui-core/src/git_seed.rs @@ -35,12 +35,28 @@ pub const GIT_SEED_COMMANDS: [&str; 2] = ["git.seed.start", "git.seed.status"]; static SERVICE: OnceLock = OnceLock::new(); +/// Test seam, debug builds only: `CMUX_TEST_GIT_SEED_GITHUB=` +/// makes the daemon's seeds clone `https://github.com/OWNER/NAME.git` from +/// `file:///OWNER/NAME.git`, with no GitHub path +/// ([`GitSeedService::with_test_github`]). Release builds never read it. +#[cfg(debug_assertions)] +const TEST_GITHUB_VAR: &str = "CMUX_TEST_GIT_SEED_GITHUB"; + /// The seed owner, when this daemon serves `fs-v1` (it seeds under the same /// roots). #[must_use] pub fn installed() -> Option<&'static GitSeedService> { let fs = crate::fs_ops::installed()?; - Some(SERVICE.get_or_init(|| GitSeedService::new(fs.roots().clone()))) + Some(SERVICE.get_or_init(|| { + #[cfg(debug_assertions)] + if let Some(dir) = std::env::var_os(TEST_GITHUB_VAR) + .map(std::path::PathBuf::from) + .filter(|dir| dir.is_absolute()) + { + return GitSeedService::with_test_github(fs.roots().clone(), &dir); + } + GitSeedService::new(fs.roots().clone()) + })) } /// The capability to advertise in `identify`, when served. diff --git a/cmux-tui/crates/cmux-tui-core/src/git_seed/job.rs b/cmux-tui/crates/cmux-tui-core/src/git_seed/job.rs index f288fd7f6fcf..c067284565ae 100644 --- a/cmux-tui/crates/cmux-tui-core/src/git_seed/job.rs +++ b/cmux-tui/crates/cmux-tui-core/src/git_seed/job.rs @@ -183,6 +183,27 @@ impl GitSeedService { Self { roots, config, jobs: Mutex::new(Jobs::default()) } } + /// For tests (and the debug-only daemon seam in [`super::installed`]): + /// a service whose GitHub is the folder `github`, so + /// `https://github.com/OWNER/NAME.git` is cloned from + /// `file:///OWNER/NAME.git`, with no credential. Nothing on the + /// wire reaches it. + #[doc(hidden)] + #[must_use] + pub fn with_test_github(roots: Roots, github: &std::path::Path) -> Self { + let local = format!("file://{}/", github.display()); + let config = Config { + git: Git { + extra_config: vec![ + format!("url.{local}.insteadOf=https://github.com/"), + "protocol.file.allow=always".into(), + ], + }, + credentials: Some(vec![Credential::Anonymous]), + }; + Self::with_config(roots, config) + } + /// Runs `cmd` with the request object `params`. pub fn call(&self, cmd: &str, params: &Value) -> Result { match cmd { diff --git a/cmux-tui/crates/cmux-tui/src/link/agent_cli.rs b/cmux-tui/crates/cmux-tui/src/link/agent_cli.rs index 03675ae50213..cf1daad5e4e5 100644 --- a/cmux-tui/crates/cmux-tui/src/link/agent_cli.rs +++ b/cmux-tui/crates/cmux-tui/src/link/agent_cli.rs @@ -65,12 +65,14 @@ //! (`agent_session.host_unverified`) and gives the chat back. A replacement //! link keeps the socket path, so a rerun still resolves an unresolved move. //! -//! `--sync push|pull --sync-socket LINK --sync-local LOCAL --sync-remote +//! `--sync push|pull|seed --sync-socket LINK --sync-local LOCAL --sync-remote //! REMOTE`: once the chat is frozen (after the begin, before the receive), the //! pump copies its folder's last changes, `cmux link sync push LOCAL REMOTE` -//! (Move to Cloud) or `cmux link sync pull LOCAL REMOTE` (Bring Back Local) -//! over the machine link socket `LINK`. A chat moved mid-turn (`--interrupt`) -//! made changes up to the freeze, after the caller's own sync; this takes +//! (Move to Cloud), `cmux link seed push --since-last LOCAL REMOTE` (Move to +//! Cloud of a folder `cmux link seed push` put there: a delta bundle, or a +//! re-push of its copy) or `cmux link sync pull LOCAL REMOTE` (Bring Back +//! Local) over the machine link socket `LINK`. A chat moved mid-turn +//! (`--interrupt`) made changes up to the freeze, after the caller's own sync; this takes //! them along, so the turn goes on there with them (acpmux queues its //! continuation, cmuxterm-hq#5846 item 1). A sync that fails, or a pull that //! conflicts (`sync_conflict` with `conflicts`), gives the chat back. It runs @@ -101,7 +103,7 @@ use record::{Record, Records}; const USAGE: &str = "usage: cmux link agent move --from-socket LOCAL_DAEMON_SOCKET \ --surface SURFACE_ID --to-socket REMOTE_CONNECT_SOCKET --cwd REMOTE_FOLDER [--interrupt] \ [--ask-first] [--vm-kind KIND [--to-pid PID]] [--state-dir DIR] \ -[--sync push|pull --sync-socket LINK_SOCKET --sync-local LOCAL --sync-remote REMOTE] +[--sync push|pull|seed --sync-socket LINK_SOCKET --sync-local LOCAL --sync-remote REMOTE] --to-socket: the local_socket of `cmux-tui remote connect --headless --json` for the \ machine (its connection-snapshot event), not the machine link socket of `cmux link sync`"; /// Bytes asked for per page (the daemon serves at most 4 MiB). @@ -127,14 +129,22 @@ struct Args { sync: Option, } -/// `--sync push|pull --sync-socket --sync-local --sync-remote` (module docs). +/// `--sync push|pull|seed --sync-socket --sync-local --sync-remote` (module docs). struct Sync { - pull: bool, + way: Way, socket: PathBuf, local: PathBuf, remote: String, } +/// Which way `--sync` copies. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Way { + Push, + Pull, + Seed, +} + /// A move that failed, as its JSON line reports it. struct Failed { code: String, @@ -254,8 +264,9 @@ fn parse(args: &[String]) -> Option { "--ask-first" => ask_first = true, "--sync" => { sync = Some(match rest.next()?.as_str() { - "push" => false, - "pull" => true, + "push" => Way::Push, + "pull" => Way::Pull, + "seed" => Way::Seed, _ => return None, }); } @@ -269,8 +280,8 @@ fn parse(args: &[String]) -> Option { // All four or none; the remote folder is absolute, as `cmux link sync` takes it. let sync = match (sync, sync_socket, sync_local, sync_remote) { (None, None, None, None) => None, - (Some(pull), Some(socket), Some(local), Some(remote)) if remote.starts_with('/') => { - Some(Sync { pull, socket, local, remote: remote.trim_end_matches('/').to_owned() }) + (Some(way), Some(socket), Some(local), Some(remote)) if remote.starts_with('/') => { + Some(Sync { way, socket, local, remote: remote.trim_end_matches('/').to_owned() }) } _ => return None, }; @@ -459,13 +470,28 @@ fn fresh(args: &Args, records: &Records) -> Result { /// `--sync`'s copy of a frozen chat's folder (module docs). fn sync_frozen(sync: &Sync) -> Result<(), Failed> { use super::sync_cli::{self, Back}; - if !sync.pull { - return sync_cli::push(&sync.socket, &sync.local, &sync.remote).map(|_| ()).map_err(|e| { - Failed::new( - sync_cli::push_failure_code(&e), - format!("the folder's last changes: {e:#}"), - ) - }); + match sync.way { + Way::Push => { + return sync_cli::push(&sync.socket, &sync.local, &sync.remote).map(|_| ()).map_err( + |e| { + Failed::new( + sync_cli::push_failure_code(&e), + format!("the folder's last changes: {e:#}"), + ) + }, + ); + } + Way::Seed => { + return super::seed_cli::push_since_last(&sync.socket, &sync.local, &sync.remote) + .map(|_| ()) + .map_err(|e| { + Failed::new( + super::seed_cli::failure_code(&e), + format!("the folder's last changes: {e:#}"), + ) + }); + } + Way::Pull => {} } match sync_cli::pull_into(&sync.socket, &sync.local, &sync.remote) { Back::Done(_) => Ok(()), diff --git a/cmux-tui/crates/cmux-tui/src/link/mod.rs b/cmux-tui/crates/cmux-tui/src/link/mod.rs index 6c671f617aba..a88d75d4fd8d 100644 --- a/cmux-tui/crates/cmux-tui/src/link/mod.rs +++ b/cmux-tui/crates/cmux-tui/src/link/mod.rs @@ -11,9 +11,10 @@ mod dial; mod dial_cli; mod enrollment; mod ensure; -// Not wired into start-chat yet; its CLI entry is `cmux link seed plan`. +// `cmux link seed plan`; `cmux link seed push|clone` is [`seed_cli`]. #[cfg_attr(not(test), allow(dead_code))] mod seed; +mod seed_cli; mod start_chat_cli; mod sync_cli; // Wired into `serve` when Cloud hosts run the link (needs the TeamDO peer @@ -119,6 +120,9 @@ pub(crate) fn run(args: &[String]) -> i32 { return sync_cli::run(&args[1..]); } if args.first().map(String::as_str) == Some("seed") { + if matches!(args.get(1).map(String::as_str), Some("push" | "clone")) { + return seed_cli::run(&args[1..]); + } return seed::run(&args[1..]); } // So do `agent` and `access`. diff --git a/cmux-tui/crates/cmux-tui/src/link/seed_cli.rs b/cmux-tui/crates/cmux-tui/src/link/seed_cli.rs new file mode 100644 index 000000000000..4818c5425b42 --- /dev/null +++ b/cmux-tui/crates/cmux-tui/src/link/seed_cli.rs @@ -0,0 +1,622 @@ +//! `cmux link seed push --socket PATH LOCAL REMOTE [--since-last]` puts a +//! project on a Cloud machine the way design/cloud.md ("Getting the project +//! onto the machine") says: a git folder with a GitHub remote is seeded +//! (`git-seed-v1`: the machine clones the remote, this Mac sends one bundle +//! of what the remote lacks, [`super::seed::plan`]); everything else takes +//! the streamed copy (`cmux link sync push`, [`super::sync_cli`]). +//! +//! A full push falls back to the copy when the plan does ([`plan::Fallback`]), +//! when the machine does not serve `git-seed-v1`, and when the machine +//! answers `seed.needs_github` (with the App's `install_url`, so the app can +//! offer the install while the chat goes on), `seed.base_missing` (after one +//! more try from the remote default branch's tip) or `seed.clone_failed`. A +//! folder the failed seed made on the machine is removed before the copy. +//! +//! `--since-last` sends only what changed since this Mac's last push of the +//! same folder to the same machine folder ([`record`]): a delta bundle after +//! a seed (nothing at all when nothing changed), or a re-push of the copy +//! (whose own record makes it incremental) after a copy. It never falls back +//! to a full copy: no record is `seed.no_previous`, and the machine's +//! `seed.since_mismatch` and `seed.machine_changed` are errors. +//! +//! The bundle is uploaded through `fs-v1` beside REMOTE +//! (`REMOTE/../.cmux-seed-.bundle`, inside the machine's fs roots), then +//! `git.seed.start` runs the seed as a job on the machine, and the job is +//! read with one `git.seed.status` request every [`STATUS_EVERY`] until it +//! ends or [`SEED_DEADLINE`] passes. That wait is inside this process. +//! +//! `cmux link seed clone --socket PATH --repo OWNER/NAME [--ref REF] REMOTE` +//! clones a GitHub repository on the machine with no folder of this Mac. +//! +//! Output: progress on stderr as JSON lines, `{"progress":{"phase":"upload", +//! "bytes_done","bytes_total"}}` while the bundle goes, then +//! `{"progress":{"phase":clone|fetch|bundle|checkout|submodules|lfs, +//! "bytes_done","objects","objects_total"}}` from the machine (the copy's own +//! lines, `{"progress":{"files_done",...}}`, for a copy). One JSON line on +//! stdout: `{"ok":true,"mode":"seed"|"delta"|"copy","unchanged":bool, +//! "bundle_bytes":N,"head":SHA|null,"work":SHA|null,"skipped_files":[{"path", +//! "kind","why"}]}`, a copy adding `"copy":{the sync summary}` and a fallback +//! `"fallback_code"`, `"fallback_why"` and `"install_url"` (needs_github); +//! a clone `{"ok":true,"mode":"clone","head","head_ref"}`. Exit 0. A failure +//! is `{"ok":false,"error","error_code","install_url"?}`, exit 1; bad usage +//! exits 64. + +mod record; + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use cmux_json::JsonSet as _; +use serde_json::{Value, json}; + +use super::seed::plan::{self, Fallback, FallbackCode, Planned, PreviousSeed}; +use super::{machine_gate, sync_cli}; +use record::{Kept, Record}; + +const USAGE: &str = "usage: cmux link seed push --socket PATH LOCAL_FOLDER REMOTE_FOLDER \ +[--since-last] + cmux link seed clone --socket PATH --repo OWNER/NAME [--ref REF] REMOTE_FOLDER"; + +/// Time between two `git.seed.status` requests. +const STATUS_EVERY: Duration = Duration::from_millis(500); +/// Longest a seed job is waited for (a first clone of a big repository). +const SEED_DEADLINE: Duration = Duration::from_secs(45 * 60); +/// Least time between two progress lines. +const PROGRESS_EVERY: Duration = Duration::from_millis(250); + +/// A push or clone that did not happen: its code and message, and the +/// GitHub App's install link when the machine had no way into the +/// repository. +#[derive(Debug)] +pub(super) struct SeedError { + code: String, + message: String, + install_url: Option, +} + +impl SeedError { + fn new(code: impl Into, message: impl Into) -> Self { + Self { code: code.into(), message: message.into(), install_url: None } + } + + /// A machine call or upload that failed, coded as the copy codes it. + fn of(error: &anyhow::Error) -> Self { + if let Some(seed) = error.downcast_ref::() { + return Self { + install_url: seed.install_url.clone(), + ..Self::new(&seed.code, &seed.message) + }; + } + let (code, message) = sync_cli::push_failure(error); + Self::new(code, message) + } + + fn line(&self) -> Value { + let mut line = json!({"ok": false, "error": self.message, "error_code": self.code}); + if let Some(url) = &self.install_url { + line.set("install_url", json!(url)); + } + line + } +} + +impl std::fmt::Display for SeedError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "{}: {}", self.code, self.message) + } +} + +impl std::error::Error for SeedError {} + +/// The error code of a failed [`push`] or [`push_since_last`]. +pub(super) fn failure_code(error: &anyhow::Error) -> String { + SeedError::of(error).code +} + +/// What a push did. +#[derive(Debug)] +pub(super) struct Pushed { + /// `seed`, `delta` or `copy`. + pub(super) mode: &'static str, + pub(super) unchanged: bool, + pub(super) bundle_bytes: u64, + pub(super) head: Option, + pub(super) work: Option, + pub(super) skipped_files: Value, + /// The copy's summary line. + pub(super) copy: Option, + /// Why a full push copied instead of seeding. + pub(super) fallback: Option, +} + +impl Pushed { + pub(super) fn json(&self) -> Value { + let mut line = json!({ + "ok": true, + "mode": self.mode, + "unchanged": self.unchanged, + "bundle_bytes": self.bundle_bytes, + "head": self.head, + "work": self.work, + "skipped_files": self.skipped_files, + }); + if let Some(copy) = &self.copy { + line.set("copy", copy.clone()); + } + if let Some(fallback) = &self.fallback { + line.set("fallback_code", json!(fallback.code)); + line.set("fallback_why", json!(fallback.message)); + if let Some(url) = &fallback.install_url { + line.set("install_url", json!(url)); + } + } + line + } +} + +pub(super) fn run(args: &[String]) -> i32 { + if matches!(args.first().map(String::as_str), None | Some("-h" | "--help" | "help")) { + println!("{USAGE}"); + return if args.is_empty() { 64 } else { 0 }; + } + let Some(command) = parse(args) else { + eprintln!("{USAGE}"); + return 64; + }; + let answer = match command { + Command::Push { socket, local, remote, since_last } => { + let pushed = if since_last { + push_since_last(&socket, &local, &remote) + } else { + push(&socket, &local, &remote) + }; + pushed.map(|pushed| pushed.json()).map_err(|error| SeedError::of(&error)) + } + Command::Clone { socket, repo, git_ref, remote } => { + clone(&socket, &repo, git_ref.as_deref(), &remote) + } + }; + match answer { + Ok(line) => { + println!("{line}"); + 0 + } + Err(error) => { + println!("{}", error.line()); + 1 + } + } +} + +enum Command { + Push { socket: PathBuf, local: PathBuf, remote: String, since_last: bool }, + Clone { socket: PathBuf, repo: String, git_ref: Option, remote: String }, +} + +fn parse(args: &[String]) -> Option { + let (action, rest) = args.split_first()?; + let (mut socket, mut repo, mut git_ref, mut since_last) = (None, None, None, false); + let mut positional = Vec::new(); + let mut rest = rest.iter(); + while let Some(arg) = rest.next() { + match arg.as_str() { + "--socket" => socket = Some(PathBuf::from(rest.next()?)), + "--repo" => repo = Some(rest.next()?.clone()), + "--ref" => git_ref = Some(rest.next()?.clone()), + "--since-last" => since_last = true, + flag if flag.starts_with('-') => return None, + value => positional.push(value), + } + } + let absolute = |remote: &str| { + (remote.starts_with('/') && remote.len() > 1) + .then(|| remote.trim_end_matches('/').to_string()) + }; + match (action.as_str(), positional.as_slice()) { + ("push", [local, remote]) if repo.is_none() && git_ref.is_none() => Some(Command::Push { + socket: socket?, + local: PathBuf::from(local), + remote: absolute(remote)?, + since_last, + }), + ("clone", [remote]) if !since_last => { + let repo = repo.filter(|repo| repo.split('/').count() == 2)?; + Some(Command::Clone { socket: socket?, repo, git_ref, remote: absolute(remote)? }) + } + _ => None, + } +} + +/// The machine as `identify` describes it. +struct Machine<'a> { + socket: &'a Path, + /// Its store id, when `identify` answered. + id: Option, + /// False when it is another build without `git-seed-v1`. + seeds: bool, +} + +impl<'a> Machine<'a> { + /// Refuses a machine too old for `fs-v1` (`machine_outdated`). + fn read(socket: &'a Path) -> Result { + let identity = machine_gate::identify(socket); + let lacks = |capability: &str| { + identity + .as_ref() + .and_then(|identity| machine_gate::outdated_identity(identity, &[capability])) + }; + if let Some(outdated) = lacks(cmux_tui_core::fs_ops::FS_CAPABILITY) { + return Err(SeedError::new(machine_gate::OUTDATED, outdated.message())); + } + let seeds = lacks(cmux_tui_core::git_seed::GIT_SEED_CAPABILITY).is_none(); + let id = identity + .as_ref() + .and_then(|identity| identity["session_id"].as_str()) + .map(str::to_owned); + Ok(Self { socket, id, seeds }) + } + + fn outdated() -> SeedError { + SeedError::new( + machine_gate::OUTDATED, + "the machine runs an older cmux-tui without git-seed-v1; update the machine", + ) + } +} + +/// A full push: the seed, else the copy (module docs). +pub(super) fn push(socket: &Path, local: &Path, remote: &str) -> anyhow::Result { + let machine = Machine::read(socket)?; + let local = std::fs::canonicalize(local).map_err(|error| { + SeedError::new("sync_failed", format!("cannot open {}: {error}", local.display())) + })?; + let fallback = match seed_full(&machine, &local, remote) { + Ok(pushed) => return Ok(pushed), + Err(Attempt::Failed(error)) => return Err(error.into()), + Err(Attempt::Fallback(why)) => why, + }; + copy(&machine, &local, remote, Some(fallback)) +} + +/// [`push`] with `--since-last` (module docs), for `cmux link agent move +/// --sync seed` too. Its errors' codes are [`failure_code`]'s. +pub(super) fn push_since_last(socket: &Path, local: &Path, remote: &str) -> anyhow::Result { + let machine = Machine::read(socket)?; + let local = std::fs::canonicalize(local).map_err(|error| { + SeedError::new("sync_failed", format!("cannot open {}: {error}", local.display())) + })?; + let no_previous = |why: &str| SeedError::new("seed.no_previous", why); + let Some(record) = record::load(&local, remote)? else { + return Err(no_previous("this Mac never pushed this folder to that machine folder").into()); + }; + if let (Some(was), Some(now)) = (&record.machine, &machine.id) + && was != now + { + return Err(no_previous("the last push of this folder went to another machine").into()); + } + if record.mode == Kept::Copy { + return copy(&machine, &local, remote, None); + } + let (Some(head), Some(index), Some(work)) = (record.head, record.index, record.work) else { + return Err(no_previous("the last seed's record is incomplete").into()); + }; + if !machine.seeds { + return Err(Machine::outdated().into()); + } + let bundle = LocalBundle::new(); + let previous = PreviousSeed { head, index, work }; + let planned = plan::plan_since(&local, &bundle.0, &previous).map_err(|fallback| { + let code = match fallback.code { + FallbackCode::NoPrevious => "seed.no_previous".to_string(), + code => format!("seed.{}", code.as_str()), + }; + SeedError::new(code, fallback.why) + })?; + if planned.unchanged { + return Ok(seeded("delta", &planned, None)); + } + let status = run_seed(machine.socket, remote, &planned, &bundle)?; + keep_seed(&machine, &local, remote, &planned); + Ok(seeded("delta", &planned, status["head"].as_str())) +} + +/// Why a full seed did not happen. +enum Attempt { + /// The copy goes instead. + Fallback(SeedError), + Failed(SeedError), +} + +/// The machine answers that make a full push copy instead. +const FALLBACK_CODES: [&str; 4] = + ["seed.needs_github", "seed.base_missing", "seed.clone_failed", "seed.unavailable"]; + +fn seed_full(machine: &Machine<'_>, local: &Path, remote: &str) -> Result { + let from_plan = |fallback: Fallback| { + Attempt::Fallback(SeedError::new(fallback.code.as_str(), fallback.why)) + }; + if !machine.seeds { + return Err(Attempt::Fallback(Machine::outdated())); + } + let bundle = LocalBundle::new(); + let mut planned = plan::plan(local, &bundle.0).map_err(from_plan)?; + let existed = sync_cli::remote_kind(machine.socket, remote) + .map_err(|error| Attempt::Failed(SeedError::of(&error)))? + .is_some(); + let mut result = run_seed(machine.socket, remote, &planned, &bundle); + // The remote lacks the base (a force-push, a branch never pushed): once + // more from the remote default branch's tip. + if let Err(error) = &result + && error.code == "seed.base_missing" + && let Some(tip) = planned.bases.last().filter(|tip| **tip != planned.params.base).cloned() + { + planned = plan::plan_with_base(local, &bundle.0, &tip).map_err(from_plan)?; + result = run_seed(machine.socket, remote, &planned, &bundle); + } + match result { + Ok(status) => { + keep_seed(machine, local, remote, &planned); + Ok(seeded("seed", &planned, status["head"].as_str())) + } + Err(error) if FALLBACK_CODES.contains(&error.code.as_str()) => { + // What the seed left is not the copy's to build on. + if !existed { + let _ = sync_cli::delete_remote(machine.socket, remote); + } + Err(Attempt::Fallback(error)) + } + Err(error) => Err(Attempt::Failed(error)), + } +} + +/// The streamed copy, kept as this folder's last push. +fn copy( + machine: &Machine<'_>, + local: &Path, + remote: &str, + fallback: Option, +) -> anyhow::Result { + let summary = sync_cli::push(machine.socket, local, remote)?; + let record = Record { + mode: Kept::Copy, + machine: machine.id.clone(), + url: None, + head: None, + index: None, + work: None, + }; + // A record that could not be kept only makes the next --since-last + // answer seed.no_previous; the copy itself is done. + let _ = record::save(local, remote, &record); + let line = summary.json(); + Ok(Pushed { + mode: "copy", + unchanged: summary.written == 0 && summary.deleted == 0, + bundle_bytes: 0, + head: None, + work: None, + skipped_files: line["skipped_files"].clone(), + copy: Some(line), + fallback, + }) +} + +fn keep_seed(machine: &Machine<'_>, local: &Path, remote: &str, planned: &Planned) { + let record = Record { + mode: Kept::Seed, + machine: machine.id.clone(), + url: Some(planned.params.url.clone()), + head: Some(planned.params.head.clone()), + index: Some(planned.params.index.clone()), + work: Some(planned.params.work.clone()), + }; + let _ = record::save(local, remote, &record); +} + +fn seeded(mode: &'static str, planned: &Planned, head: Option<&str>) -> Pushed { + let skipped: Vec = planned + .skipped + .iter() + .map(|item| { + let kind = if item.why.starts_with("a separate git repository") { + "repository" + } else { + "secret" + }; + json!({"path": item.path, "kind": kind, "why": item.why}) + }) + .collect(); + Pushed { + mode, + unchanged: planned.unchanged, + bundle_bytes: planned.bundle_bytes, + head: Some(head.unwrap_or(&planned.params.head).to_owned()), + work: Some(planned.params.work.clone()), + skipped_files: Value::Array(skipped), + copy: None, + fallback: None, + } +} + +/// Uploads the bundle beside `remote`, runs the seed and waits for it; the +/// job's last status. +fn run_seed( + socket: &Path, + remote: &str, + planned: &Planned, + bundle: &LocalBundle, +) -> Result { + let (parent, name) = remote + .rsplit_once('/') + .filter(|(parent, name)| !parent.is_empty() && !name.is_empty()) + .ok_or_else(|| SeedError::new("seed.bad_request", "the machine folder has no parent"))?; + let target = format!("{parent}/.cmux-seed-{name}-{}.bundle", unique()); + let total = planned.bundle_bytes; + let mut report = throttled(); + report(json!({"phase": "upload", "bytes_done": 0, "bytes_total": total}), true); + let uploaded = sync_cli::upload(socket, &bundle.0, &target, &mut |done| { + report(json!({"phase": "upload", "bytes_done": done, "bytes_total": total}), done >= total); + }); + let result = uploaded.map_err(|error| SeedError::of(&error)).and_then(|()| { + let mut start = serde_json::to_value(&planned.params) + .map_err(|error| SeedError::new("seed.bad_request", error.to_string()))?; + start.set("folder", json!(remote)); + start.set("bundle", json!(target)); + start_and_wait(socket, start) + }); + // The machine copied it when the job started; gone already after a seed. + let _ = sync_cli::delete_remote(socket, &target); + result +} + +/// `git.seed.start` with `start`, then its job's status until it ends. +fn start_and_wait(socket: &Path, start: Value) -> Result { + let started = sync_cli::machine_call(socket, "git.seed.start", start).map_err(|error| { + let mut refused = SeedError::of(&error); + // A daemon that does not know the op (or a link gate that does not + // admit it) does not seed. + if !refused.code.starts_with("seed.") && sync_cli::is_refusal(&error) { + refused.code = "seed.unavailable".into(); + } + refused + })?; + let job = started["job"] + .as_str() + .ok_or_else(|| SeedError::new("seed.bad_request", "git.seed.start answered no job"))? + .to_owned(); + let deadline = Instant::now() + SEED_DEADLINE; + let mut report = throttled(); + loop { + let status = sync_cli::machine_call(socket, "git.seed.status", json!({"job": job})) + .map_err(|error| SeedError::of(&error))?; + match status["state"].as_str() { + Some("done") => return Ok(status), + Some("failed") => { + return Err(SeedError { + install_url: status["install_url"].as_str().map(str::to_owned), + ..SeedError::new( + status["error_code"].as_str().unwrap_or("seed.failed"), + status["error"].as_str().unwrap_or("the seed failed on the machine"), + ) + }); + } + _ => {} + } + let progress = &status["progress"]; + report( + json!({ + "phase": status["phase"], + "bytes_done": progress["received_bytes"], + "objects": progress["objects"], + "objects_total": progress["total_objects"], + }), + false, + ); + if Instant::now() >= deadline { + return Err(SeedError::new( + "seed.timeout", + format!("the seed did not finish within {} minutes", SEED_DEADLINE.as_secs() / 60), + )); + } + std::thread::sleep(STATUS_EVERY); + } +} + +/// `cmux link seed clone` (module docs). +fn clone( + socket: &Path, + repo: &str, + git_ref: Option<&str>, + remote: &str, +) -> Result { + if !Machine::read(socket)?.seeds { + return Err(Machine::outdated()); + } + let mut start = json!({"folder": remote, "url": format!("https://github.com/{repo}.git"), + "repo": repo}); + if let Some(git_ref) = git_ref { + start.set("ref", json!(git_ref)); + } + let status = start_and_wait(socket, start)?; + Ok(json!({"ok": true, "mode": "clone", "head": status["head"], "head_ref": status["head_ref"]})) +} + +/// Prints `{"progress": PROGRESS}` on stderr, at most every +/// [`PROGRESS_EVERY`] unless `always`. +fn throttled() -> impl FnMut(Value, bool) { + let mut last: Option = None; + move |progress, always| { + if !always && last.is_some_and(|at| at.elapsed() < PROGRESS_EVERY) { + return; + } + last = Some(Instant::now()); + eprintln!("{}", json!({ "progress": progress })); + } +} + +/// A name no other push of this process or another uses at once. +fn unique() -> String { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_nanos()); + format!("{}-{nanos}", std::process::id()) +} + +/// This Mac's bundle file, removed when dropped. +struct LocalBundle(PathBuf); + +impl LocalBundle { + fn new() -> Self { + Self(std::env::temp_dir().join(format!("cmux-seed-push-{}.bundle", unique()))) + } +} + +impl Drop for LocalBundle { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +#[cfg(test)] +mod tests { + use super::{Command, parse}; + + fn args(list: &[&str]) -> Vec { + list.iter().map(|arg| (*arg).to_string()).collect() + } + + #[test] + fn parses_push_and_clone() { + let Some(Command::Push { local, remote, since_last, .. }) = + parse(&args(&["push", "--socket", "/s", "/a", "/home/w/a/", "--since-last"])) + else { + panic!("push"); + }; + assert_eq!(local.to_str(), Some("/a")); + assert_eq!(remote, "/home/w/a"); + assert!(since_last); + let Some(Command::Clone { repo, git_ref, remote, .. }) = + parse(&args(&["clone", "--socket", "/s", "--repo", "o/n", "--ref", "v1", "/home/w/n"])) + else { + panic!("clone"); + }; + assert_eq!( + (repo.as_str(), git_ref.as_deref(), remote.as_str()), + ("o/n", Some("v1"), "/home/w/n") + ); + for bad in [ + &["push", "--socket", "/s", "/a"][..], + &["push", "/a", "/home/w/a"], + &["push", "--socket", "/s", "/a", "relative"], + &["push", "--socket", "/s", "/a", "/"], + &["push", "--socket", "/s", "--repo", "o/n", "/a", "/b"], + &["clone", "--socket", "/s", "--repo", "o", "/b"], + &["clone", "--socket", "/s", "--repo", "o/n", "/b", "--since-last"], + &["clone", "--socket", "/s", "/b"], + &["plan", "--socket", "/s", "/a", "/b"], + &["push", "--socket", "/s", "/a", "/b", "--x"], + ] { + assert!(parse(&args(bad)).is_none(), "{bad:?}"); + } + } +} diff --git a/cmux-tui/crates/cmux-tui/src/link/seed_cli/record.rs b/cmux-tui/crates/cmux-tui/src/link/seed_cli/record.rs new file mode 100644 index 000000000000..04791522926c --- /dev/null +++ b/cmux-tui/crates/cmux-tui/src/link/seed_cli/record.rs @@ -0,0 +1,92 @@ +//! This Mac's record of the last `cmux link seed push` of a folder to a +//! machine folder: whether it seeded or copied, the machine it went to, and +//! for a seed the commits the machine now has, so `--since-last` sends only +//! what changed since ([`super::super::seed::plan::plan_since`]). +//! +//! The record is `/link-seed/.json` +//! (0600, its folder 0700), beside `cmux link sync`'s own record +//! (`/link-sync`), written to a temporary name and renamed. + +use std::io::Write as _; +use std::os::unix::fs::{DirBuilderExt as _, OpenOptionsExt as _}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, bail}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; + +/// How the last push put the folder on the machine. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(super) enum Kept { + /// A `git-seed-v1` seed (full or delta): the machine has its commits. + Seed, + /// The streamed copy (`cmux link sync push`), whose own record makes a + /// re-push incremental. + Copy, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(super) struct Record { + pub(super) mode: Kept, + /// The machine daemon's store id (`identify`'s `session_id`), when it + /// answered one. + #[serde(default)] + pub(super) machine: Option, + /// For a seed: the GitHub URL and the commits the machine has. + #[serde(default)] + pub(super) url: Option, + #[serde(default)] + pub(super) head: Option, + #[serde(default)] + pub(super) index: Option, + #[serde(default)] + pub(super) work: Option, +} + +fn path_of(local: &Path, remote_root: &str) -> anyhow::Result { + let Some(state) = cmux_tui_core::platform::workspace_state_dir() else { + bail!("this Mac has no state folder to keep the seed record in"); + }; + let mut key = Sha256::new(); + key.update(local.as_os_str().as_encoded_bytes()); + key.update([0]); + key.update(remote_root.as_bytes()); + let name: String = key.finalize().iter().map(|byte| format!("{byte:02x}")).collect(); + Ok(state.join("link-seed").join(format!("{name}.json"))) +} + +/// The record of `local` pushed to `remote_root`; None when there is none. +pub(super) fn load(local: &Path, remote_root: &str) -> anyhow::Result> { + let path = path_of(local, remote_root)?; + match std::fs::read(&path) { + Ok(bytes) => serde_json::from_slice(&bytes) + .map(Some) + .with_context(|| format!("{} is not a seed record", path.display())), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error).with_context(|| format!("cannot read {}", path.display())), + } +} + +/// Keeps `record` for `local` pushed to `remote_root`. +pub(super) fn save(local: &Path, remote_root: &str, record: &Record) -> anyhow::Result<()> { + let path = path_of(local, remote_root)?; + let dir = path.parent().context("no state folder")?; + std::fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(dir) + .with_context(|| format!("cannot make {}", dir.display()))?; + let part = path.with_extension(format!("{}.part", std::process::id())); + let _ = std::fs::remove_file(&part); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(&part) + .with_context(|| format!("cannot write {}", part.display()))?; + file.write_all(&serde_json::to_vec(record)?)?; + file.sync_all()?; + std::fs::rename(&part, &path).with_context(|| format!("cannot write {}", path.display())) +} diff --git a/cmux-tui/crates/cmux-tui/src/link/sync_cli.rs b/cmux-tui/crates/cmux-tui/src/link/sync_cli.rs index 2ec6d663c28f..1f22529f6705 100644 --- a/cmux-tui/crates/cmux-tui/src/link/sync_cli.rs +++ b/cmux-tui/crates/cmux-tui/src/link/sync_cli.rs @@ -244,6 +244,12 @@ pub(super) fn pull_into(socket: &Path, local: &Path, remote_root: &str) -> Back } } +/// Whether `error` is the machine daemon's own refusal (an `ok: false` +/// answer), not a connection that failed. +pub(super) fn is_refusal(error: &anyhow::Error) -> bool { + remote::refusal_code(error).is_some() +} + /// The error code of a failed push. pub(super) fn push_failure_code(error: &anyhow::Error) -> String { push_failure(error).0 @@ -295,6 +301,43 @@ pub(super) fn copy_file_with_progress( write_with_progress(&mut remote, target, &file, progress).map(|_| ()) } +/// Uploads the regular file at `local` (never through a symlink) to +/// `target` on the machine as 0600, making its folder; `progress` hears the +/// bytes read so far as the parts go (`cmux link seed push`'s bundle). +pub(super) fn upload( + socket: &Path, + local: &Path, + target: &str, + progress: &mut dyn FnMut(u64), +) -> anyhow::Result<()> { + struct Counted { + inner: R, + read: std::rc::Rc>, + } + impl std::io::Read for Counted { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let read = self.inner.read(buffer)?; + self.read.set(self.read.get() + read as u64); + Ok(read) + } + } + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(local) + .with_context(|| format!("cannot read {}", local.display()))?; + if !file.metadata()?.is_file() { + anyhow::bail!("{} is not a regular file", local.display()); + } + let read = std::rc::Rc::new(std::cell::Cell::new(0)); + let mut reader = Counted { inner: file, read: read.clone() }; + let mut remote = Remote::connect(socket)?; + let (parent, _) = remote::split(target)?; + remote.mkdir_all(parent)?; + remote.write_with_progress(target, &mut reader, 0o600, &mut || progress(read.get()))?; + Ok(()) +} + /// Deletes the folder at `path` on the machine whose daemon `socket` /// reaches; one already gone is fine. pub(super) fn delete_remote(socket: &Path, path: &str) -> anyhow::Result<()> { diff --git a/cmux-tui/crates/cmux-tui/tests/cloud_move_journey.rs b/cmux-tui/crates/cmux-tui/tests/cloud_move_journey.rs index c8e6ea14dc63..705e737bba24 100644 --- a/cmux-tui/crates/cmux-tui/tests/cloud_move_journey.rs +++ b/cmux-tui/crates/cmux-tui/tests/cloud_move_journey.rs @@ -13,9 +13,13 @@ //! (link/cloud_fs.rs, agent_start_host.rs), which stands in for the //! root-owned `/etc/cmux`. //! -//! The journey runs the real CLI: `cmux link sync push` copies the folder with -//! its git state through the VM daemon's `fs-v1`, then `cmux link agent move` -//! pumps the chat between the two daemons' `agent-transfer-v1` verbs. Both +//! The journey runs the real CLI: `cmux link seed push` seeds the folder with +//! its git state through the VM daemon's `git-seed-v1` (the VM clones the +//! folder's GitHub remote, a folder of bare repositories named by the +//! debug-only `CMUX_TEST_GIT_SEED_GITHUB` seam, and applies one bundle of the +//! local diff it got through `fs-v1`), then `cmux link agent move` pumps the +//! chat between the two daemons' `agent-transfer-v1` verbs, sending what the +//! running turn wrote since as a delta bundle (`--sync seed`). Both //! commands are given the VM daemon's own Unix socket, as the trusted local //! client the Cloud link's `remote connect` bridge makes the app on the VM. //! @@ -37,14 +41,23 @@ const QUEUED: &str = "queued for the cloud"; const QUEUED_ID: &str = "p-journey-queued"; const DRAFT: &str = "an unsent thought"; -/// A working copy with a commit, an uncommitted change and an untracked file. -fn dirty_repo(at: &Path) -> PathBuf { +/// A working copy with a commit pushed to its GitHub remote (`acme/app` +/// under `github`), an uncommitted change and an untracked file. +fn dirty_repo(at: &Path, github: &Path) -> PathBuf { let repo = at.join("app"); fs::create_dir_all(repo.join("src")).unwrap(); git(&repo, &["init", "-q", "-b", "main"]); fs::write(repo.join("src/lib.txt"), "committed\n").unwrap(); git(&repo, &["add", "-A"]); git(&repo, &["commit", "-q", "-m", "first"]); + let bare = github.join("acme/app.git"); + fs::create_dir_all(&bare).unwrap(); + git(&bare, &["init", "-q", "--bare"]); + git(&bare, &["config", "uploadpack.allowFilter", "true"]); + git(&repo, &["remote", "add", "origin", "https://github.com/acme/app.git"]); + let local = format!("url.file://{}/.insteadOf", github.display()); + git(&repo, &["config", &local, "https://github.com/"]); + git(&repo, &["-c", "protocol.file.allow=always", "push", "-q", "-u", "origin", "main"]); fs::write(repo.join("src/lib.txt"), "committed\nuncommitted change\n").unwrap(); fs::write(repo.join("notes.txt"), "untracked\n").unwrap(); repo @@ -100,29 +113,34 @@ fn cloud_move_journey_moves_a_dirty_folder_and_its_chat_to_a_personal_cloud_vm() let root = fs::canonicalize(&root).unwrap(); let _cleanup = Cleanup(root.clone()); + let github = root.join("github"); let mac = Machine::start(&root, "mac", Vec::new()); let vm = Machine::start( &root, "vm", vec![ ("CMUX_TEST_CLOUD_ETC", cloud_etc(&root).to_string_lossy().into_owned()), + ("CMUX_TEST_GIT_SEED_GITHUB", github.to_string_lossy().into_owned()), ("CMUX_VM_ID", "vm-cloud-move-journey".to_owned()), ("CMUX_CODEROUTER_URL", "http://127.0.0.1:9".to_owned()), ], ); - // 1. The dirty folder goes to the VM's ~/work with its git state. - let repo = dirty_repo(&mac.home()); + // 1. The dirty folder goes to the VM's ~/work with its git state: a + // clone of its GitHub remote and a bundle of the local diff. + let repo = dirty_repo(&mac.home(), &github); let synced = vm.home().join("work/app"); let (ok, summary) = run_json( Command::new(bin()) .env("CMUX_TUI_STATE_DIR", mac.dir.join("state")) - .args(["link", "sync", "push", "--socket"]) + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .args(["link", "seed", "push", "--socket"]) .arg(&vm.socket) .arg(&repo) .arg(&synced), ); - assert!(ok && summary["ok"] == json!(true), "sync push: {summary}"); + assert!(ok && summary["ok"] == json!(true), "seed push: {summary}"); + assert_eq!(summary["mode"], json!("seed"), "the folder was not seeded: {summary}"); assert!(synced.join(".git").is_dir(), "no repository on the VM"); assert_eq!( fs::read_to_string(synced.join("src/lib.txt")).unwrap(), @@ -134,14 +152,17 @@ fn cloud_move_journey_moves_a_dirty_folder_and_its_chat_to_a_personal_cloud_vm() assert_eq!(git(&synced, &["rev-parse", "HEAD"]), git(&repo, &["rev-parse", "HEAD"])); // 2. The chat moves mid-turn: its running turn stops for the move, and - // what the turn wrote after the sync above goes with it (`--sync push`, - // copied once the chat is frozen). + // what the turn wrote after the seed above goes with it (`--sync seed`, a + // delta bundle sent once the chat is frozen). let mut mac_acpmux = mac.acpmux(); let session = chat_with_history_draft_and_queue(&mut mac_acpmux, &repo); let surface = bound_agent_tab(&mac, &session); fs::write(repo.join("src/turn.txt"), "written by the running turn\n").unwrap(); let (ok, moved) = run_json( Command::new(bin()) + // This Mac's state, where the seed above kept its record. + .env("CMUX_TUI_STATE_DIR", mac.dir.join("state")) + .env("GIT_CONFIG_GLOBAL", "/dev/null") .args(["link", "agent", "move", "--from-socket"]) .arg(&mac.socket) .args(["--surface", &surface.to_string(), "--to-socket"]) @@ -150,7 +171,7 @@ fn cloud_move_journey_moves_a_dirty_folder_and_its_chat_to_a_personal_cloud_vm() .arg(&synced) .arg("--interrupt") .args(["--vm-kind", "owner"]) - .args(["--sync", "push", "--sync-socket"]) + .args(["--sync", "seed", "--sync-socket"]) .arg(&vm.socket) .arg("--sync-local") .arg(&repo) diff --git a/cmux-tui/crates/cmux-tui/tests/link_seed.rs b/cmux-tui/crates/cmux-tui/tests/link_seed.rs new file mode 100644 index 000000000000..554c6b58277f --- /dev/null +++ b/cmux-tui/crates/cmux-tui/tests/link_seed.rs @@ -0,0 +1,367 @@ +//! `cmux link seed push|clone`: a git folder with a GitHub remote reaches a +//! Cloud machine as a clone plus one bundle of the local diff, and anything +//! else as the streamed copy. The machine side is the real `FsService` and +//! `GitSeedService` behind a socket that answers lines the way the daemon +//! does; "GitHub" is a folder of bare repositories the service clones from +//! (`GitSeedService::with_test_github`). +#![cfg(unix)] + +use std::io::{BufRead as _, BufReader, Write as _}; +use std::os::unix::net::UnixListener; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use cmux_tui_core::fs_ops::{FsService, Roots, stream}; +use cmux_tui_core::git_seed::{self, GitSeedService}; +use serde_json::{Value, json}; + +const URL: &str = "https://github.com/acme/widget.git"; + +fn git(dir: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .args(["-c", "user.name=t", "-c", "user.email=t@example.com", "-c", "commit.gpgsign=false"]) + .args(["-c", "init.defaultBranch=main", "-c", "protocol.file.allow=always"]) + .args(args) + .current_dir(dir) + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_NOSYSTEM", "1") + .output() + .unwrap(); + assert!(output.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&output.stderr)); + String::from_utf8(output.stdout).unwrap().trim().to_owned() +} + +fn write(path: &Path, text: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap(); +} + +/// How the fake machine answers. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Serves { + Seed, + /// Another build without `git-seed-v1`. + CopyOnly, +} + +/// One test's world: "GitHub", this Mac's working copy, the machine's home +/// and its socket. +struct World { + _dir: tempfile::TempDir, + _sockets: tempfile::TempDir, + root: PathBuf, + github: PathBuf, + work: PathBuf, + home: PathBuf, + socket: PathBuf, + /// `fs.write` requests the machine answered. + writes: &'static AtomicUsize, +} + +impl World { + fn new(serves: Serves) -> Self { + let dir = tempfile::tempdir().unwrap(); + let root = std::fs::canonicalize(dir.path()).unwrap(); + let github = root.join("github"); + let bare = github.join("acme/widget.git"); + std::fs::create_dir_all(&bare).unwrap(); + git(&bare, &["init", "-q", "--bare"]); + git(&bare, &["config", "uploadpack.allowFilter", "true"]); + git(&bare, &["config", "uploadpack.allowAnySHA1InWant", "true"]); + let work = root.join("widget"); + git(&root, &["clone", "-q", &format!("file://{}", bare.display()), "widget"]); + write(&work.join("README.md"), "widget\n"); + write(&work.join("src/lib.rs"), "pub fn one() {}\n"); + write(&work.join(".gitignore"), "node_modules/\n"); + git(&work, &["add", "-A"]); + git(&work, &["commit", "-q", "-m", "first"]); + git(&work, &["push", "-q", "origin", "HEAD:main"]); + git(&work, &["branch", "-q", "--set-upstream-to=origin/main"]); + git(&work, &["remote", "set-url", "origin", URL]); + git( + &work, + &[ + "config", + &format!("url.file://{}/.insteadOf", github.display()), + "https://github.com/", + ], + ); + let home = root.join("home"); + std::fs::create_dir_all(home.join("work")).unwrap(); + let sockets = tempfile::Builder::new().prefix("ls").tempdir_in("/tmp").unwrap(); + let socket = sockets.path().join("m.sock"); + let writes = serve(&socket, &home, &github, serves); + Self { _dir: dir, _sockets: sockets, root, github, work, home, socket, writes } + } + + fn remote(&self, name: &str) -> PathBuf { + self.home.join("work").join(name) + } + + /// `cmux link seed ARGS`: exit status, its answer line, its stderr lines. + fn seed(&self, args: &[&str]) -> (bool, Value, Vec) { + let output = Command::new(env!("CARGO_BIN_EXE_cmux-tui")) + .env("CMUX_TUI_STATE_DIR", self.root.join("state")) + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_NOSYSTEM", "1") + .args(["link", "seed"]) + .args(args) + .output() + .unwrap(); + let answer = serde_json::from_slice(&output.stdout).unwrap_or_else(|_| { + panic!( + "one JSON answer on stdout: {}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) + }); + let progress = String::from_utf8_lossy(&output.stderr) + .lines() + .filter_map(|line| serde_json::from_str(line).ok()) + .collect(); + (output.status.success(), answer, progress) + } + + fn push(&self, local: &Path, remote: &Path, since_last: bool) -> (bool, Value, Vec) { + let mut args = vec!["push", "--socket", self.socket.to_str().unwrap()]; + args.extend([local.to_str().unwrap(), remote.to_str().unwrap()]); + if since_last { + args.push("--since-last"); + } + self.seed(&args) + } +} + +/// Serves `fs.*` and `git.seed.*` lines (and `identify`) on `socket`, one +/// thread per connection, until the test process ends. +fn serve(socket: &Path, home: &Path, github: &Path, serves: Serves) -> &'static AtomicUsize { + let listener = UnixListener::bind(socket).unwrap(); + let roots = Roots::new([home.to_path_buf()]); + let fs: &'static FsService = Box::leak(Box::new(FsService::new(roots.clone()))); + let seed: &'static GitSeedService = + Box::leak(Box::new(GitSeedService::with_test_github(roots, github))); + let writes: &'static AtomicUsize = Box::leak(Box::default()); + std::thread::spawn(move || { + for connection in listener.incoming() { + let Ok(connection) = connection else { return }; + std::thread::spawn(move || { + let mut writer = connection.try_clone().unwrap(); + for line in BufReader::new(connection).lines() { + let Ok(line) = line else { return }; + let request: Value = serde_json::from_str(&line).unwrap(); + let id = request["id"].clone(); + let cmd = request["cmd"].as_str().unwrap_or_default().to_string(); + let reply = if cmd == "identify" { + let capabilities = match serves { + Serves::Seed => json!(["fs-v1", "git-seed-v1"]), + Serves::CopyOnly => json!(["fs-v1"]), + }; + json!({"id": id, "ok": true, "data": {"capabilities": capabilities, + "build_id": "machine-build", "session_id": "machine-a"}}) + } else if git_seed::frame_command(&line).is_some() { + if serves == Serves::Seed { + git_seed::answer_line(Some(seed), true, &line) + } else { + json!({"id": id, "ok": false, "error_code": "unknown_command", + "error": "unknown command"}) + } + } else { + if cmd == "fs.write" { + writes.fetch_add(1, Ordering::SeqCst); + } + stream::answer(&id, fs.call("local", &cmd, request)) + }; + let mut text = reply.to_string(); + text.push('\n'); + if writer.write_all(text.as_bytes()).is_err() { + return; + } + } + }); + } + }); + writes +} + +/// Every file below `root` but `.git`, by relative path. +fn files(root: &Path) -> std::collections::BTreeMap { + let mut found = std::collections::BTreeMap::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir).unwrap() { + let entry = entry.unwrap(); + if entry.file_name() == ".git" { + continue; + } + if entry.file_type().unwrap().is_dir() { + stack.push(entry.path()); + } else { + let relative = entry.path().strip_prefix(root).unwrap().display().to_string(); + found.insert(relative, std::fs::read_to_string(entry.path()).unwrap_or_default()); + } + } + } + found +} + +#[test] +fn a_full_seed_then_deltas_carry_the_working_state() { + let w = World::new(Serves::Seed); + write(&w.work.join("src/lib.rs"), "pub fn one() {}\npub fn two() {}\n"); + git(&w.work, &["commit", "-q", "-am", "unpushed"]); + write(&w.work.join("README.md"), "staged\n"); + git(&w.work, &["add", "README.md"]); + write(&w.work.join("notes/todo.md"), "untracked\n"); + write(&w.work.join(".env"), "TOKEN=secret\n"); + write(&w.work.join("node_modules/x/index.js"), "ignored\n"); + let remote = w.remote("widget"); + + let (ok, seeded, progress) = w.push(&w.work, &remote, false); + assert!(ok, "{seeded}"); + assert_eq!(seeded["mode"], json!("seed"), "{seeded}"); + assert_eq!(seeded["unchanged"], json!(false)); + assert!(seeded["bundle_bytes"].as_u64().unwrap() > 0, "{seeded}"); + assert_eq!(seeded["head"], json!(git(&w.work, &["rev-parse", "HEAD"]))); + assert_eq!(seeded["skipped_files"][0]["path"], json!(".env"), "{seeded}"); + assert_eq!(seeded["skipped_files"][0]["kind"], json!("secret")); + assert!(seeded.get("fallback_code").is_none(), "{seeded}"); + let phases: Vec<&str> = + progress.iter().filter_map(|line| line["progress"]["phase"].as_str()).collect(); + assert_eq!(phases.first(), Some(&"upload"), "{progress:?}"); + // The machine has the branch at HEAD, the staged file and the tree. + assert_eq!(git(&remote, &["rev-parse", "HEAD"]), git(&w.work, &["rev-parse", "HEAD"])); + assert_eq!(git(&remote, &["symbolic-ref", "HEAD"]), "refs/heads/main"); + assert_eq!(git(&remote, &["diff", "--cached", "--name-only"]), "README.md"); + let mut expected = files(&w.work); + expected.retain(|path, _| path != ".env" && !path.starts_with("node_modules/")); + assert_eq!(files(&remote), expected); + // No bundle is left beside the folder. + let left: Vec<_> = std::fs::read_dir(w.home.join("work")) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!(left, ["widget"], "{left:?}"); + + // A change since: only it goes. + write(&w.work.join("notes/todo.md"), "untracked, edited\n"); + let (ok, delta, _) = w.push(&w.work, &remote, true); + assert!(ok, "{delta}"); + assert_eq!(delta["mode"], json!("delta"), "{delta}"); + assert!(delta["bundle_bytes"].as_u64().unwrap() > 0, "{delta}"); + assert_eq!( + std::fs::read_to_string(remote.join("notes/todo.md")).unwrap(), + "untracked, edited\n" + ); + + // Nothing changed: nothing is uploaded. + let before = w.writes.load(Ordering::SeqCst); + let (ok, same, _) = w.push(&w.work, &remote, true); + assert!(ok, "{same}"); + assert_eq!(same["mode"], json!("delta")); + assert_eq!(same["unchanged"], json!(true), "{same}"); + assert_eq!(same["bundle_bytes"], json!(0)); + assert_eq!(w.writes.load(Ordering::SeqCst), before, "an unchanged delta uploaded"); +} + +#[test] +fn since_last_never_copies_and_says_why() { + let w = World::new(Serves::Seed); + let remote = w.remote("widget"); + let (ok, none, _) = w.push(&w.work, &remote, true); + assert!(!ok); + assert_eq!(none["error_code"], json!("seed.no_previous"), "{none}"); + assert!(!remote.exists(), "--since-last copied"); + + let (ok, seeded, _) = w.push(&w.work, &remote, false); + assert!(ok, "{seeded}"); + // A file changed on the machine since the seed. + write(&remote.join("README.md"), "the agent was here\n"); + write(&w.work.join("src/lib.rs"), "changed here\n"); + let (ok, changed, _) = w.push(&w.work, &remote, true); + assert!(!ok); + assert_eq!(changed["error_code"], json!("seed.machine_changed"), "{changed}"); + // The folder on the machine is no seed of this repository any more. + std::fs::remove_dir_all(remote.join(".git")).unwrap(); + let (ok, mismatch, _) = w.push(&w.work, &remote, true); + assert!(!ok); + assert_eq!(mismatch["error_code"], json!("seed.since_mismatch"), "{mismatch}"); +} + +#[test] +fn folders_that_cannot_be_seeded_are_copied_and_say_why() { + let w = World::new(Serves::Seed); + // Not git. + let plain = w.root.join("plain"); + write(&plain.join("a.txt"), "a\n"); + let (ok, copied, _) = w.push(&plain, &w.remote("plain"), false); + assert!(ok, "{copied}"); + assert_eq!(copied["mode"], json!("copy")); + assert_eq!(copied["fallback_code"], json!("not_git"), "{copied}"); + assert!(copied["fallback_why"].as_str().is_some_and(|why| !why.is_empty())); + assert_eq!(copied["copy"]["written"], json!(1), "{copied}"); + assert_eq!(std::fs::read_to_string(w.remote("plain").join("a.txt")).unwrap(), "a\n"); + // A copy's --since-last re-pushes it, incrementally. + write(&plain.join("b.txt"), "b\n"); + let (ok, again, _) = w.push(&plain, &w.remote("plain"), true); + assert!(ok, "{again}"); + assert_eq!(again["mode"], json!("copy"), "{again}"); + assert_eq!( + (again["copy"]["written"].clone(), again["copy"]["unchanged"].clone()), + (json!(1), json!(1)) + ); + assert!(again.get("fallback_code").is_none()); + + // Not GitHub. + git(&w.work, &["remote", "set-url", "origin", "https://gitlab.com/acme/widget.git"]); + let (ok, gitlab, _) = w.push(&w.work, &w.remote("gitlab"), false); + assert!(ok, "{gitlab}"); + assert_eq!(gitlab["mode"], json!("copy")); + assert_eq!(gitlab["fallback_code"], json!("not_github"), "{gitlab}"); + assert!(w.remote("gitlab").join(".git").is_dir()); +} + +#[test] +fn a_repository_the_machine_cannot_clone_is_copied_and_leaves_nothing_behind() { + let w = World::new(Serves::Seed); + // "GitHub" no longer has it. + std::fs::rename(w.github.join("acme"), w.github.join("gone")).unwrap(); + let remote = w.remote("widget"); + let (ok, copied, _) = w.push(&w.work, &remote, false); + assert!(ok, "{copied}"); + assert_eq!(copied["mode"], json!("copy"), "{copied}"); + assert_eq!(copied["fallback_code"], json!("seed.clone_failed"), "{copied}"); + assert_eq!(std::fs::read_to_string(remote.join("README.md")).unwrap(), "widget\n"); +} + +#[test] +fn a_machine_without_git_seed_gets_the_copy() { + let w = World::new(Serves::CopyOnly); + let remote = w.remote("widget"); + let (ok, copied, _) = w.push(&w.work, &remote, false); + assert!(ok, "{copied}"); + assert_eq!(copied["mode"], json!("copy")); + assert_eq!(copied["fallback_code"], json!("machine_outdated"), "{copied}"); + let (ok, clone, _) = + w.seed(&["clone", "--socket", w.socket.to_str().unwrap(), "--repo", "acme/widget", "/x/y"]); + assert!(!ok); + assert_eq!(clone["error_code"], json!("machine_outdated"), "{clone}"); +} + +#[test] +fn seed_clone_checks_out_a_repository_with_no_folder_here() { + let w = World::new(Serves::Seed); + let remote = w.remote("cloned"); + let socket = w.socket.to_str().unwrap(); + let (ok, cloned, _) = + w.seed(&["clone", "--socket", socket, "--repo", "acme/widget", remote.to_str().unwrap()]); + assert!(ok, "{cloned}"); + assert_eq!(cloned["mode"], json!("clone")); + assert_eq!(cloned["head"], json!(git(&w.work, &["rev-parse", "HEAD"])), "{cloned}"); + assert_eq!(std::fs::read_to_string(remote.join("README.md")).unwrap(), "widget\n"); + let missing = w.remote("missing"); + let (ok, failed, _) = + w.seed(&["clone", "--socket", socket, "--repo", "acme/nothing", missing.to_str().unwrap()]); + assert!(!ok); + assert_eq!(failed["error_code"], json!("seed.clone_failed"), "{failed}"); +} From 680454bc95ab641c27e3d5b495a76df859ab3321 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 01:09:16 -0800 Subject: [PATCH 2/4] test: the fresh-image journey's repo has its GitHub remote too Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-3bab30503f0b9a7059e5 --- cmux-tui/crates/cmux-tui/tests/cloud_move_journey.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/tests/cloud_move_journey.rs b/cmux-tui/crates/cmux-tui/tests/cloud_move_journey.rs index 705e737bba24..7aa202358d3c 100644 --- a/cmux-tui/crates/cmux-tui/tests/cloud_move_journey.rs +++ b/cmux-tui/crates/cmux-tui/tests/cloud_move_journey.rs @@ -301,7 +301,7 @@ fn a_fresh_image_vm_serves_fs_v1_from_the_baked_env_and_the_bound_id() { "the daemon must not write a per-home model-plane.env (it would shadow the image's)" ); - let repo = dirty_repo(&mac.home()); + let repo = dirty_repo(&mac.home(), &root.join("github")); let synced = vm.home().join("work/app"); let (ok, summary) = run_json( Command::new(bin()) From aad041c4f477ea5925cc2d60b98945bc388f307b Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 01:23:03 -0800 Subject: [PATCH 3/4] link seed push: say why before a fallback copy's progress Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-3bab30503f0b9a7059e5 --- cmux-tui/crates/cmux-tui/src/link/seed_cli.rs | 11 ++++++++++- cmux-tui/crates/cmux-tui/tests/link_seed.rs | 8 +++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/link/seed_cli.rs b/cmux-tui/crates/cmux-tui/src/link/seed_cli.rs index 4818c5425b42..3097dc007f1f 100644 --- a/cmux-tui/crates/cmux-tui/src/link/seed_cli.rs +++ b/cmux-tui/crates/cmux-tui/src/link/seed_cli.rs @@ -32,7 +32,9 @@ //! "bytes_done","bytes_total"}}` while the bundle goes, then //! `{"progress":{"phase":clone|fetch|bundle|checkout|submodules|lfs, //! "bytes_done","objects","objects_total"}}` from the machine (the copy's own -//! lines, `{"progress":{"files_done",...}}`, for a copy). One JSON line on +//! lines, `{"progress":{"files_done",...}}`, for a copy, after one +//! `{"progress":{"phase":"copy","fallback_code","install_url"?}}` saying why +//! a seed copied instead). One JSON line on //! stdout: `{"ok":true,"mode":"seed"|"delta"|"copy","unchanged":bool, //! "bundle_bytes":N,"head":SHA|null,"work":SHA|null,"skipped_files":[{"path", //! "kind","why"}]}`, a copy adding `"copy":{the sync summary}` and a fallback @@ -275,6 +277,13 @@ pub(super) fn push(socket: &Path, local: &Path, remote: &str) -> anyhow::Result< Err(Attempt::Failed(error)) => return Err(error.into()), Err(Attempt::Fallback(why)) => why, }; + // The copy's own lines follow; this one says why, so the card can tell + // the person (and offer the App install) while the copy runs. + let mut announce = json!({"phase": "copy", "fallback_code": fallback.code}); + if let Some(url) = &fallback.install_url { + announce["install_url"] = json!(url); + } + eprintln!("{}", json!({ "progress": announce })); copy(&machine, &local, remote, Some(fallback)) } diff --git a/cmux-tui/crates/cmux-tui/tests/link_seed.rs b/cmux-tui/crates/cmux-tui/tests/link_seed.rs index 554c6b58277f..4b390e407be9 100644 --- a/cmux-tui/crates/cmux-tui/tests/link_seed.rs +++ b/cmux-tui/crates/cmux-tui/tests/link_seed.rs @@ -294,10 +294,16 @@ fn folders_that_cannot_be_seeded_are_copied_and_say_why() { // Not git. let plain = w.root.join("plain"); write(&plain.join("a.txt"), "a\n"); - let (ok, copied, _) = w.push(&plain, &w.remote("plain"), false); + let (ok, copied, progress) = w.push(&plain, &w.remote("plain"), false); assert!(ok, "{copied}"); assert_eq!(copied["mode"], json!("copy")); assert_eq!(copied["fallback_code"], json!("not_git"), "{copied}"); + // The copy says why before its own lines, for the card's live row. + assert_eq!( + progress.first().map(|line| &line["progress"]), + Some(&json!({"phase": "copy", "fallback_code": "not_git"})), + "{progress:?}" + ); assert!(copied["fallback_why"].as_str().is_some_and(|why| !why.is_empty())); assert_eq!(copied["copy"]["written"], json!(1), "{copied}"); assert_eq!(std::fs::read_to_string(w.remote("plain").join("a.txt")).unwrap(), "a\n"); From 9aabf92913841016619d40206d2c6193e1cae8ed Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 01:27:38 -0800 Subject: [PATCH 4/4] link seed push: bundle in a private folder; clean a failed upload's part; LFS on clone; a machine that does not answer identify is an error Review (gpt-6-astra) at aad041c4: the bundle was world-readable under umask 022. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-3bab30503f0b9a7059e5 --- cmux-tui/crates/cmux-tui/src/link/seed_cli.rs | 64 +++++++++++++++---- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/link/seed_cli.rs b/cmux-tui/crates/cmux-tui/src/link/seed_cli.rs index 3097dc007f1f..a8b4fa94bcfd 100644 --- a/cmux-tui/crates/cmux-tui/src/link/seed_cli.rs +++ b/cmux-tui/crates/cmux-tui/src/link/seed_cli.rs @@ -242,6 +242,12 @@ impl<'a> Machine<'a> { /// Refuses a machine too old for `fs-v1` (`machine_outdated`). fn read(socket: &'a Path) -> Result { let identity = machine_gate::identify(socket); + if identity.is_none() { + return Err(SeedError::new( + "sync_failed", + format!("the machine at {} did not answer", socket.display()), + )); + } let lacks = |capability: &str| { identity .as_ref() @@ -312,9 +318,9 @@ pub(super) fn push_since_last(socket: &Path, local: &Path, remote: &str) -> anyh if !machine.seeds { return Err(Machine::outdated().into()); } - let bundle = LocalBundle::new(); + let bundle = LocalBundle::new()?; let previous = PreviousSeed { head, index, work }; - let planned = plan::plan_since(&local, &bundle.0, &previous).map_err(|fallback| { + let planned = plan::plan_since(&local, &bundle.file, &previous).map_err(|fallback| { let code = match fallback.code { FallbackCode::NoPrevious => "seed.no_previous".to_string(), code => format!("seed.{}", code.as_str()), @@ -347,8 +353,8 @@ fn seed_full(machine: &Machine<'_>, local: &Path, remote: &str) -> Result, local: &Path, remote: &str) -> Result= total); }); let result = uploaded.map_err(|error| SeedError::of(&error)).and_then(|()| { @@ -475,6 +485,7 @@ fn run_seed( }); // The machine copied it when the job started; gone already after a seed. let _ = sync_cli::delete_remote(socket, &target); + let _ = sync_cli::delete_remote(socket, &part); result } @@ -541,8 +552,9 @@ fn clone( if !Machine::read(socket)?.seeds { return Err(Machine::outdated()); } + // LFS files come down too; a machine without git-lfs notes it and goes on. let mut start = json!({"folder": remote, "url": format!("https://github.com/{repo}.git"), - "repo": repo}); + "repo": repo, "lfs": true}); if let Some(git_ref) = git_ref { start.set("ref", json!(git_ref)); } @@ -571,24 +583,48 @@ fn unique() -> String { format!("{}-{nanos}", std::process::id()) } -/// This Mac's bundle file, removed when dropped. -struct LocalBundle(PathBuf); +/// This Mac's bundle file, in a folder of its own only this user can open +/// (git writes the bundle with the umask's mode, often world-readable, and +/// it holds the project's unpushed work); the folder goes when dropped. +struct LocalBundle { + folder: PathBuf, + file: PathBuf, +} impl LocalBundle { - fn new() -> Self { - Self(std::env::temp_dir().join(format!("cmux-seed-push-{}.bundle", unique()))) + fn new() -> Result { + use std::os::unix::fs::DirBuilderExt; + let folder = std::env::temp_dir().join(format!("cmux-seed-push-{}", unique())); + // Fails on a folder already there, so nobody else's folder is used. + std::fs::DirBuilder::new().mode(0o700).create(&folder).map_err(|error| { + SeedError::new("sync_failed", format!("cannot make a folder for the bundle: {error}")) + })?; + let file = folder.join("seed.bundle"); + Ok(Self { folder, file }) } } impl Drop for LocalBundle { fn drop(&mut self) { - let _ = std::fs::remove_file(&self.0); + let _ = std::fs::remove_dir_all(&self.folder); } } #[cfg(test)] mod tests { - use super::{Command, parse}; + use super::{Command, LocalBundle, parse}; + + #[test] + fn the_bundle_lives_in_a_folder_only_this_user_opens_and_goes_with_it() { + use std::os::unix::fs::PermissionsExt; + let bundle = LocalBundle::new().unwrap(); + let folder = bundle.folder.clone(); + assert_eq!(std::fs::metadata(&folder).unwrap().permissions().mode() & 0o777, 0o700); + assert_eq!(bundle.file.parent(), Some(folder.as_path())); + std::fs::write(&bundle.file, b"x").unwrap(); + drop(bundle); + assert!(!folder.exists()); + } fn args(list: &[&str]) -> Vec { list.iter().map(|arg| (*arg).to_string()).collect()