Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
730d312
fix(executor): stop persisting runtime task status
qdaxb Aug 19, 2026
53662fd
fix(executor): preserve legacy archive metadata
qdaxb Aug 19, 2026
08952bc
docs(executor): clarify restart state ownership
qdaxb Aug 19, 2026
b7c2a1b
fix(executor): preserve restart reconciliation log
qdaxb Aug 19, 2026
95441b5
fix(executor): track interrupted worktree executions
qdaxb Aug 19, 2026
0d807a9
test(wework): keep claude cancellation stream active
qdaxb Aug 19, 2026
e2ab5ee
fix(executor): harden worktree execution leases
qdaxb Aug 19, 2026
ee1a7e6
test(wework): align restart checks with worktree state
qdaxb Aug 19, 2026
0815a13
fix(executor): clear leases after stop acknowledgement
qdaxb Aug 19, 2026
a769cc3
test(wework): synchronize queued follow-up input
qdaxb Aug 19, 2026
3356efe
fix(wework): remove timed conversation restore state
qdaxb Aug 20, 2026
5c3f89c
Merge remote-tracking branch 'origin/main' into fix/executor-restart-…
qdaxb Aug 20, 2026
ca36318
fix(wework): release restored scroll on send
qdaxb Aug 20, 2026
9c69748
fix(wework): persist newly created project selection
qdaxb Aug 20, 2026
8b15ec2
test(wework): wait for selected worktree ref
qdaxb Aug 20, 2026
3ddaca9
Merge remote-tracking branch 'origin/main' into fix/executor-restart-…
qdaxb Aug 20, 2026
c300d5c
fix(executor): preserve attachment-only messages
qdaxb Aug 20, 2026
7e1b67d
Merge remote-tracking branch 'origin/main' into fix/executor-restart-…
qdaxb Aug 20, 2026
21398d4
fix(wework): detect local harnesses concurrently
qdaxb Aug 20, 2026
75f5db7
fix(wework): stabilize harness and attachment previews
qdaxb Aug 20, 2026
b6f3509
Merge remote-tracking branch 'origin/main' into fix/executor-restart-…
qdaxb Aug 20, 2026
abbe35d
fix(executor): persist runtime project context
qdaxb Aug 20, 2026
cb076f6
fix(wework): use active backend for attachment previews
qdaxb Aug 20, 2026
6e41512
fix(wework): preserve blob response content type
qdaxb Aug 20, 2026
41c2625
fix(wework): preserve model provider selection
qdaxb Aug 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 41 additions & 4 deletions docs/en/architecture/git-worktree-execution.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ flowchart LR
EXECUTOR[target Executor<br/>only Git data plane]
REPO[(base Git workspace)]
WORKTREE[(managed Worktree)]
STATE[(Runtime Task Store<br/>worktrees.json<br/>snapshot refs)]
STATE[(Runtime Task Metadata Store<br/>no execution status)]
EXECUTION[(persisted turn queue<br/>queued execution intent)]
WORKTREE_STATE[(worktrees.json<br/>Worktree lifecycle + execution lease + snapshot refs)]
VOLUME[(stable Executor Home volume)]
ATTEST[deployment durability attestation<br/>persistentStorageVerified]

Expand All @@ -30,10 +32,14 @@ flowchart LR
EXECUTOR --> REPO
EXECUTOR --> WORKTREE
EXECUTOR --> STATE
EXECUTOR --> EXECUTION
EXECUTOR --> WORKTREE_STATE
ATTEST -->|explicit startup configuration| EXECUTOR
VOLUME --> REPO
VOLUME --> WORKTREE
VOLUME --> STATE
VOLUME --> EXECUTION
VOLUME --> WORKTREE_STATE
VOLUME -->|device-config.json| EXECUTOR
```

Expand Down Expand Up @@ -66,6 +72,37 @@ sequenceDiagram
E-->>U: running/failed events and final path
```

```mermaid
sequenceDiagram
participant E as restarted Executor
participant T as Runtime Task Metadata Store
participant Q as Persisted Turn Queue
participant G as worktrees.json
participant R as Agent Runtime

E->>T: load identity, thread, path, title, and archive metadata
Note over E,T: Task Store never persists status/running/threadStatus/turnStatus
E->>Q: restore queued execution intent
E->>G: reconcile Worktree lifecycle
alt Worktree was left in preparing
G->>G: validate the recorded Worktree identity
alt valid Worktree exists
G->>G: move preparing to active and preserve recovery diagnostics
else missing or invalid Worktree
G->>G: move preparing to failed and preserve the validation error
end
G-->>E: explicit interrupted-preparation evidence
E->>T: mark only the linked Runtime Task failed
else Worktree retains an uncleared execution lease
G->>G: clear the lease and preserve interrupted-execution diagnostics
G-->>E: explicit interrupted-execution evidence
E->>T: mark only the linked Runtime Task failed
else ordinary history or queue-only task
E->>E: infer no failure and preserve task activity time
end
E->>R: resume only supported non-Worktree queued executions
```

```mermaid
sequenceDiagram
participant U as Wework
Expand Down Expand Up @@ -95,7 +132,7 @@ sequenceDiagram
| DeviceWorkspace selection, availability, preferences, and UI | `wework/src/features/workbench/`, `wework/src/components/chat/composer/` |
| Local/Cloud/Remote Runtime routing and task projection | `wework/src/api/`, `wework/src/features/workbench/useWorkbenchRuntimeMessaging.ts`, `wework/src/features/workbench/workbenchReducer.ts` |
| Logical-device authorization, persistent Runtime identity, and socket resolution | `backend/app/services/device/`, `backend/app/api/ws/` |
| Worktree capability, preflight, Git lifecycle, and state | `executor/src/runtime_work/` |
| Worktree capability, preflight, Git lifecycle, task metadata, and execution evidence | `executor/src/runtime_work/` |
| Cloud persistent volume, stable mount path, and single writer | cloud device provider, deployment configuration, `docker/device/` |
| Cross-layer acceptance | `wework/e2e/desktop/`, `scripts/acceptance/`, Backend and Executor contract tests |

Expand All @@ -111,12 +148,12 @@ Essential invariants:
8. An existing target path is adopted only after proving that it is the expected Git Worktree for the same repository and Worktree ID.
9. Blocking Git and directory scans never run on the asynchronous RPC worker.
10. Deletion requires stop acknowledgement from every linked Runtime. Runtime exit or panic sends that acknowledgement through a scope-exit guard. Archive, snapshot, and removal happen in order, every failure preserves diagnosable data, and bulk archive uses bounded concurrency instead of accumulating stop timeouts linearly per stuck task.
11. Restart reconciliation may recover a manageable Worktree, but never auto-resumes an interrupted agent. Persistent failures are throttled with a fixed retry interval while preserving later retries.
11. Restart reconciliation may recover a manageable Worktree, but never auto-resumes an interrupted agent. The Runtime Task Store persists durable task metadata, but never persists `status`, `running`, `threadStatus`, or `turnStatus`. Persistent reconciliation failures are throttled with a fixed retry interval while preserving later retries.
12. One Executor process writes an Executor Home at a time. Cloud and Remote Executor Home, repositories, Worktrees, Runtime Store, and absolute mount path remain durably stable.
13. Launch-mode and branch preferences are scoped by DeviceWorkspace; execution location is not a launch mode.
14. Offline, missing route, timeout, disconnect, unsupported, non-Git, path-conflict, and non-durable-storage failures have stable distinct errors. Timeout and disconnect are marked retryable, but mutating RPCs are never automatically retried.
15. Terminal, IDE, file tree, Git, archive, restore, and settings always use the owning device and the task's final Worktree path. Remote file commands fail closed when Backend cannot authenticate an allowed workspace root; an empty root set never means unrestricted access.
16. Once restart reconciliation marks an interrupted task failed or cancelled, task-list projection never revives it as active from stale provider thread metadata. Only a provider turn newer than the local completion time can represent a new execution.
16. Whether a task was executing before restart is determined only by independent durable execution evidence, never by Runtime Task metadata, a Worktree path, non-archive state, provider history, or missing status fields. Before a Runtime actually starts, Executor writes an execution lease with the current process owner to `worktrees.json` and clears it after Runtime termination is acknowledged; startup reconciliation consumes only leases left by another process and must not classify a newly started execution in the current process as interrupted. An unfinished `preparing` record proves only interrupted Worktree preparation, while a persisted queue entry proves only that execution had not started. Without either kind of interruption evidence, restart reconciliation never writes task failure, completion time, or activity time.
17. Cloud and Remote `runtimeInstanceId` is stored in Executor Home and becomes immutable after Backend first records it. Container or instance replacement must reuse that value. Registration of the same logical device with a new Runtime Instance ID is rejected as a persistent-storage identity mismatch; Backend never overwrites the established value or creates a bypass device record.
18. Deferred creation persists the source-repository fingerprint captured while planning and compares it with a freshly computed fingerprint after acquiring a slot. Replacing the source directory with another repository fails as `worktree_source_changed`.
19. A stop timeout has an unknown outcome. It never clears Runtime cancellation control or treats the task as stopped; an archive/delete retry still requires a stop acknowledgement from the same Runtime.
Expand Down
45 changes: 41 additions & 4 deletions docs/zh/architecture/git-worktree-execution.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ flowchart LR
EXECUTOR[目标 Executor<br/>唯一 Git 数据面]
REPO[(主 Git 工作区)]
WORKTREE[(托管 Worktree)]
STATE[(Runtime Task Store<br/>worktrees.json<br/>snapshot refs)]
STATE[(Runtime Task Metadata Store<br/>无执行状态)]
EXECUTION[(持久化 turn queue<br/>排队执行意图)]
WORKTREE_STATE[(worktrees.json<br/>Worktree 生命周期 + execution lease + snapshot refs)]
VOLUME[(稳定 Executor Home 持久卷)]
ATTEST[部署持久化证明<br/>persistentStorageVerified]

Expand All @@ -30,10 +32,14 @@ flowchart LR
EXECUTOR --> REPO
EXECUTOR --> WORKTREE
EXECUTOR --> STATE
EXECUTOR --> EXECUTION
EXECUTOR --> WORKTREE_STATE
ATTEST -->|显式启动配置| EXECUTOR
VOLUME --> REPO
VOLUME --> WORKTREE
VOLUME --> STATE
VOLUME --> EXECUTION
VOLUME --> WORKTREE_STATE
VOLUME -->|device-config.json| EXECUTOR
```

Expand Down Expand Up @@ -66,6 +72,37 @@ sequenceDiagram
E-->>U: running/failed 事件和最终路径
```

```mermaid
sequenceDiagram
participant E as 重启后的 Executor
participant T as Runtime Task Metadata Store
participant Q as Persisted Turn Queue
participant G as worktrees.json
participant R as Agent Runtime

E->>T: 读取身份、线程、路径、标题和归档元数据
Note over E,T: Task Store 不持久化 status/running/threadStatus/turnStatus
E->>Q: 恢复排队执行意图
E->>G: 对账 Worktree 生命周期
alt Worktree 上次停在 preparing
G->>G: 校验记录中的 Worktree 身份
alt 合法 Worktree 已存在
G->>G: preparing 转为 active,并保留恢复诊断
else Worktree 缺失或身份无效
G->>G: preparing 转为 failed,并保留校验错误
end
G-->>E: 明确的中断准备证据
E->>T: 仅将关联的 Runtime Task 标记为失败
else Worktree 保留未清理的 execution lease
G->>G: 清除 lease,保留“执行被中断”诊断
G-->>E: 明确的中断执行证据
E->>T: 仅将关联的 Runtime Task 标记为失败
else 普通历史任务或仅排队任务
E->>E: 不推导失败、不改任务活动时间
end
E->>R: 仅恢复允许续跑的普通排队执行
```

```mermaid
sequenceDiagram
participant U as Wework
Expand Down Expand Up @@ -95,7 +132,7 @@ sequenceDiagram
| DeviceWorkspace 选择、可用性、偏好和 UI | `wework/src/features/workbench/`、`wework/src/components/chat/composer/` |
| Local/Cloud/Remote Runtime 路由和任务投影 | `wework/src/api/`、`wework/src/features/workbench/useWorkbenchRuntimeMessaging.ts`、`wework/src/features/workbench/workbenchReducer.ts` |
| 逻辑设备鉴权、持久 Runtime 身份和 Socket 解析 | `backend/app/services/device/`、`backend/app/api/ws/` |
| Worktree capability、preflight、Git 生命周期和状态 | `executor/src/runtime_work/` |
| Worktree capability、preflight、Git 生命周期、任务元数据和执行证据 | `executor/src/runtime_work/` |
| 云设备持久卷、固定挂载路径和单写实例 | 云设备 Provider、部署配置、`docker/device/` |
| 跨层验收 | `wework/e2e/desktop/`、`scripts/acceptance/`、Backend 和 Executor 契约测试 |

Expand All @@ -111,12 +148,12 @@ sequenceDiagram
8. 已存在目标路径只有在验证为同一源仓库、同一 Worktree ID 的合法 Git Worktree 后才能幂等接管。
9. 阻塞 Git 和目录扫描不得运行在异步 RPC 主线程。
10. 删除前必须获得所有关联 Runtime 的停止确认;Runtime 退出或 panic 都必须由作用域退出守卫发送停止确认。归档、快照和删除按顺序执行,任一步失败都保留可诊断数据;批量归档必须有界并发,不能按故障任务数线性累积停止超时。
11. Executor 重启对账可以恢复 Worktree 的可管理状态,但不得自动续跑中断的 Agent;持续失败的对账必须按固定间隔限流,同时保留后续重试能力。
11. Executor 重启对账可以恢复 Worktree 的可管理状态,但不得自动续跑中断的 Agent;Runtime Task Store 持久化耐久任务元数据,但不持久化 `status`、`running`、`threadStatus` 或 `turnStatus`。持续失败的对账必须按固定间隔限流,同时保留后续重试能力。
12. 一个 Executor Home 同一时刻只能有一个写入 Executor;Cloud 和 Remote 的 Executor Home、主仓库、Worktree、Runtime Store 和绝对挂载路径必须稳定持久化。
13. 启动模式和分支偏好按 DeviceWorkspace 隔离;运行位置不是启动模式。
14. 离线、路由缺失、超时、断开、不支持、非 Git、路径冲突和存储不持久使用稳定可区分错误;超时和断开必须标记为可重试,但有副作用的 RPC 不自动重试。
15. Terminal、IDE、文件树、Git、归档、恢复和设置始终使用任务所属设备及最终 Worktree 路径。远端文件命令缺少 Backend 认证出的工作区根时必须关闭访问,不能把空根集合解释为无限制。
16. 重启对账已将中断任务标记为失败或取消后,任务列表不得因过期的 Provider 线程元数据把它重新投影为运行中;只有本地完成时间之后出现的新 Provider Turn 才能表示一次新的执行。
16. 重启前是否正在执行只能由独立的持久执行证据判断,不能从 Runtime Task 元数据、Worktree 路径、非归档状态、Provider 历史线程或缺失状态字段推导。Runtime 真正启动前,Executor 必须在 `worktrees.json` 写入带当前进程 owner 的 execution lease,并在 Runtime 终止确认时清除;启动对账只消费其他进程遗留的 lease,不能把当前进程刚开始的执行误判为中断。未完成的 `preparing` 记录只能证明 Worktree 准备被中断,持久队列只能证明尚未运行。没有这两类中断证据时,重启对账不得写任务失败、完成时间或活动时间。
17. Cloud 和 Remote 的 `runtimeInstanceId` 保存在 Executor Home,并在 Backend 首次登记后固定;容器或实例替换必须复用该值。相同逻辑设备携带新的 Runtime Instance ID 时必须拒绝注册并报告持久存储身份不匹配,不能覆盖既有值或创建旁路设备记录。
18. 延迟创建必须持久化计划阶段的源仓库 fingerprint,并在获得 slot 后与创建前重新计算的 fingerprint 比较;源目录被替换为另一个仓库时必须以 `worktree_source_changed` 失败。
19. 停止超时是未知结果,不得清除 Runtime 取消控制或把任务当作已停止;重试归档/删除前仍必须获得同一 Runtime 的停止 ACK。
Expand Down
16 changes: 8 additions & 8 deletions executor/src/runtime_work/handler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,6 @@ pub struct RuntimeWorkRpcHandler {
turn_scheduler: Arc<Mutex<RuntimeTurnScheduler>>,
turn_queue_operation: Arc<AsyncMutex<()>>,
turn_queue_path: Arc<PathBuf>,
interrupted_worktree_turns: Arc<AsyncMutex<Option<VecDeque<SpawnTurnRequest>>>>,
preparing_worktree_turns: Arc<Mutex<HashMap<String, PreparingWorktreeTurn>>>,
active_turn_cancellations: Arc<Mutex<HashMap<String, ActiveTurnCancellation>>>,
active_codex_turns: Arc<Mutex<HashMap<String, ActiveCodexTurn>>>,
Expand Down Expand Up @@ -483,6 +482,8 @@ struct WorktreeReconciliationState {
struct ActiveTurnCancellation {
execution_id: u64,
stop_requested: bool,
stop_acknowledged: bool,
managed_worktree_path: Option<PathBuf>,
cancel: oneshot::Sender<()>,
stopped: oneshot::Receiver<()>,
}
Expand Down Expand Up @@ -590,17 +591,17 @@ impl RuntimeWorkRpcHandler {
let store = RuntimeWorkStore::from_env();
let worktrees = WorktreeManager::from_env(&device_id);
let turn_queue_path = turns::runtime_turn_queue_path();
let restored_turns =
let mut queued_turns =
turns::read_runtime_turn_queue(&turn_queue_path).unwrap_or_else(|error| {
log_executor_event("runtime turn queue restore failed", &[("error", error)]);
VecDeque::new()
});
let (queued_turns, interrupted_worktree_turns) =
turns::partition_restored_turns(&worktrees, restored_turns);
if !interrupted_worktree_turns.is_empty() {
let removed_worktree_turn_count =
turns::remove_worktree_turns_after_restart(&worktrees, &mut queued_turns);
if removed_worktree_turn_count > 0 {
log_executor_event(
"persisted worktree turns quarantined after executor restart",
&[("count", interrupted_worktree_turns.len().to_string())],
"persisted worktree turns removed after executor restart",
&[("count", removed_worktree_turn_count.to_string())],
);
}
let handler = Self {
Expand All @@ -620,7 +621,6 @@ impl RuntimeWorkRpcHandler {
))),
turn_queue_operation: Arc::new(AsyncMutex::new(())),
turn_queue_path: Arc::new(turn_queue_path),
interrupted_worktree_turns: Arc::new(AsyncMutex::new(Some(interrupted_worktree_turns))),
preparing_worktree_turns: Arc::new(Mutex::new(HashMap::new())),
active_turn_cancellations: Arc::new(Mutex::new(HashMap::new())),
active_codex_turns: Arc::new(Mutex::new(HashMap::new())),
Expand Down
22 changes: 18 additions & 4 deletions executor/src/runtime_work/handler/claude_turns.rs
Original file line number Diff line number Diff line change
Expand Up @@ -205,8 +205,21 @@ impl RuntimeWorkRpcHandler {
pub(super) fn start_claude_turn(&self, local_task_id: String, request: ExecutionRequest) {
let (cancel_tx, cancel_rx) = oneshot::channel();
let (stopped_tx, stopped_rx) = oneshot::channel();
let execution_id =
self.start_local_task_execution(local_task_id.clone(), cancel_tx, stopped_rx);
let execution_id = match self.start_local_task_execution(
local_task_id.clone(),
request
.project_workspace_path
.as_deref()
.or_else(|| request.cwd()),
cancel_tx,
stopped_rx,
) {
Ok(execution_id) => execution_id,
Err(error) => {
self.fail_local_task_execution_start(&local_task_id, &error);
return;
}
};
let handler = self.clone();
tokio::spawn(async move {
let _stopped_turn_guard = StoppedTurnGuard::new(stopped_tx);
Expand Down Expand Up @@ -588,8 +601,9 @@ mod tests {
handler.store = RuntimeWorkStore::new(directory.path().join("index.json"));
let (cancel_tx, _cancel_rx) = oneshot::channel();
let (_stopped_tx, stopped_rx) = oneshot::channel();
let execution_id =
handler.start_local_task_execution("task-1".to_owned(), cancel_tx, stopped_rx);
let execution_id = handler
.start_local_task_execution("task-1".to_owned(), None, cancel_tx, stopped_rx)
.expect("local execution should start");
let transcript = Arc::new(Mutex::new(ClaudeTurnTranscript::default()));
let sink = ClaudeRuntimeEventSink {
handler: handler.clone(),
Expand Down
Loading
Loading