From 6c4c44b84cf05bbea172d37f163e41248dcc7f92 Mon Sep 17 00:00:00 2001 From: Hulk Date: Tue, 6 Oct 2026 23:22:44 -0700 Subject: [PATCH 1/5] fix(lifecycle): propose shorter restart drain grace Separate the proposed two-second active/scoped-drain grace from the existing sixty-second cleanup watchdog and seventy-second replacement-readiness budget. Keep API and dashboard timing aligned, cover lifecycle timing and ownership with synthetic clocks, and document interrupted-execution uncertainty. Proposal for owner review of the shared lifecycle default; does not authorize automatic replay or guarantee a two-second recovery. Refs lidge-jun/opencodex#6643 --- .../docs/fr/troubleshooting/windows-memory.md | 2 +- .../docs/ja/troubleshooting/windows-memory.md | 2 +- .../docs/ko/troubleshooting/windows-memory.md | 2 +- .../content/docs/reference/management-api.md | 9 + .../docs/ru/troubleshooting/windows-memory.md | 4 +- .../docs/tr/troubleshooting/windows-memory.md | 5 +- .../docs/troubleshooting/windows-memory.md | 9 +- .../zh-cn/troubleshooting/windows-memory.md | 2 +- .../components/MemoryObservabilityCard.tsx | 7 +- gui/src/remote-link-api.ts | 2 +- gui/tests/memory-observability-card.test.tsx | 3 +- scripts/test-layout/layout.json | 1 + src/lib/system-restart-contract.ts | 4 +- src/lib/system-restart-timing.ts | 6 + src/server/lifecycle.ts | 2 +- src/server/management/system-restart.ts | 20 +- src/server/management/system-routes.ts | 2 +- src/tray/windows-tray.ps1 | 2 +- structure/ops/service-and-sidecars.md | 12 + structure/remote-link.md | 2 +- structure/runtime.md | 2 +- tests/fixtures/test-layout-expected.json | 2 +- tests/server/system-restart-admission.test.ts | 328 ++++++++++++++++++ tests/server/system-restart.test.ts | 9 +- 24 files changed, 401 insertions(+), 38 deletions(-) create mode 100644 src/lib/system-restart-timing.ts create mode 100644 tests/server/system-restart-admission.test.ts diff --git a/docs-site/src/content/docs/fr/troubleshooting/windows-memory.md b/docs-site/src/content/docs/fr/troubleshooting/windows-memory.md index 38fd2b99294..f72167171b7 100644 --- a/docs-site/src/content/docs/fr/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/fr/troubleshooting/windows-memory.md @@ -23,7 +23,7 @@ Il s’agit d’une atténuation bornée et d’outils d’observation — **pas - **Surveillance de la mémoire** — le proxy échantillonne sa propre mémoire toutes les minutes et consigne un avertissement à fréquence limitée lorsque la mémoire observée dépasse 4 GiB. La mémoire observée correspond à la plus grande des valeurs RSS, `external` et `arrayBuffers` (et non à leur somme), car les compteurs d’ensemble de travail/RSS de Windows peuvent sous-estimer la rétention externe validée. - **`ocx doctor`** — une section « Memory / runtime » affiche la version de Bun du processus de *service*, les compteurs RSS, external/ArrayBuffers, le contexte du tas JS et la décision relative au mode de flux. Avec le runtime Bun 1.3.14 intégré, `heapUsed` / `jscHeap` ne permettent pas à eux seuls de caractériser une fuite ; comparez la mémoire observée à `responseState` et à plusieurs échantillons avant de conclure à une fuite au niveau de l’application. -- **`GET /api/system/memory`** — expose les mêmes données via l’API de gestion authentifiée, à destination des tableaux de bord ou des scripts. Outre les compteurs RSS/heap/external, la réponse contient un bloc scalaire `responseState` (nombre d’entrées, nombre total/maximal d’octets sérialisés et âge de l’entrée la plus ancienne) pour le magasin de continuation en mémoire `previous_response_id` du proxy. Ces données permettent de mieux attribuer la croissance : si `responseState.totalBytes` et la mémoire observée augmentent ensemble, la rétention des conversations est probablement en cause (les longues chaînes `store:false` sont redéveloppées à chaque tour) ; si `responseState` reste stable tandis que la mémoire observée augmente, il faut chercher ailleurs. Les valeurs sont exclusivement scalaires : aucun corps de requête, jeton, chemin ou identifiant de compte n’est exposé. La lecture est dépourvue d’effet secondaire : elle n’élague ni n’expulse jamais de données. La carte **Observabilité de la mémoire** du tableau de bord affiche les mêmes champs et propose une action **Drainer et redémarrer** soumise à confirmation. Elle indique le nombre actuel de tours actifs, attend leur fin jusqu’à 60 s (en réutilisant le drainage existant avec 503 + `Retry-After`), puis interrompt les tours restants. Le proxy actif contrôle l’autorisation du redémarrage et la coordination du drainage, puis se termine ; lorsqu’un gestionnaire de services est installé, il lance le processus de remplacement. L’action ne signale une réussite qu’après avoir vérifié qu’un autre processus, dont l’identité est confirmée, est sain sur le même port, sans retirer l’injection Codex. Ce recyclage est plus long et mieux contrôlé que le court drainage déclenché par `POST /api/stop`. +- **`GET /api/system/memory`** — expose les mêmes données via l’API de gestion authentifiée, à destination des tableaux de bord ou des scripts. Outre les compteurs RSS/heap/external, la réponse contient un bloc scalaire `responseState` (nombre d’entrées, nombre total/maximal d’octets sérialisés et âge de l’entrée la plus ancienne) pour le magasin de continuation en mémoire `previous_response_id` du proxy. Ces données permettent de mieux attribuer la croissance : si `responseState.totalBytes` et la mémoire observée augmentent ensemble, la rétention des conversations est probablement en cause (les longues chaînes `store:false` sont redéveloppées à chaque tour) ; si `responseState` reste stable tandis que la mémoire observée augmente, il faut chercher ailleurs. Les valeurs sont exclusivement scalaires : aucun corps de requête, jeton, chemin ou identifiant de compte n’est exposé. La lecture est dépourvue d’effet secondaire : elle n’élague ni n’expulse jamais de données. La carte **Observabilité de la mémoire** du tableau de bord affiche les mêmes champs et propose une action **Drainer et redémarrer** soumise à confirmation. Elle indique le nombre actuel de tours actifs, attend leur fin jusqu’à 2 s après acceptation (en réutilisant le drainage existant avec 503 + `Retry-After`), puis interrompt les tours restants. Le proxy actif contrôle l’autorisation du redémarrage et la coordination du drainage, puis se termine ; lorsqu’un gestionnaire de services est installé, il lance le processus de remplacement. L’action ne signale une réussite qu’après avoir vérifié qu’un autre processus, dont l’identité est confirmée, est sain sur le même port, sans retirer l’injection Codex. Le nettoyage et le démarrage ont des délais distincts ; une requête interrompue peut déjà avoir été exécutée et n’est pas rejouée automatiquement. Voir [délais et sécurité des nouvelles tentatives](/reference/management-api/#system-lifecycle) (anglais). - **Autre chemin de flux soumis à activation** — un relais borné à lecteur unique qui élimine entièrement le modèle de mise en mémoire tampon non bornée. Sous Windows, il deviendra automatiquement le chemin par défaut dès qu’une version intégrée de Bun inclura de manière vérifiable le correctif #32111 ; pour l’instant, il faut l’activer explicitement (voir ci-dessous). Sous macOS, il restera soumis à une activation explicite même après une telle version : le passage de `auto` sous macOS fera l’objet d’une décision distincte. L’amélioration réelle du RSS apportée par ces changements reste **à vérifier par des utilisateurs Windows** — nous n’affirmons pas que la fuite est corrigée. diff --git a/docs-site/src/content/docs/ja/troubleshooting/windows-memory.md b/docs-site/src/content/docs/ja/troubleshooting/windows-memory.md index 6d25a176336..044aa6d35f2 100644 --- a/docs-site/src/content/docs/ja/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/ja/troubleshooting/windows-memory.md @@ -26,7 +26,7 @@ Windows では、#32111 クラッシュを回避するために、opencodex は - **`ocx doctor`** — 「メモリ / ランタイム」セクションには *サービス* が表示されます プロセスの Bun バージョン、RSS、外部/ArrayBuffers カウンター、JS ヒープ コンテキスト、およびストリーム モードの決定。バンドルされている Bun 1.3.14 ランタイムでは、`heapUsed` / `jscHeap` 単独ではリーク識別子ではありません。アプリレベルのリークを割り当てる前に、観察されたメモリを `responseState` および繰り返しサンプルと比較します。 - **`GET /api/system/memory`** — 認証済みの同じデータ -ダッシュボードまたはスクリプトの管理 API。 RSS/ヒープ/外部カウンターとともに、プロキシのメモリ内 `previous_response_id` 継続ストアのスカラー `responseState` ブロック (エントリ数、シリアル化された合計/最大バイト数、最も古いエントリの経過時間) を報告します。これはさらに成長に起因します。観察された記憶の上昇下での `responseState.totalBytes` の上昇は会話の保持を指します (長い `store:false` チェーンはターンごとに再拡張します)。一方、観察された記憶の上昇の下での横ばいの `responseState` はそのストアから遠ざかることを示します。値はスカラーのみであり、リクエスト本文、トークン、パス、アカウント識別子はありません。また、読み取りには副作用はありません (プルーニングや削除は行われません)。ダッシュボードの **メモリ可観測性** カードは同じフィールドをレンダリングし、確認ゲート付き **ドレインと再起動** アクションを提供します。現在のアクティブ ターン数を表示し、アクティブ ターンを最大 60 秒待機し (既存の 503 + `Retry-After` ドレインを再利用)、残りのターンを中止します。実行中のプロキシは再起動の認可とドレイン調整を担当して終了し、サービス管理下ではインストール済みサービス マネージャーが置換プロセスを起動します。同じポートで、別の ID 検証済みプロセスが正常であることを確認した場合にのみ成功し、Codex インジェクションは破棄しません。これは、`POST /api/stop` の短いドレインよりも長く、情報に基づいたリサイクルです。 +ダッシュボードまたはスクリプトの管理 API。 RSS/ヒープ/外部カウンターとともに、プロキシのメモリ内 `previous_response_id` 継続ストアのスカラー `responseState` ブロック (エントリ数、シリアル化された合計/最大バイト数、最も古いエントリの経過時間) を報告します。これはさらに成長に起因します。観察された記憶の上昇下での `responseState.totalBytes` の上昇は会話の保持を指します (長い `store:false` チェーンはターンごとに再拡張します)。一方、観察された記憶の上昇の下での横ばいの `responseState` はそのストアから遠ざかることを示します。値はスカラーのみであり、リクエスト本文、トークン、パス、アカウント識別子はありません。また、読み取りには副作用はありません (プルーニングや削除は行われません)。ダッシュボードの **メモリ可観測性** カードは同じフィールドをレンダリングし、確認ゲート付き **ドレインと再起動** アクションを提供します。現在のアクティブ ターン数を表示し、受け付けから最大 2 秒間アクティブ ターンを待機し (既存の 503 + `Retry-After` ドレインを再利用)、残りのターンを中止します。実行中のプロキシは再起動の認可とドレイン調整を担当して終了し、サービス管理下ではインストール済みサービス マネージャーが置換プロセスを起動します。同じポートで、別の ID 検証済みプロセスが正常であることを確認した場合にのみ成功し、Codex インジェクションは破棄しません。クリーンアップと起動には別の待機時間があります。中断された要求は上流ですでに実行された可能性があり、自動再送はしません。[再起動時間と再試行の安全性](/reference/management-api/#system-lifecycle)（英語）を参照してください。 - **ゲートされた代替ストリーム パス** — 制限された単一リーダー リレー。 境界のないバッファリング形状を完全に削除します。 Windows では、バンドルされた Bun リリースに #32111 修正が確実に適用されると、これが自動的にデフォルトになります。現在はオプトインのみとなっています (以下を参照)。 macOS では、そのようなリリース後でもオプトインのままになります。macOS `auto` を切り替えるかどうかは別の決定となります。 diff --git a/docs-site/src/content/docs/ko/troubleshooting/windows-memory.md b/docs-site/src/content/docs/ko/troubleshooting/windows-memory.md index 3feb6b682a3..a76dac87cdf 100644 --- a/docs-site/src/content/docs/ko/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/ko/troubleshooting/windows-memory.md @@ -23,7 +23,7 @@ Windows에서는 opencodex가 #32111 충돌을 피하기 위해 스트리밍 응 - **메모리 감시기** - 프록시는 1분마다 자체 메모리를 샘플링하고, 관측된 메모리가 4 GiB를 넘으면 속도 제한이 걸린 경고를 기록합니다. 관측된 메모리는 RSS, `external`, `arrayBuffers`의 합이 아니라 그중 가장 큰 값입니다. Windows의 working-set/RSS 카운터가 커밋된 external 잔존량을 낮게 잡을 수 있기 때문입니다. - **`ocx doctor`** - "Memory / runtime" 섹션에서 *서비스* 프로세스의 Bun 버전, RSS, external/ArrayBuffers 카운터, JS 힙 문맥, 스트림 모드 결정을 보여줍니다. 번들된 Bun 1.3.14 런타임에서는 `heapUsed` / `jscHeap`만으로 누수를 판별할 수 없습니다. 애플리케이션 수준 누수로 단정하기 전에 관측된 메모리, `responseState`, 반복 샘플을 함께 보아야 합니다. -- **`GET /api/system/memory`** - 대시보드나 스크립트에서 쓸 수 있도록 같은 데이터를 인증된 관리 API로 제공합니다. RSS/heap/external 카운터와 함께, 프록시의 메모리 내 `previous_response_id` 이어받기 저장소에 대한 스칼라 `responseState` 블록(항목 수, 직렬화된 총/최대 바이트, 가장 오래된 항목의 경과 시간)을 보고합니다. 이를 통해 증가 원인을 더 잘 구분할 수 있습니다. 관측된 메모리가 함께 증가하면서 `responseState.totalBytes`도 늘면 대화 보존(long `store:false` 체인이 매 턴 다시 확장되는 경우)을 가리키고, 관측된 메모리는 늘지만 `responseState`는 평평하면 그 저장소와는 무관한 원인을 가리킵니다. 값은 스칼라만 포함하며 요청 본문, 토큰, 경로, 계정 식별자는 포함하지 않습니다. 또한 읽기 동작은 부작용이 없습니다. 절대 prune하거나 evict하지 않습니다. 대시보드의 **Memory observability** 카드는 같은 필드를 렌더링하고, 확인을 거쳐야 하는 **Drain & restart** 동작도 제공합니다. 현재 활성 턴 수를 보여주고, 기존 503 + `Retry-After` 드레인과 같은 방식으로 최대 60초 동안 활성 턴을 기다린 뒤, 남아 있는 턴을 강제로 중단하고 Codex 주입을 해제하지 않은 채 확인된 실행 프로세스가 스스로 교체되게 한 다음 같은 포트의 다른 PID를 검증합니다. 이는 `POST /api/stop`의 짧은 드레인보다 더 길고, 더 많은 정보를 반영한 재순환입니다. +- **`GET /api/system/memory`** - 대시보드나 스크립트에서 쓸 수 있도록 같은 데이터를 인증된 관리 API로 제공합니다. RSS/heap/external 카운터와 함께, 프록시의 메모리 내 `previous_response_id` 이어받기 저장소에 대한 스칼라 `responseState` 블록(항목 수, 직렬화된 총/최대 바이트, 가장 오래된 항목의 경과 시간)을 보고합니다. 이를 통해 증가 원인을 더 잘 구분할 수 있습니다. 관측된 메모리가 함께 증가하면서 `responseState.totalBytes`도 늘면 대화 보존(long `store:false` 체인이 매 턴 다시 확장되는 경우)을 가리키고, 관측된 메모리는 늘지만 `responseState`는 평평하면 그 저장소와는 무관한 원인을 가리킵니다. 값은 스칼라만 포함하며 요청 본문, 토큰, 경로, 계정 식별자는 포함하지 않습니다. 또한 읽기 동작은 부작용이 없습니다. 절대 prune하거나 evict하지 않습니다. 대시보드의 **Memory observability** 카드는 같은 필드를 렌더링하고, 확인을 거쳐야 하는 **Drain & restart** 동작도 제공합니다. 현재 활성 턴 수를 보여주고, 기존 503 + `Retry-After` 드레인과 같은 방식으로 수락 후 최대 2초 동안 활성 턴을 기다린 뒤, 남아 있는 턴을 강제로 중단하고 Codex 주입을 해제하지 않은 채 확인된 실행 프로세스가 스스로 교체되게 한 다음 같은 포트의 다른 PID를 검증합니다. 정리와 시작에는 별도의 제한 시간이 적용됩니다. 중단된 요청은 업스트림에서 이미 실행되었을 수 있으며 자동으로 재전송하지 않습니다. [재시작 시간과 재시도 안전성](/reference/management-api/#system-lifecycle)(영문)을 참고하세요. - **가드된 대체 스트림 경로** - unbounded buffering 형태를 완전히 제거하는 bounded single-reader relay입니다. Windows에서는 번들된 Bun 릴리스가 #32111 수정을 실제로 포함하고 있음이 확인되면 자동으로 기본값이 됩니다. 지금은 아래에서 설명하는 opt-in만 가능합니다. macOS에서는 그런 릴리스 이후에도 계속 opt-in입니다. macOS의 `auto`를 바꾸는 것은 별도의 결정입니다. 이 변경들로 실제 RSS가 얼마나 좋아지는지는 **Windows 사용자의 검증을 기다리고 있습니다**. 아직 이 누수가 해결되었다고 말하지는 않습니다. diff --git a/docs-site/src/content/docs/reference/management-api.md b/docs-site/src/content/docs/reference/management-api.md index 3bd3700766c..43526726624 100644 --- a/docs-site/src/content/docs/reference/management-api.md +++ b/docs-site/src/content/docs/reference/management-api.md @@ -668,6 +668,15 @@ During a restart drain, new data-plane requests receive HTTP 503 with JSON and the receiving listener's CORS policy. This code lets every Codex version retry the 503 without reporting model capacity; provider overload errors retain their separate mapping. +Restart gives active requests and pre-existing scoped drains up to two seconds from acceptance, +including the response-flush delay. The returned `drainTimeoutMs` describes this grace, not total +restart time. Cleanup keeps its separate 60-second watchdog, and replacement readiness can take up +to another 70 seconds. A request that finishes during grace is preserved; a remaining request is +interrupted and may already have executed upstream. OpenCodex does not automatically replay it. +Check its outcome before resubmitting: a healthy replacement does not prove a retry is safe. +The shorter grace reduces time spent refusing new work, but does not guarantee that every client +can reconnect within its retry window. Let important active work finish before requesting restart. + | Method and path | Purpose | Notable errors | | --- | --- | --- | | `GET /api/system/memory` | Return scalar process, heap, stream, response-state, watchdog, and active-turn metrics. Response-state diagnostics include spill-write status, consecutive failures, fixed privacy-safe failure class, and last failure/success timestamps. `spillLastWriteFailureOrigin` is `retry_returned_timeout`, `timeout_memo_refusal`, or null; cumulative `spillAclRetryReturnedTimeouts` and `spillAclTimeoutMemoRefusals` count terminal failed publications. See [Windows spill diagnostics](/troubleshooting/windows-memory/) for process-local semantics. Raw errors and paths are never returned. | — | diff --git a/docs-site/src/content/docs/ru/troubleshooting/windows-memory.md b/docs-site/src/content/docs/ru/troubleshooting/windows-memory.md index 7dd47d8da15..2189c1c5491 100644 --- a/docs-site/src/content/docs/ru/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/ru/troubleshooting/windows-memory.md @@ -48,10 +48,10 @@ opencodex поставляет рантайм Bun (сейчас это **1.3.14* токенов, путей и идентификаторов аккаунтов — и чтение не имеет побочных эффектов: ничего не prune'ится и не evict'ится. Карточка **Memory observability** в дашборде показывает те же поля и даёт confirm-gated действие **Drain & restart**: она показывает текущее число активных ходов, - ждёт до 60 секунд, пока они закончатся (используя уже существующий drain через 503 + + ждёт до 2 секунд с момента принятия перезапуска, пока они закончатся (используя уже существующий drain через 503 + `Retry-After`), затем прерывает оставшиеся ходы, просит аттестованный живой процесс заменить себя и проверяет другой PID на том же порту, не снимая внедрение - в Codex. Это более длинный и осознанный recycle, чем короткий drain у `POST /api/stop`. + в Codex. Очистка и запуск имеют отдельные сроки. Прерванный запрос уже мог выполниться на стороне провайдера; автоматической повторной отправки нет. См. [сроки перезапуска и безопасность повторов](/reference/management-api/#system-lifecycle) (англ.). - **Альтернативный stream path под флагом** — bounded single-reader relay, полностью убирающий форму неограниченной буферизации. На Windows он станет значением по умолчанию автоматически, когда в комплектный Bun *подтверждённо* войдёт исправление #32111; сегодня это только opt-in diff --git a/docs-site/src/content/docs/tr/troubleshooting/windows-memory.md b/docs-site/src/content/docs/tr/troubleshooting/windows-memory.md index dee7fcdc31d..d8094384c91 100644 --- a/docs-site/src/content/docs/tr/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/tr/troubleshooting/windows-memory.md @@ -56,14 +56,13 @@ Sınırlı azaltma ve görünürlük — **bir düzeltme değil**. Paketlenmiş tanımlayıcıları yoktur — ve okuma yan etkisizdir (asla budamaz veya çıkarmaz). Kontrol panelinin **Bellek gözlemlenebilirliği** kartı aynı alanları işler ve onay korumalı bir **Boşalt ve yeniden başlat (Drain & restart)** eylemi sunar: - geçerli aktif tur sayısını gösterir, aktif turlar için 60 saniyeye kadar + geçerli aktif tur sayısını gösterir, kabulden itibaren aktif turlar için 2 saniyeye kadar bekler (mevcut 503 + `Retry-After` boşaltmasını yeniden kullanarak), ardından kalan turları iptal eder. Çalışan proxy yeniden başlatma yetkilendirmesine ve boşaltma koordinasyonuna sahiptir, ardından çıkar; kurulu bir servis yöneticisi geçerli olduğunda yenisini başlatır. Eylem, yalnızca aynı portta kimliği doğrulanmış farklı bir süreç sağlıklı olduğunda, Codex enjeksiyonunu - kaldırmadan başarı bildirir. Bu, `POST /api/stop` üzerindeki kısa boşaltmadan - daha uzun, bilgilendirilmiş bir geri dönüşümdür. + kaldırmadan başarı bildirir. Temizleme ve başlatma için ayrı süreler vardır. Kesilen istek sağlayıcıda zaten çalışmış olabilir; otomatik olarak yeniden gönderilmez. [Yeniden başlatma süresi ve tekrar güvenliği](/reference/management-api/#system-lifecycle) (İngilizce). - **Geçitli bir alternatif akış yolu** — sınırsız arabellek şeklini tamamen ortadan kaldıran sınırlı bir tek okuyucu aktarımı. Windows'ta, paketlenmiş bir Bun sürümü #32111 düzeltmesini doğrulanabilir şekilde taşıdığında otomatik diff --git a/docs-site/src/content/docs/troubleshooting/windows-memory.md b/docs-site/src/content/docs/troubleshooting/windows-memory.md index e65b9c74cfe..f5cd087591e 100644 --- a/docs-site/src/content/docs/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/troubleshooting/windows-memory.md @@ -77,14 +77,15 @@ runtime the leak itself remains an upstream problem: signal. The dashboard's **Memory observability** card renders the memory and continuation-size fields from this endpoint and offers a confirm-gated **Drain & restart** action: it shows - the current active-turn count, waits up to 60s for active turns (reusing + the current active-turn count, allows up to 2s from acceptance for active turns (reusing the existing HTTP 503 JSON `server_restarting` + `Retry-After: 5` drain), - then aborts any remaining turns. + then aborts any remaining turns. Cleanup and replacement startup have separate, longer + budgets. Interrupted work may already have executed upstream and is never automatically + replayed; see [restart timing and retry safety](/reference/management-api/#system-lifecycle). The running proxy owns restart authorization and drain coordination, then exits; an installed service manager launches the replacement when applicable. The action reports success only after a different, identity-verified process - is healthy on the same port, without tearing down Codex injection. That is a - longer, informed recycle than the short drain on `POST /api/stop`. + is healthy on the same port, without tearing down Codex injection. For a scriptable snapshot of the complete authenticated payload, run `ocx observe memory --json`; the CLI forwards the same response-state fields. diff --git a/docs-site/src/content/docs/zh-cn/troubleshooting/windows-memory.md b/docs-site/src/content/docs/zh-cn/troubleshooting/windows-memory.md index 7734ebe23da..07fa9f5fd53 100644 --- a/docs-site/src/content/docs/zh-cn/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/zh-cn/troubleshooting/windows-memory.md @@ -23,7 +23,7 @@ opencodex 打包了 Bun 运行时（当前为 **1.3.14**）。这类内存增长 - **内存监视器** — 代理每分钟采样一次自身内存，并在观测到的内存超过 4 GiB 时记录限频告警。观测到的内存取 RSS、`external` 和 `arrayBuffers` 三者中的最大值（不是它们的总和），因为 Windows 的工作集/RSS 计数可能低报已提交的外部保留。 - **`ocx doctor`** — `"Memory / runtime"` 部分会显示*服务*进程的 Bun 版本、RSS、external/ArrayBuffers 计数、JS 堆上下文以及流模式决策。在捆绑的 Bun 1.3.14 运行时上，单看 `heapUsed` / `jscHeap` 不能作为泄漏判据；在认定为应用层泄漏之前，应把观测到的内存与 `responseState` 以及多次采样一起比较。 -- **`GET /api/system/memory`** — 通过已认证的管理 API 提供同样的数据，便于仪表板或脚本使用。除了 RSS/heap/external 计数之外，它还会报告一个标量的 `responseState` 块（条目数、序列化总字节数/最大字节数、最老条目的年龄），对应代理内存中的 `previous_response_id` 续接存储。这能进一步归因增长：在观测到的内存上升时，如果 `responseState.totalBytes` 也在上升，说明是对话保留在增长（较长的 `store:false` 链在每轮中重新扩张）；而在观测到的内存上升时，如果 `responseState` 保持平稳，则更像不是这个存储造成的。返回值只包含标量，不包含请求正文、token、路径或账户标识，而且读取没有副作用（不会执行 prune，也不会 evict）。仪表板中的 **Memory observability** 卡片会渲染相同字段，并提供一个需要确认的 **Drain & restart** 操作：它会显示当前活动轮次数量，最多等待 60 秒让活动轮次结束（复用现有的 503 + `Retry-After` 排空机制），然后中止剩余轮次。运行中的代理负责重启授权和排空协调并退出；如果由服务管理器托管，则由已安装的服务管理器启动替换进程。只有确认同一端口上另一个经过身份验证且健康的进程后才会报告成功，同时不会拆除 Codex 注入。这是一种比 `POST /api/stop` 的短排空更长、更知情的回收方式。 +- **`GET /api/system/memory`** — 通过已认证的管理 API 提供同样的数据，便于仪表板或脚本使用。除了 RSS/heap/external 计数之外，它还会报告一个标量的 `responseState` 块（条目数、序列化总字节数/最大字节数、最老条目的年龄），对应代理内存中的 `previous_response_id` 续接存储。这能进一步归因增长：在观测到的内存上升时，如果 `responseState.totalBytes` 也在上升，说明是对话保留在增长（较长的 `store:false` 链在每轮中重新扩张）；而在观测到的内存上升时，如果 `responseState` 保持平稳，则更像不是这个存储造成的。返回值只包含标量，不包含请求正文、token、路径或账户标识，而且读取没有副作用（不会执行 prune，也不会 evict）。仪表板中的 **Memory observability** 卡片会渲染相同字段，并提供一个需要确认的 **Drain & restart** 操作：它会显示当前活动轮次数量，从接受重启起最多等待 2 秒让活动轮次结束（复用现有的 503 + `Retry-After` 排空机制），然后中止剩余轮次。运行中的代理负责重启授权和排空协调并退出；如果由服务管理器托管，则由已安装的服务管理器启动替换进程。只有确认同一端口上另一个经过身份验证且健康的进程后才会报告成功，同时不会拆除 Codex 注入。清理和替换进程启动使用各自的时限。中断的请求可能已在上游执行，不会自动重放；请参阅[重启时限与重试安全](/reference/management-api/#system-lifecycle)（英文）。 - **有门控的替代流路径** — 一种有界的单读者中继，能彻底消除无界缓冲的形态。在 Windows 上，一旦某个捆绑的 Bun 版本可验证地包含 #32111 的修复，它就会自动成为默认路径；目前它仅支持显式启用（见下文）。在 macOS 上，即使到了那样的版本，它也仍然保持显式启用状态；切换 macOS 的 `auto` 是另一项独立决定。 这些改动带来的真实世界 RSS 改善，仍在等待 Windows 用户验证，我们并不宣称泄漏已经修复。 diff --git a/gui/src/components/MemoryObservabilityCard.tsx b/gui/src/components/MemoryObservabilityCard.tsx index 8bea5857ca1..ca079e7d1cd 100644 --- a/gui/src/components/MemoryObservabilityCard.tsx +++ b/gui/src/components/MemoryObservabilityCard.tsx @@ -1,4 +1,5 @@ import { useEffect, useState } from "react"; +import { RESTART_DRAIN_GRACE_MS } from "../../../src/lib/system-restart-timing"; import { formatUptime } from "../formatUptime"; import { IconActivity } from "../icons"; import { useI18n, type Locale, type TFn } from "../i18n/shared"; @@ -9,8 +10,8 @@ import { startVisibilityPoll } from "../visibility-poll"; /** * Memory observability card. Polls GET /api/system/memory (#314 WP3) every 5s * and renders scalar diagnostics. Also hosts the confirm-gated Drain & restart - * action (#563): longer 60s drain, then respawn via ensure/service — not the - * short /api/stop teardown path. + * action (#563): short in-flight grace, then restart with a separate cleanup + * budget, preserving client injection rather than the /api/stop teardown. */ interface MemorySample { @@ -198,7 +199,7 @@ function MemoryPressure({ ); } -const DRAIN_TIMEOUT_S = 60; +const DRAIN_TIMEOUT_S = RESTART_DRAIN_GRACE_MS / 1_000; const RECONNECT_POLL_MS = 1500; const RECONNECT_GIVE_UP_MS = 120_000; diff --git a/gui/src/remote-link-api.ts b/gui/src/remote-link-api.ts index 9dec2e1cea6..13f96f9a15b 100644 --- a/gui/src/remote-link-api.ts +++ b/gui/src/remote-link-api.ts @@ -170,7 +170,7 @@ export const CHILD_RESTART_POLL_MS = 1_000; export const CHILD_RESTART_SLOW_POLL_MS = 5_000; /** * The server's handoff budget plus a margin, the window `ocx restart` also observes: up to 60 s of - * drain (`MEMORY_DRAIN_RESTART_MS`), then up to 70 s for the replacement to answer + * terminal cleanup (`MEMORY_DRAIN_RESTART_MS`), then up to 70 s for the replacement to answer * (`REPLACEMENT_READY_TIMEOUT_MS`), plus 15 s. Past it the page says the restart is slow and keeps * checking, so a Child that comes up late still reloads the page. */ diff --git a/gui/tests/memory-observability-card.test.tsx b/gui/tests/memory-observability-card.test.tsx index 6c134c161ef..83137fe9f68 100644 --- a/gui/tests/memory-observability-card.test.tsx +++ b/gui/tests/memory-observability-card.test.tsx @@ -158,7 +158,7 @@ test("Drain & restart posts /api/system/restart only after the in-page dialog is return Response.json({ success: true, activeTurnCount: 2, - drainTimeoutMs: 60_000, + drainTimeoutMs: 2_000, alreadyDraining: false, }, { status: 202 }); } @@ -178,6 +178,7 @@ test("Drain & restart posts /api/system/restart only after the in-page dialog is // The click alone must not restart anything: the consent is a real dialog now, and // inside the app the platform one drew nothing, so nothing could answer it. expect(actionDialogOpen(testWindow.document as unknown as Document)).toBe(true); + expect(testWindow.document.body.textContent).toContain("up to 2s; remaining requests are cut on timeout"); expect(restartPosts).toBe(0); await act(async () => { diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 5a6f6c0f21f..d0c20723265 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1807,6 +1807,7 @@ "system-restart-client-package-tree.test.ts": "cli", "system-restart-client.test.ts": "cli", "system-restart-contract-security.test.ts": "lib", + "system-restart-admission.test.ts": "server", "system-restart.test.ts": "server", "system-routes.test.ts": "server", "systemd-install-cleanup-hardening.test.ts": "service", diff --git a/src/lib/system-restart-contract.ts b/src/lib/system-restart-contract.ts index 963b721abba..ba7fec0c271 100644 --- a/src/lib/system-restart-contract.ts +++ b/src/lib/system-restart-contract.ts @@ -8,9 +8,7 @@ export const SYSTEM_RESTART_EXPECTED_PID_HEADER = "x-opencodex-restart-expected- export const SYSTEM_RESTART_NONCE_HEADER = "x-opencodex-restart-nonce"; export const SYSTEM_RESTART_CAPABILITY_HEADER = "x-opencodex-restart-capability"; -/** Fixed drain and replacement budgets shared by the server, CLI, and tray. */ -export const MEMORY_DRAIN_RESTART_MS = 60_000; -export const REPLACEMENT_READY_TIMEOUT_MS = 70_000; +export { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS } from "./system-restart-timing"; /** * The env var a restarting process hands its replacement `ocx start`: the restarting (parent) pid. diff --git a/src/lib/system-restart-timing.ts b/src/lib/system-restart-timing.ts new file mode 100644 index 00000000000..9eba6fb7004 --- /dev/null +++ b/src/lib/system-restart-timing.ts @@ -0,0 +1,6 @@ +/** Active turns and pre-existing scoped drains get this grace from restart acceptance. */ +export const RESTART_DRAIN_GRACE_MS = 2_000; +/** Terminal cleanup watchdog; also part of the CLI/tray restart observation budget. */ +export const MEMORY_DRAIN_RESTART_MS = 60_000; +/** Replacement startup has its own budget; short drain grace does not guarantee short downtime. */ +export const REPLACEMENT_READY_TIMEOUT_MS = 70_000; diff --git a/src/server/lifecycle.ts b/src/server/lifecycle.ts index e36bc460682..c4da2c28a16 100644 --- a/src/server/lifecycle.ts +++ b/src/server/lifecycle.ts @@ -479,7 +479,7 @@ export async function drainAndShutdown( beginBackgroundShellShutdown(); try { while (admittedTurns.size > 0 && Date.now() < deadline) { - await Bun.sleep(100); + await Bun.sleep(Math.min(100, Math.max(0, deadline - Date.now()))); } if (admittedTurns.size > 0) { console.warn(`⚠️ Aborting ${admittedTurns.size} in-flight turn(s) after ${timeoutMs}ms deadline`); diff --git a/src/server/management/system-restart.ts b/src/server/management/system-restart.ts index 5f7ff8e3f7e..655970043dd 100644 --- a/src/server/management/system-restart.ts +++ b/src/server/management/system-restart.ts @@ -1,8 +1,8 @@ /** * Dashboard memory-card drain-and-restart (#563). * - * Longer than POST /api/stop's short drain: waits up to 60s for active turns, - * then respawns. Never runs restoreNativeCodex / stripGrokConfig — this is a + * Gives active turns/scoped drains a short grace, with a separate 60s cleanup + * watchdog, then respawns. Never runs restoreNativeCodex / stripGrokConfig — this is a * recycle to reclaim RSS, not a teardown. * * Respawn policy (matches real supervisor configs in src/service.ts): @@ -59,6 +59,7 @@ import { MEMORY_DRAIN_RESTART_MS, isDesktopSupervised, } from "../../lib/system-restart-contract"; +import { RESTART_DRAIN_GRACE_MS } from "../../lib/system-restart-timing"; import { spawnReplacementStart } from "../restart-replacement"; export { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS } from "../../lib/system-restart-contract"; @@ -352,7 +353,7 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: }; if (automatic && !temporaryDrain) { restartAccepted = false; - return { accepted: true, alreadyDraining: true, activeTurnCount, drainTimeoutMs: MEMORY_DRAIN_RESTART_MS }; + return { accepted: true, alreadyDraining: true, activeTurnCount, drainTimeoutMs: RESTART_DRAIN_GRACE_MS }; } let pending = true; let vetoed = false; @@ -363,7 +364,9 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: releasePending(); }); const now = io.now ?? Date.now; - const restartDeadlineMs = now() + MEMORY_DRAIN_RESTART_MS; + const acceptedAtMs = now(); + const restartDeadlineMs = acceptedAtMs + MEMORY_DRAIN_RESTART_MS; + const drainDeadlineMs = acceptedAtMs + RESTART_DRAIN_GRACE_MS; // Reject new data-plane traffic immediately (503), before the 200ms response-flush delay. if (!automatic) { if (io.beginShutdownDrain) io.beginShutdownDrain(); @@ -390,8 +393,11 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: // closes the listener and makes both the server ref and runtime metadata stale. const restartPort = (io.listenPort ?? resolveListenPort)(); const drain = io.drainAndShutdown ?? drainAndShutdown; - const remainingMs = Math.max(0, restartDeadlineMs - now()); - const drainPromise = Promise.resolve().then(() => drain(undefined, remainingMs)); + // Bound active/scoped-drain waiting independently of cleanup and readiness. + // A cut turn may already have executed upstream; never replay it automatically. + const drainPromise = Promise.resolve().then( + () => drain(undefined, Math.max(0, drainDeadlineMs - now())), + ); const scheduleDeadline = io.scheduleDeadline ?? ((fn, ms) => { const timer = setTimeout(fn, ms); return () => clearTimeout(timer); @@ -458,6 +464,6 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: accepted: true, alreadyDraining, activeTurnCount, - drainTimeoutMs: MEMORY_DRAIN_RESTART_MS, + drainTimeoutMs: RESTART_DRAIN_GRACE_MS, }; } diff --git a/src/server/management/system-routes.ts b/src/server/management/system-routes.ts index 849cde12540..455432115ff 100644 --- a/src/server/management/system-routes.ts +++ b/src/server/management/system-routes.ts @@ -179,7 +179,7 @@ export async function handleSystemRoutes(ctx: ManagementContext): Promise/link/client-tunnel.pid` with `{ versio The dashboard link routes (`GET /api/link/status`, `GET /api/link/candidates`, `POST /api/link/probe`, `POST /api/link/confirm-host`, `POST /api/link/apply` and `DELETE /api/link/{id}`) admit a paired GUI session, or, on a standalone runtime only, the current loopback-issued GUI session that reached the public listener bound to a loopback hostname. `POST /api/link/join` is stricter and requires the paired session because it durably redirects this machine's client traffic. The hub-link, hub-management and claude-intercept ingresses are never trusted loopback ingress, a stale session is refused, and a hub keeps the paired-only rule. The Tailscale identity refusal runs before either check. A loopback session is minted by the loopback dashboard bootstrap without a credential, so it proves possession, not user presence: any local process can fetch the bootstrap and replay its token and CSRF value, which is why it cannot authorize joining and why `src/client/machine-listener.ts` keeps durable machine changes on the connected listener away from that session. The remaining link routes accept this casual-path trade, the same as `POST /api/github/star` in `src/server/management/sidebar-routes.ts`; it is not a secret-backed boundary like an operator-created pairing grant or the admin token. -A successful join restarts this proxy (a 503 drain of up to a minute while running turns finish, then a closed listener) into the client runtime on the configured port. `GET /api/link/status` tells a GUI-session caller whether it may join: the response gains `joinAvailable`, true only for a paired dashboard session on a standalone runtime whose live port is its configured port, and `joinDenied`, which is `null` when allowed or the first failed gate (`pairing_required`, `standalone_required`, then `join_port_mismatch`). An admin-token caller gets the exact K16 document without either field, because `ocx link status` validates it key by key. The dashboard reads an absent availability field as false and an absent or unknown denial as generic unavailability; while unavailable the Child role card cannot be selected by pointer or keyboard. A pairing refusal says to pair this machine first and offers code entry only on the existing same-origin literal-loopback transport. A port refusal instead asks the operator to restart on the configured port; neither status reads nor displaying this guidance mint a grant or change pairing, keys, routing, or runtime state. Before **Connect as Child** the confirmation panel says that the restart briefly interrupts Codex and that Codex keeps its local address. The dashboard reads the standalone's pid from same-origin `/healthz`, sends the join, then reads `/healthz` once a second, skipping status polls meanwhile, and reloads only when it reports `role: "client"` under another pid, so the reloaded document carries the client role and a fresh session. It never gives up by itself: past the server's own handoff budget (60 s drain plus 70 s replacement readiness, plus a margin: 145 seconds) it says the restart is slow and keeps reading `/healthz` every 5 seconds until the Child answers or the page is left. +A successful join restarts this proxy (a 503 fence with a two-second active-turn grace, followed by cleanup and a closed listener; see [restart handoff](ops/service-and-sidecars.md#restart-handoff)) into the client runtime on the configured port. `GET /api/link/status` tells a GUI-session caller whether it may join: the response gains `joinAvailable`, true only for a paired dashboard session on a standalone runtime whose live port is its configured port, and `joinDenied`, which is `null` when allowed or the first failed gate (`pairing_required`, `standalone_required`, then `join_port_mismatch`). An admin-token caller gets the exact K16 document without either field, because `ocx link status` validates it key by key. The dashboard reads an absent availability field as false and an absent or unknown denial as generic unavailability; while unavailable the Child role card cannot be selected by pointer or keyboard. A pairing refusal says to pair this machine first and offers code entry only on the existing same-origin literal-loopback transport. A port refusal instead asks the operator to restart on the configured port; neither status reads nor displaying this guidance mint a grant or change pairing, keys, routing, or runtime state. Before **Connect as Child** the confirmation panel says that the restart briefly interrupts Codex and that Codex keeps its local address. The dashboard reads the standalone's pid from same-origin `/healthz`, sends the join, then reads `/healthz` once a second, skipping status polls meanwhile, and reloads only when it reports `role: "client"` under another pid, so the reloaded document carries the client role and a fresh session. It never gives up by itself: past the server's own handoff budget (60 s cleanup watchdog plus 70 s replacement readiness, plus a margin: 145 seconds) it says the restart is slow and keeps reading `/healthz` every 5 seconds until the Child answers or the page is left. A connected Child answers `GET` and `HEAD /api/link/status` on its own listener for a GUI session: `src/client/link-status.ts` projects the client sidecar and the tunnel supervisor into the K16 document with `role: "child"`, the listener off, no links and the child row, plus `joinAvailable: false`. A Home-initiated Child has no sidecar and reports `child: null`. In link mode `/api/machine/status` advertises the machine origin as the shared plane, because the tunnel's hub-link ingress serves no `/api/*` and no session bootstrap. diff --git a/structure/runtime.md b/structure/runtime.md index d3eac545039..3e67f412435 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -193,7 +193,7 @@ until shutdown. Normal shutdown restores native Codex; a sibling instance beside `OCX_SERVICE=1`, so managed restarts do not repeatedly restore/reinject; explicit service stop and uninstall still restore. `src/service/cli.ts` removes the service token on uninstall only when persisted client state is disconnected and no pending connect marker owns the newly issued key. `src/client/connect.ts` publishes that fingerprint marker before the key, then clears it with the connection commit or rollback under the client lifecycle and config mutation locks. Connected, invalid, or mismatched client state retains an existing token. A valid pending marker retains only its matching fingerprint; an older marker does not own a replacement service key. An absent token is reported as absent; unsafe, malformed, or unreadable markers and lock, state-read, or deletion failures leave cleanup unverified. `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart timing and replacement ownership follow the [restart handoff contract](ops/service-and-sidecars.md#restart-handoff). A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 700841acd0a..adbcf03b2af 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1023,7 +1023,7 @@ "sync-client-integrations.test.ts": "clients", "synced-state-location.test.ts": "lib", "synthetic-tool.test.ts": "images", "system-env.test.ts": "server", "system-restart-client-package-tree.test.ts": "cli", "system-restart-client.test.ts": "cli", - "system-restart-contract-security.test.ts": "lib", "system-restart.test.ts": "server", + "system-restart-contract-security.test.ts": "lib", "system-restart-admission.test.ts": "server", "system-restart.test.ts": "server", "system-routes.test.ts": "server", "systemd-install-cleanup-hardening.test.ts": "service", "tencent-siliconflow-providers.test.ts": "gui", "terminal-continuation-owner-rotation.test.ts": "adapters", "terminal-guard-server.test.ts": "server", "terminal-guard.test.ts": "server", diff --git a/tests/server/system-restart-admission.test.ts b/tests/server/system-restart-admission.test.ts new file mode 100644 index 00000000000..d3529714517 --- /dev/null +++ b/tests/server/system-restart-admission.test.ts @@ -0,0 +1,328 @@ +/** #6643: active-turn grace must not consume the whole restart cleanup budget. */ +import { afterEach, expect, spyOn, test } from "bun:test"; +import { + acceptSystemRestart, + setSystemRestartIoForTests, + noteExplicitShutdownRequested, +} from "../../src/server/management/system-restart"; + +import { + abortAndReleaseAllTurns, + acquireTemporaryDrain, + drainAndShutdown, + getActiveTurnCount, + isDraining, + registerTurn, + resetLifecycleDrainStateForTests, + stopServerListener, + tryAdmitTurn, +} from "../../src/server/lifecycle"; + +afterEach(() => setSystemRestartIoForTests()); + +test("restart spends at most two seconds on active turns, including the response-flush delay", async () => { + let now = 10_000; + let scheduled!: () => void | Promise; + let turnWaitMs = -1; + let cleanupWaitMs = -1; + acceptSystemRestart({ + now: () => now, + isDraining: () => false, + getActiveTurnCount: () => 1, + setDraining: () => {}, + schedule: fn => { scheduled = fn; }, + scheduleDeadline: (_fn, ms) => { cleanupWaitMs = ms; return () => {}; }, + drainAndShutdown: async (_server, ms) => { turnWaitMs = ms; }, + isDesktopSupervised: () => false, + isSupervisedServiceChild: () => false, + listenPort: () => 10123, + stopListener: () => {}, + spawnStart: () => {}, + markRecycling: () => {}, + exitProcess: () => {}, + }); + now += 200; + await scheduled(); + expect(turnWaitMs).toBe(1_800); + expect(cleanupWaitMs).toBe(59_800); +}); + + +class RestartClock { + now = 10_000; + private timers = new Set<{ at: number; run: () => void }>(); + schedule = (run: () => void, ms: number) => { + const timer = { at: this.now + ms, run }; + this.timers.add(timer); + return () => { this.timers.delete(timer); }; + }; + async settle() { + // Let real shutdown cleanup promises settle without advancing virtual time. + for (let i = 0; i < 40; i += 1) await Promise.resolve(); + } + async advance(ms: number) { + const until = this.now + ms; + while (true) { + const next = [...this.timers].filter(timer => timer.at <= until) + .sort((a, b) => a.at - b.at)[0]; + if (!next) break; + this.now = next.at; + this.timers.delete(next); + next.run(); + await this.settle(); + } + this.now = until; + await this.settle(); + } +} + +const restore: Array<() => void> = []; +afterEach(() => { + for (const reset of restore.splice(0)) reset(); + abortAndReleaseAllTurns(); + resetLifecycleDrainStateForTests(); +}); + +function restartFixture(options: { automatic?: boolean; holdCleanup?: boolean; holdReadiness?: boolean; failSpawn?: boolean; flushDelayMs?: number } = {}) { + const clock = new RestartClock(); + const date = spyOn(Date, "now").mockImplementation(() => clock.now); + const sleep = spyOn(Bun, "sleep").mockImplementation((ms) => new Promise(resolve => { + clock.schedule(resolve, typeof ms === "number" ? ms : ms.getTime() - clock.now); + })); + restore.push(() => date.mockRestore(), () => sleep.mockRestore()); + const calls: string[] = []; + let finishCleanup!: () => void; + const cleanup = new Promise(resolve => { finishCleanup = resolve; }); + let finishReadiness!: () => void; + const readiness = new Promise(resolve => { finishReadiness = resolve; }); + let veto!: () => void; + let running: void | Promise; + const server = { + stop: () => { calls.push(`stop:${clock.now}`); }, + } as unknown as ReturnType; + const accept = () => acceptSystemRestart({ + now: () => clock.now, + getActiveTurnCount, + schedule: (fn, ms) => { clock.schedule(() => { running = fn(); }, options.flushDelayMs ?? ms); }, + scheduleDeadline: clock.schedule, + drainAndShutdown: async (_server, ms) => { + calls.push(`drain:${ms}`); + const result = await drainAndShutdown(server, ms); + if (options.holdCleanup) await cleanup; + return result; + }, + stopListener: () => stopServerListener(server), + listenPort: () => 10123, + isDesktopSupervised: () => false, + isSupervisedServiceChild: () => false, + spawnStart: async (port, waitForHealth) => { + calls.push(`spawn:${port}:${waitForHealth}:${clock.now}`); + if (options.failSpawn) throw Object.assign(new Error("fixture"), { code: "EACCES" }); + if (options.holdReadiness) await readiness; + }, + markRecycling: () => { calls.push("recycle"); }, + isClientConnected: () => false, + exitProcess: code => { calls.push(`exit:${code}`); }, + }, options.automatic ? { onAccepted: callback => { veto = callback; } } : {}); + return { clock, calls, accept, finishCleanup, finishReadiness, veto: () => veto(), running: () => running }; +} + +function upstreamTurn(bindController = true) { + const lease = tryAdmitTurn(); + expect(lease).not.toBeNull(); + const controller = new AbortController(); + if (bindController) registerTurn(controller, lease!); + // The admitted handler may have sent work whose upstream outcome is not yet known. + return { lease: lease!, controller }; +} + +for (const automatic of [false, true]) { + const mode = automatic ? "automatic" : "manual"; + test(`${mode} restart preserves a completed turn and hands off without spending the grace`, async () => { + const fixture = restartFixture({ automatic }); + const turn = upstreamTurn(); + turn.lease.release(); + expect(fixture.accept().drainTimeoutMs).toBe(2_000); + expect(tryAdmitTurn()).toBeNull(); + await fixture.clock.advance(200); + await fixture.running(); + expect(fixture.calls).toContain("spawn:10123:true:10200"); + expect(turn.controller.signal.aborted).toBe(false); + }); + + test(`${mode} restart lets a turn finish inside the grace without abort`, async () => { + const fixture = restartFixture({ automatic }); + const turn = upstreamTurn(); + fixture.accept(); + await fixture.clock.advance(700); + expect(turn.controller.signal.aborted).toBe(false); + expect(fixture.calls.some(call => call.startsWith("spawn:"))).toBe(false); + turn.lease.release(); + await fixture.clock.advance(100); + await fixture.running(); + expect(fixture.calls).toContain("spawn:10123:true:10800"); + expect(turn.controller.signal.aborted).toBe(false); + }); + + test(`${mode} restart cuts an ambiguous active turn at grace expiry`, async () => { + const fixture = restartFixture({ automatic }); + const turn = upstreamTurn(); + fixture.accept(); + await fixture.clock.advance(1_999); + expect(turn.controller.signal.aborted).toBe(false); + expect(getActiveTurnCount()).toBe(1); + await fixture.clock.advance(1); + await fixture.running(); + expect(turn.controller.signal.aborted).toBe(true); + expect(getActiveTurnCount()).toBe(0); + expect(fixture.calls).toContain("spawn:10123:true:12000"); + // Successful cleanup/readiness does not reopen admission in the old process. + expect(tryAdmitTurn()).toBeNull(); + }); +} + +test("an admitted turn without a controller is released at the same deadline", async () => { + const fixture = restartFixture(); + upstreamTurn(false); + fixture.accept(); + await fixture.clock.advance(2_000); + await fixture.running(); + expect(getActiveTurnCount()).toBe(0); + expect(fixture.calls).toContain("exit:0"); +}); + +test("client cancellation finishes the drain without waiting out the grace", async () => { + const fixture = restartFixture(); + const turn = upstreamTurn(); + turn.controller.signal.addEventListener("abort", () => turn.lease.release(), { once: true }); + fixture.accept(); + await fixture.clock.advance(300); + turn.controller.abort(new Error("client cancelled")); + await fixture.clock.advance(100); + await fixture.running(); + expect(fixture.calls).toContain("spawn:10123:true:10400"); +}); + +test("held cleanup retains the original 60s watchdog and ignores late completion", async () => { + const fixture = restartFixture({ holdCleanup: true }); + fixture.accept(); + await fixture.clock.advance(2_000); + expect(fixture.calls.some(call => call.startsWith("spawn:"))).toBe(false); + expect(isDraining()).toBe(true); + await fixture.clock.advance(58_000); + await fixture.running(); + expect(fixture.calls).toContain("spawn:10123:false:70000"); + fixture.finishCleanup(); + await fixture.clock.settle(); + expect(fixture.calls.filter(call => call.startsWith("spawn:"))).toHaveLength(1); + expect(fixture.calls.filter(call => call.startsWith("exit:"))).toEqual(["exit:0"]); +}); + +test("delayed replacement readiness does not reopen the old process while a cut turn stays aborted", async () => { + const fixture = restartFixture({ holdReadiness: true }); + const turn = upstreamTurn(); + fixture.accept(); + await fixture.clock.advance(2_000); + expect(fixture.calls).toContain("spawn:10123:true:12000"); + await fixture.clock.advance(65_000); + expect(fixture.calls.some(call => call.startsWith("exit:"))).toBe(false); + expect(tryAdmitTurn()).toBeNull(); + expect(turn.controller.signal.aborted).toBe(true); + fixture.finishReadiness(); + await fixture.running(); + expect(fixture.calls).toContain("exit:0"); +}); + +test("automatic pending cancellation releases only its admission fence", async () => { + const fixture = restartFixture({ automatic: true }); + fixture.accept(); + expect(isDraining()).toBe(true); + fixture.veto(); + await fixture.clock.advance(2_000); + expect(fixture.calls).toEqual([]); + const admitted = tryAdmitTurn(); + expect(admitted).not.toBeNull(); + admitted?.release(); +}); + +for (const automatic of [false, true]) { + test(`explicit stop during ${automatic ? "automatic" : "manual"} grace preserves restart ownership`, async () => { + const fixture = restartFixture({ automatic }); + upstreamTurn(); + fixture.accept(); + await fixture.clock.advance(200); + noteExplicitShutdownRequested(); + await fixture.clock.advance(1_800); + await fixture.running(); + expect(fixture.calls.filter(call => call.startsWith("spawn:"))).toHaveLength(automatic ? 0 : 1); + expect(isDraining()).toBe(true); + }); +} + +test("spawn failure after a cut turn exits once without recycling", async () => { + const fixture = restartFixture({ failSpawn: true }); + upstreamTurn(); + fixture.accept(); + await fixture.clock.advance(2_000); + await fixture.running(); + expect(fixture.calls.filter(call => call.startsWith("exit:"))).toEqual(["exit:1"]); + expect(fixture.calls).not.toContain("recycle"); +}); + +test("waiting to enter the drain microtask does not renew the acceptance-based grace", async () => { + let now = 10_000; + let scheduled!: () => void | Promise; + let turnWaitMs = -1; + acceptSystemRestart({ + now: () => now, + isDraining: () => false, + getActiveTurnCount: () => 1, + setDraining: () => {}, + schedule: fn => { scheduled = fn; }, + scheduleDeadline: () => () => {}, + drainAndShutdown: async (_server, ms) => { turnWaitMs = ms; }, + isDesktopSupervised: () => false, + isSupervisedServiceChild: () => false, + listenPort: () => 10123, + stopListener: () => {}, + spawnStart: () => {}, + markRecycling: () => {}, + exitProcess: () => {}, + }); + now += 200; + const running = scheduled(); + now += 350; + await running; + expect(turnWaitMs).toBe(1_450); +}); + +test("a non-round response-flush delay does not extend grace by a full lifecycle poll", async () => { + const fixture = restartFixture({ flushDelayMs: 275 }); + const turn = upstreamTurn(); + fixture.accept(); + await fixture.clock.advance(1_999); + expect(turn.controller.signal.aborted).toBe(false); + await fixture.clock.advance(1); + await fixture.running(); + expect(turn.controller.signal.aborted).toBe(true); + expect(fixture.calls).toContain("drain:1725"); + expect(fixture.calls).toContain("spawn:10123:true:12000"); +}); + +test("an existing scoped drain and active turn share the same grace", async () => { + const fixture = restartFixture(); + const turn = upstreamTurn(); + const scoped = acquireTemporaryDrain("fixture-scoped-drain"); + expect(scoped).not.toBeNull(); + fixture.accept(); + await fixture.clock.advance(500); + expect(fixture.calls).toEqual(["drain:1800"]); + scoped!.release(); + await fixture.clock.settle(); + await fixture.clock.advance(1_499); + expect(turn.controller.signal.aborted).toBe(false); + await fixture.clock.advance(1); + await fixture.running(); + expect(turn.controller.signal.aborted).toBe(true); + expect(fixture.calls).toContain("spawn:10123:true:12000"); +}); diff --git a/tests/server/system-restart.test.ts b/tests/server/system-restart.test.ts index 1a7c67ff046..ced98f461d0 100644 --- a/tests/server/system-restart.test.ts +++ b/tests/server/system-restart.test.ts @@ -20,6 +20,7 @@ import { waitForReplacementReady, } from "../../src/server/management/system-restart"; import { SYSTEM_RESTART_EXPECTED_PID_HEADER } from "../../src/lib/system-restart-contract"; +import { RESTART_DRAIN_GRACE_MS } from "../../src/lib/system-restart-timing"; import type { OcxConfig } from "../../src/types"; function config(): OcxConfig { @@ -101,7 +102,7 @@ describe("acceptSystemRestart", () => { accepted: true, alreadyDraining: false, activeTurnCount: 3, - drainTimeoutMs: MEMORY_DRAIN_RESTART_MS, + drainTimeoutMs: RESTART_DRAIN_GRACE_MS, }); // Data-plane reject must arm before the 200ms flush delay runs. expect(calls).toEqual(["draining:true"]); @@ -109,11 +110,11 @@ describe("acceptSystemRestart", () => { expect(scheduled).not.toBeNull(); now += 15_000; await scheduled!(); - expect(calls).toEqual(["draining:true", "drain:45000", "stop", "start:10123:ready", "recycle", "exit:0"]); + expect(calls).toEqual(["draining:true", "drain:0", "stop", "start:10123:ready", "recycle", "exit:0"]); expect(deadlineDelay).toBe(45_000); expect(deadlineCancellations).toBe(1); deadlineCallback?.(); - expect(calls).toEqual(["draining:true", "drain:45000", "stop", "start:10123:ready", "recycle", "exit:0"]); + expect(calls).toEqual(["draining:true", "drain:0", "stop", "start:10123:ready", "recycle", "exit:0"]); }); test("snapshots the restart port before normal drain invalidates listener metadata", async () => { @@ -921,7 +922,7 @@ describe("POST /api/system/restart", () => { }; expect(body.success).toBe(true); expect(body.activeTurnCount).toBe(1); - expect(body.drainTimeoutMs).toBe(60_000); + expect(body.drainTimeoutMs).toBe(RESTART_DRAIN_GRACE_MS); expect(body.alreadyDraining).toBe(false); expect(body.message.toLowerCase()).toContain("drain"); }); From bb391daedeb3876ad27fc10b648efc2ce6329e67 Mon Sep 17 00:00:00 2001 From: BigHulk Date: Wed, 7 Oct 2026 22:59:29 +0800 Subject: [PATCH 2/5] feat(lifecycle): make shorter restart grace an explicit API option Signed-off-by: BigHulk --- .../docs/fr/troubleshooting/windows-memory.md | 2 +- .../docs/ja/troubleshooting/windows-memory.md | 2 +- .../docs/ko/troubleshooting/windows-memory.md | 2 +- .../content/docs/reference/management-api.md | 28 +++-- .../docs/ru/troubleshooting/windows-memory.md | 4 +- .../docs/tr/troubleshooting/windows-memory.md | 5 +- .../docs/troubleshooting/windows-memory.md | 9 +- .../zh-cn/troubleshooting/windows-memory.md | 2 +- .../components/MemoryObservabilityCard.tsx | 7 +- gui/src/remote-link-api.ts | 2 +- gui/tests/memory-observability-card.test.tsx | 3 +- src/lib/system-restart-contract.ts | 4 +- src/lib/system-restart-timing.ts | 6 - src/server/management/system-restart.ts | 24 ++-- src/server/management/system-routes.ts | 26 ++++- src/tray/windows-tray.ps1 | 2 +- structure/ops/service-and-sidecars.md | 19 ++- structure/remote-link.md | 2 +- structure/runtime.md | 2 +- tests/server/system-restart-admission.test.ts | 108 +++++++++++++++++- tests/server/system-restart.test.ts | 9 +- 21 files changed, 198 insertions(+), 70 deletions(-) delete mode 100644 src/lib/system-restart-timing.ts diff --git a/docs-site/src/content/docs/fr/troubleshooting/windows-memory.md b/docs-site/src/content/docs/fr/troubleshooting/windows-memory.md index f72167171b7..38fd2b99294 100644 --- a/docs-site/src/content/docs/fr/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/fr/troubleshooting/windows-memory.md @@ -23,7 +23,7 @@ Il s’agit d’une atténuation bornée et d’outils d’observation — **pas - **Surveillance de la mémoire** — le proxy échantillonne sa propre mémoire toutes les minutes et consigne un avertissement à fréquence limitée lorsque la mémoire observée dépasse 4 GiB. La mémoire observée correspond à la plus grande des valeurs RSS, `external` et `arrayBuffers` (et non à leur somme), car les compteurs d’ensemble de travail/RSS de Windows peuvent sous-estimer la rétention externe validée. - **`ocx doctor`** — une section « Memory / runtime » affiche la version de Bun du processus de *service*, les compteurs RSS, external/ArrayBuffers, le contexte du tas JS et la décision relative au mode de flux. Avec le runtime Bun 1.3.14 intégré, `heapUsed` / `jscHeap` ne permettent pas à eux seuls de caractériser une fuite ; comparez la mémoire observée à `responseState` et à plusieurs échantillons avant de conclure à une fuite au niveau de l’application. -- **`GET /api/system/memory`** — expose les mêmes données via l’API de gestion authentifiée, à destination des tableaux de bord ou des scripts. Outre les compteurs RSS/heap/external, la réponse contient un bloc scalaire `responseState` (nombre d’entrées, nombre total/maximal d’octets sérialisés et âge de l’entrée la plus ancienne) pour le magasin de continuation en mémoire `previous_response_id` du proxy. Ces données permettent de mieux attribuer la croissance : si `responseState.totalBytes` et la mémoire observée augmentent ensemble, la rétention des conversations est probablement en cause (les longues chaînes `store:false` sont redéveloppées à chaque tour) ; si `responseState` reste stable tandis que la mémoire observée augmente, il faut chercher ailleurs. Les valeurs sont exclusivement scalaires : aucun corps de requête, jeton, chemin ou identifiant de compte n’est exposé. La lecture est dépourvue d’effet secondaire : elle n’élague ni n’expulse jamais de données. La carte **Observabilité de la mémoire** du tableau de bord affiche les mêmes champs et propose une action **Drainer et redémarrer** soumise à confirmation. Elle indique le nombre actuel de tours actifs, attend leur fin jusqu’à 2 s après acceptation (en réutilisant le drainage existant avec 503 + `Retry-After`), puis interrompt les tours restants. Le proxy actif contrôle l’autorisation du redémarrage et la coordination du drainage, puis se termine ; lorsqu’un gestionnaire de services est installé, il lance le processus de remplacement. L’action ne signale une réussite qu’après avoir vérifié qu’un autre processus, dont l’identité est confirmée, est sain sur le même port, sans retirer l’injection Codex. Le nettoyage et le démarrage ont des délais distincts ; une requête interrompue peut déjà avoir été exécutée et n’est pas rejouée automatiquement. Voir [délais et sécurité des nouvelles tentatives](/reference/management-api/#system-lifecycle) (anglais). +- **`GET /api/system/memory`** — expose les mêmes données via l’API de gestion authentifiée, à destination des tableaux de bord ou des scripts. Outre les compteurs RSS/heap/external, la réponse contient un bloc scalaire `responseState` (nombre d’entrées, nombre total/maximal d’octets sérialisés et âge de l’entrée la plus ancienne) pour le magasin de continuation en mémoire `previous_response_id` du proxy. Ces données permettent de mieux attribuer la croissance : si `responseState.totalBytes` et la mémoire observée augmentent ensemble, la rétention des conversations est probablement en cause (les longues chaînes `store:false` sont redéveloppées à chaque tour) ; si `responseState` reste stable tandis que la mémoire observée augmente, il faut chercher ailleurs. Les valeurs sont exclusivement scalaires : aucun corps de requête, jeton, chemin ou identifiant de compte n’est exposé. La lecture est dépourvue d’effet secondaire : elle n’élague ni n’expulse jamais de données. La carte **Observabilité de la mémoire** du tableau de bord affiche les mêmes champs et propose une action **Drainer et redémarrer** soumise à confirmation. Elle indique le nombre actuel de tours actifs, attend leur fin jusqu’à 60 s (en réutilisant le drainage existant avec 503 + `Retry-After`), puis interrompt les tours restants. Le proxy actif contrôle l’autorisation du redémarrage et la coordination du drainage, puis se termine ; lorsqu’un gestionnaire de services est installé, il lance le processus de remplacement. L’action ne signale une réussite qu’après avoir vérifié qu’un autre processus, dont l’identité est confirmée, est sain sur le même port, sans retirer l’injection Codex. Ce recyclage est plus long et mieux contrôlé que le court drainage déclenché par `POST /api/stop`. - **Autre chemin de flux soumis à activation** — un relais borné à lecteur unique qui élimine entièrement le modèle de mise en mémoire tampon non bornée. Sous Windows, il deviendra automatiquement le chemin par défaut dès qu’une version intégrée de Bun inclura de manière vérifiable le correctif #32111 ; pour l’instant, il faut l’activer explicitement (voir ci-dessous). Sous macOS, il restera soumis à une activation explicite même après une telle version : le passage de `auto` sous macOS fera l’objet d’une décision distincte. L’amélioration réelle du RSS apportée par ces changements reste **à vérifier par des utilisateurs Windows** — nous n’affirmons pas que la fuite est corrigée. diff --git a/docs-site/src/content/docs/ja/troubleshooting/windows-memory.md b/docs-site/src/content/docs/ja/troubleshooting/windows-memory.md index 044aa6d35f2..6d25a176336 100644 --- a/docs-site/src/content/docs/ja/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/ja/troubleshooting/windows-memory.md @@ -26,7 +26,7 @@ Windows では、#32111 クラッシュを回避するために、opencodex は - **`ocx doctor`** — 「メモリ / ランタイム」セクションには *サービス* が表示されます プロセスの Bun バージョン、RSS、外部/ArrayBuffers カウンター、JS ヒープ コンテキスト、およびストリーム モードの決定。バンドルされている Bun 1.3.14 ランタイムでは、`heapUsed` / `jscHeap` 単独ではリーク識別子ではありません。アプリレベルのリークを割り当てる前に、観察されたメモリを `responseState` および繰り返しサンプルと比較します。 - **`GET /api/system/memory`** — 認証済みの同じデータ -ダッシュボードまたはスクリプトの管理 API。 RSS/ヒープ/外部カウンターとともに、プロキシのメモリ内 `previous_response_id` 継続ストアのスカラー `responseState` ブロック (エントリ数、シリアル化された合計/最大バイト数、最も古いエントリの経過時間) を報告します。これはさらに成長に起因します。観察された記憶の上昇下での `responseState.totalBytes` の上昇は会話の保持を指します (長い `store:false` チェーンはターンごとに再拡張します)。一方、観察された記憶の上昇の下での横ばいの `responseState` はそのストアから遠ざかることを示します。値はスカラーのみであり、リクエスト本文、トークン、パス、アカウント識別子はありません。また、読み取りには副作用はありません (プルーニングや削除は行われません)。ダッシュボードの **メモリ可観測性** カードは同じフィールドをレンダリングし、確認ゲート付き **ドレインと再起動** アクションを提供します。現在のアクティブ ターン数を表示し、受け付けから最大 2 秒間アクティブ ターンを待機し (既存の 503 + `Retry-After` ドレインを再利用)、残りのターンを中止します。実行中のプロキシは再起動の認可とドレイン調整を担当して終了し、サービス管理下ではインストール済みサービス マネージャーが置換プロセスを起動します。同じポートで、別の ID 検証済みプロセスが正常であることを確認した場合にのみ成功し、Codex インジェクションは破棄しません。クリーンアップと起動には別の待機時間があります。中断された要求は上流ですでに実行された可能性があり、自動再送はしません。[再起動時間と再試行の安全性](/reference/management-api/#system-lifecycle)（英語）を参照してください。 +ダッシュボードまたはスクリプトの管理 API。 RSS/ヒープ/外部カウンターとともに、プロキシのメモリ内 `previous_response_id` 継続ストアのスカラー `responseState` ブロック (エントリ数、シリアル化された合計/最大バイト数、最も古いエントリの経過時間) を報告します。これはさらに成長に起因します。観察された記憶の上昇下での `responseState.totalBytes` の上昇は会話の保持を指します (長い `store:false` チェーンはターンごとに再拡張します)。一方、観察された記憶の上昇の下での横ばいの `responseState` はそのストアから遠ざかることを示します。値はスカラーのみであり、リクエスト本文、トークン、パス、アカウント識別子はありません。また、読み取りには副作用はありません (プルーニングや削除は行われません)。ダッシュボードの **メモリ可観測性** カードは同じフィールドをレンダリングし、確認ゲート付き **ドレインと再起動** アクションを提供します。現在のアクティブ ターン数を表示し、アクティブ ターンを最大 60 秒待機し (既存の 503 + `Retry-After` ドレインを再利用)、残りのターンを中止します。実行中のプロキシは再起動の認可とドレイン調整を担当して終了し、サービス管理下ではインストール済みサービス マネージャーが置換プロセスを起動します。同じポートで、別の ID 検証済みプロセスが正常であることを確認した場合にのみ成功し、Codex インジェクションは破棄しません。これは、`POST /api/stop` の短いドレインよりも長く、情報に基づいたリサイクルです。 - **ゲートされた代替ストリーム パス** — 制限された単一リーダー リレー。 境界のないバッファリング形状を完全に削除します。 Windows では、バンドルされた Bun リリースに #32111 修正が確実に適用されると、これが自動的にデフォルトになります。現在はオプトインのみとなっています (以下を参照)。 macOS では、そのようなリリース後でもオプトインのままになります。macOS `auto` を切り替えるかどうかは別の決定となります。 diff --git a/docs-site/src/content/docs/ko/troubleshooting/windows-memory.md b/docs-site/src/content/docs/ko/troubleshooting/windows-memory.md index a76dac87cdf..3feb6b682a3 100644 --- a/docs-site/src/content/docs/ko/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/ko/troubleshooting/windows-memory.md @@ -23,7 +23,7 @@ Windows에서는 opencodex가 #32111 충돌을 피하기 위해 스트리밍 응 - **메모리 감시기** - 프록시는 1분마다 자체 메모리를 샘플링하고, 관측된 메모리가 4 GiB를 넘으면 속도 제한이 걸린 경고를 기록합니다. 관측된 메모리는 RSS, `external`, `arrayBuffers`의 합이 아니라 그중 가장 큰 값입니다. Windows의 working-set/RSS 카운터가 커밋된 external 잔존량을 낮게 잡을 수 있기 때문입니다. - **`ocx doctor`** - "Memory / runtime" 섹션에서 *서비스* 프로세스의 Bun 버전, RSS, external/ArrayBuffers 카운터, JS 힙 문맥, 스트림 모드 결정을 보여줍니다. 번들된 Bun 1.3.14 런타임에서는 `heapUsed` / `jscHeap`만으로 누수를 판별할 수 없습니다. 애플리케이션 수준 누수로 단정하기 전에 관측된 메모리, `responseState`, 반복 샘플을 함께 보아야 합니다. -- **`GET /api/system/memory`** - 대시보드나 스크립트에서 쓸 수 있도록 같은 데이터를 인증된 관리 API로 제공합니다. RSS/heap/external 카운터와 함께, 프록시의 메모리 내 `previous_response_id` 이어받기 저장소에 대한 스칼라 `responseState` 블록(항목 수, 직렬화된 총/최대 바이트, 가장 오래된 항목의 경과 시간)을 보고합니다. 이를 통해 증가 원인을 더 잘 구분할 수 있습니다. 관측된 메모리가 함께 증가하면서 `responseState.totalBytes`도 늘면 대화 보존(long `store:false` 체인이 매 턴 다시 확장되는 경우)을 가리키고, 관측된 메모리는 늘지만 `responseState`는 평평하면 그 저장소와는 무관한 원인을 가리킵니다. 값은 스칼라만 포함하며 요청 본문, 토큰, 경로, 계정 식별자는 포함하지 않습니다. 또한 읽기 동작은 부작용이 없습니다. 절대 prune하거나 evict하지 않습니다. 대시보드의 **Memory observability** 카드는 같은 필드를 렌더링하고, 확인을 거쳐야 하는 **Drain & restart** 동작도 제공합니다. 현재 활성 턴 수를 보여주고, 기존 503 + `Retry-After` 드레인과 같은 방식으로 수락 후 최대 2초 동안 활성 턴을 기다린 뒤, 남아 있는 턴을 강제로 중단하고 Codex 주입을 해제하지 않은 채 확인된 실행 프로세스가 스스로 교체되게 한 다음 같은 포트의 다른 PID를 검증합니다. 정리와 시작에는 별도의 제한 시간이 적용됩니다. 중단된 요청은 업스트림에서 이미 실행되었을 수 있으며 자동으로 재전송하지 않습니다. [재시작 시간과 재시도 안전성](/reference/management-api/#system-lifecycle)(영문)을 참고하세요. +- **`GET /api/system/memory`** - 대시보드나 스크립트에서 쓸 수 있도록 같은 데이터를 인증된 관리 API로 제공합니다. RSS/heap/external 카운터와 함께, 프록시의 메모리 내 `previous_response_id` 이어받기 저장소에 대한 스칼라 `responseState` 블록(항목 수, 직렬화된 총/최대 바이트, 가장 오래된 항목의 경과 시간)을 보고합니다. 이를 통해 증가 원인을 더 잘 구분할 수 있습니다. 관측된 메모리가 함께 증가하면서 `responseState.totalBytes`도 늘면 대화 보존(long `store:false` 체인이 매 턴 다시 확장되는 경우)을 가리키고, 관측된 메모리는 늘지만 `responseState`는 평평하면 그 저장소와는 무관한 원인을 가리킵니다. 값은 스칼라만 포함하며 요청 본문, 토큰, 경로, 계정 식별자는 포함하지 않습니다. 또한 읽기 동작은 부작용이 없습니다. 절대 prune하거나 evict하지 않습니다. 대시보드의 **Memory observability** 카드는 같은 필드를 렌더링하고, 확인을 거쳐야 하는 **Drain & restart** 동작도 제공합니다. 현재 활성 턴 수를 보여주고, 기존 503 + `Retry-After` 드레인과 같은 방식으로 최대 60초 동안 활성 턴을 기다린 뒤, 남아 있는 턴을 강제로 중단하고 Codex 주입을 해제하지 않은 채 확인된 실행 프로세스가 스스로 교체되게 한 다음 같은 포트의 다른 PID를 검증합니다. 이는 `POST /api/stop`의 짧은 드레인보다 더 길고, 더 많은 정보를 반영한 재순환입니다. - **가드된 대체 스트림 경로** - unbounded buffering 형태를 완전히 제거하는 bounded single-reader relay입니다. Windows에서는 번들된 Bun 릴리스가 #32111 수정을 실제로 포함하고 있음이 확인되면 자동으로 기본값이 됩니다. 지금은 아래에서 설명하는 opt-in만 가능합니다. macOS에서는 그런 릴리스 이후에도 계속 opt-in입니다. macOS의 `auto`를 바꾸는 것은 별도의 결정입니다. 이 변경들로 실제 RSS가 얼마나 좋아지는지는 **Windows 사용자의 검증을 기다리고 있습니다**. 아직 이 누수가 해결되었다고 말하지는 않습니다. diff --git a/docs-site/src/content/docs/reference/management-api.md b/docs-site/src/content/docs/reference/management-api.md index 43526726624..5b7e28afe56 100644 --- a/docs-site/src/content/docs/reference/management-api.md +++ b/docs-site/src/content/docs/reference/management-api.md @@ -662,25 +662,33 @@ whether to star the repository. ### System lifecycle +`POST /api/system/restart` keeps the 60-second drain by default, including requests +with no body or `{}`. An authenticated caller can explicitly send a JSON body such +as `{"drainGraceMs":2000}` to choose a shorter active-request/scoped-drain grace. +The value must be an integer from 1 to 60000 milliseconds. Invalid bodies or values +return 400 before starting a drain; the standard management-body limit still applies. +The target-bound local restart capability retains the default grace; setting this +body option requires a management session or admin token (otherwise 403). +The response's `drainTimeoutMs` reports the accepted grace. Repeated calls retain +the first accepted restart's grace and do not change its deadline. + +Grace is measured from acceptance, including the response-flush delay. The cleanup +watchdog remains 60 seconds and replacement readiness retains its separate 70-second +budget. A short grace can interrupt a turn that already executed upstream. Check its +outcome before resubmitting; OpenCodex adds no automatic replay of ambiguous work, +and a healthy replacement does not prove resending is safe. The dashboard, CLI, tray +and automatic restarts continue using the 60-second default. + During a restart drain, new data-plane requests receive HTTP 503 with JSON `error.type: "server_error"`, `error.code: "server_restarting"`, and the message "OpenCodex is restarting; retry this request." Responses retain `Retry-After: 5` and the receiving listener's CORS policy. This code lets every Codex version retry the 503 without reporting model capacity; provider overload errors retain their separate mapping. -Restart gives active requests and pre-existing scoped drains up to two seconds from acceptance, -including the response-flush delay. The returned `drainTimeoutMs` describes this grace, not total -restart time. Cleanup keeps its separate 60-second watchdog, and replacement readiness can take up -to another 70 seconds. A request that finishes during grace is preserved; a remaining request is -interrupted and may already have executed upstream. OpenCodex does not automatically replay it. -Check its outcome before resubmitting: a healthy replacement does not prove a retry is safe. -The shorter grace reduces time spent refusing new work, but does not guarantee that every client -can reconnect within its retry window. Let important active work finish before requesting restart. - | Method and path | Purpose | Notable errors | | --- | --- | --- | | `GET /api/system/memory` | Return scalar process, heap, stream, response-state, watchdog, and active-turn metrics. Response-state diagnostics include spill-write status, consecutive failures, fixed privacy-safe failure class, and last failure/success timestamps. `spillLastWriteFailureOrigin` is `retry_returned_timeout`, `timeout_memo_refusal`, or null; cumulative `spillAclRetryReturnedTimeouts` and `spillAclTimeoutMemoRefusals` count terminal failed publications. See [Windows spill diagnostics](/troubleshooting/windows-memory/) for process-local semantics. Raw errors and paths are never returned. | — | -| `POST /api/system/restart` | Begin a drain-aware process restart without removing client injection | Returns 202; repeated calls report the existing drain | +| `POST /api/system/restart` | Begin a drain-aware process restart without removing client injection; optional JSON `drainGraceMs` explicitly shortens the default 60s grace | Returns 202; repeated calls report the existing grace; 400 invalid body or grace | | `POST /api/stop` | Stop the service, restore native Codex, remove managed Grok injection, and drain the proxy | 409 service ownership conflict; 409 `respawnable_service` when a Windows Task Scheduler wrapper could respawn the proxy and the caller is not `ocx stop` (nothing is changed); 409 `self_unload_service` when this proxy is running as the installed launchd/systemd service, because stopping the manager from inside it would end the process before native Codex is restored — run `ocx stop` instead (nothing is changed); 409 when the installed manager refuses to stop; 409 `service_state_unknown` when the Task Scheduler state cannot be read (nothing is changed; repair the query and retry) | | `GET /api/system/codex-app-server` | Report whether running Codex app-servers predate the current model catalog | — | | `POST /api/system/codex-restart` | Refresh the catalog, then restart stale Codex app-servers and fully quit and relaunch the Codex desktop app so the model picker reloads. When the proxy itself is running inside the Codex app, the desktop restart is refused rather than handed off. | Returns 200 with `code: partially_stopped` when a target survives | diff --git a/docs-site/src/content/docs/ru/troubleshooting/windows-memory.md b/docs-site/src/content/docs/ru/troubleshooting/windows-memory.md index 2189c1c5491..7dd47d8da15 100644 --- a/docs-site/src/content/docs/ru/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/ru/troubleshooting/windows-memory.md @@ -48,10 +48,10 @@ opencodex поставляет рантайм Bun (сейчас это **1.3.14* токенов, путей и идентификаторов аккаунтов — и чтение не имеет побочных эффектов: ничего не prune'ится и не evict'ится. Карточка **Memory observability** в дашборде показывает те же поля и даёт confirm-gated действие **Drain & restart**: она показывает текущее число активных ходов, - ждёт до 2 секунд с момента принятия перезапуска, пока они закончатся (используя уже существующий drain через 503 + + ждёт до 60 секунд, пока они закончатся (используя уже существующий drain через 503 + `Retry-After`), затем прерывает оставшиеся ходы, просит аттестованный живой процесс заменить себя и проверяет другой PID на том же порту, не снимая внедрение - в Codex. Очистка и запуск имеют отдельные сроки. Прерванный запрос уже мог выполниться на стороне провайдера; автоматической повторной отправки нет. См. [сроки перезапуска и безопасность повторов](/reference/management-api/#system-lifecycle) (англ.). + в Codex. Это более длинный и осознанный recycle, чем короткий drain у `POST /api/stop`. - **Альтернативный stream path под флагом** — bounded single-reader relay, полностью убирающий форму неограниченной буферизации. На Windows он станет значением по умолчанию автоматически, когда в комплектный Bun *подтверждённо* войдёт исправление #32111; сегодня это только opt-in diff --git a/docs-site/src/content/docs/tr/troubleshooting/windows-memory.md b/docs-site/src/content/docs/tr/troubleshooting/windows-memory.md index d8094384c91..dee7fcdc31d 100644 --- a/docs-site/src/content/docs/tr/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/tr/troubleshooting/windows-memory.md @@ -56,13 +56,14 @@ Sınırlı azaltma ve görünürlük — **bir düzeltme değil**. Paketlenmiş tanımlayıcıları yoktur — ve okuma yan etkisizdir (asla budamaz veya çıkarmaz). Kontrol panelinin **Bellek gözlemlenebilirliği** kartı aynı alanları işler ve onay korumalı bir **Boşalt ve yeniden başlat (Drain & restart)** eylemi sunar: - geçerli aktif tur sayısını gösterir, kabulden itibaren aktif turlar için 2 saniyeye kadar + geçerli aktif tur sayısını gösterir, aktif turlar için 60 saniyeye kadar bekler (mevcut 503 + `Retry-After` boşaltmasını yeniden kullanarak), ardından kalan turları iptal eder. Çalışan proxy yeniden başlatma yetkilendirmesine ve boşaltma koordinasyonuna sahiptir, ardından çıkar; kurulu bir servis yöneticisi geçerli olduğunda yenisini başlatır. Eylem, yalnızca aynı portta kimliği doğrulanmış farklı bir süreç sağlıklı olduğunda, Codex enjeksiyonunu - kaldırmadan başarı bildirir. Temizleme ve başlatma için ayrı süreler vardır. Kesilen istek sağlayıcıda zaten çalışmış olabilir; otomatik olarak yeniden gönderilmez. [Yeniden başlatma süresi ve tekrar güvenliği](/reference/management-api/#system-lifecycle) (İngilizce). + kaldırmadan başarı bildirir. Bu, `POST /api/stop` üzerindeki kısa boşaltmadan + daha uzun, bilgilendirilmiş bir geri dönüşümdür. - **Geçitli bir alternatif akış yolu** — sınırsız arabellek şeklini tamamen ortadan kaldıran sınırlı bir tek okuyucu aktarımı. Windows'ta, paketlenmiş bir Bun sürümü #32111 düzeltmesini doğrulanabilir şekilde taşıdığında otomatik diff --git a/docs-site/src/content/docs/troubleshooting/windows-memory.md b/docs-site/src/content/docs/troubleshooting/windows-memory.md index f5cd087591e..e65b9c74cfe 100644 --- a/docs-site/src/content/docs/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/troubleshooting/windows-memory.md @@ -77,15 +77,14 @@ runtime the leak itself remains an upstream problem: signal. The dashboard's **Memory observability** card renders the memory and continuation-size fields from this endpoint and offers a confirm-gated **Drain & restart** action: it shows - the current active-turn count, allows up to 2s from acceptance for active turns (reusing + the current active-turn count, waits up to 60s for active turns (reusing the existing HTTP 503 JSON `server_restarting` + `Retry-After: 5` drain), - then aborts any remaining turns. Cleanup and replacement startup have separate, longer - budgets. Interrupted work may already have executed upstream and is never automatically - replayed; see [restart timing and retry safety](/reference/management-api/#system-lifecycle). + then aborts any remaining turns. The running proxy owns restart authorization and drain coordination, then exits; an installed service manager launches the replacement when applicable. The action reports success only after a different, identity-verified process - is healthy on the same port, without tearing down Codex injection. + is healthy on the same port, without tearing down Codex injection. That is a + longer, informed recycle than the short drain on `POST /api/stop`. For a scriptable snapshot of the complete authenticated payload, run `ocx observe memory --json`; the CLI forwards the same response-state fields. diff --git a/docs-site/src/content/docs/zh-cn/troubleshooting/windows-memory.md b/docs-site/src/content/docs/zh-cn/troubleshooting/windows-memory.md index 07fa9f5fd53..7734ebe23da 100644 --- a/docs-site/src/content/docs/zh-cn/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/zh-cn/troubleshooting/windows-memory.md @@ -23,7 +23,7 @@ opencodex 打包了 Bun 运行时（当前为 **1.3.14**）。这类内存增长 - **内存监视器** — 代理每分钟采样一次自身内存，并在观测到的内存超过 4 GiB 时记录限频告警。观测到的内存取 RSS、`external` 和 `arrayBuffers` 三者中的最大值（不是它们的总和），因为 Windows 的工作集/RSS 计数可能低报已提交的外部保留。 - **`ocx doctor`** — `"Memory / runtime"` 部分会显示*服务*进程的 Bun 版本、RSS、external/ArrayBuffers 计数、JS 堆上下文以及流模式决策。在捆绑的 Bun 1.3.14 运行时上，单看 `heapUsed` / `jscHeap` 不能作为泄漏判据；在认定为应用层泄漏之前，应把观测到的内存与 `responseState` 以及多次采样一起比较。 -- **`GET /api/system/memory`** — 通过已认证的管理 API 提供同样的数据，便于仪表板或脚本使用。除了 RSS/heap/external 计数之外，它还会报告一个标量的 `responseState` 块（条目数、序列化总字节数/最大字节数、最老条目的年龄），对应代理内存中的 `previous_response_id` 续接存储。这能进一步归因增长：在观测到的内存上升时，如果 `responseState.totalBytes` 也在上升，说明是对话保留在增长（较长的 `store:false` 链在每轮中重新扩张）；而在观测到的内存上升时，如果 `responseState` 保持平稳，则更像不是这个存储造成的。返回值只包含标量，不包含请求正文、token、路径或账户标识，而且读取没有副作用（不会执行 prune，也不会 evict）。仪表板中的 **Memory observability** 卡片会渲染相同字段，并提供一个需要确认的 **Drain & restart** 操作：它会显示当前活动轮次数量，从接受重启起最多等待 2 秒让活动轮次结束（复用现有的 503 + `Retry-After` 排空机制），然后中止剩余轮次。运行中的代理负责重启授权和排空协调并退出；如果由服务管理器托管，则由已安装的服务管理器启动替换进程。只有确认同一端口上另一个经过身份验证且健康的进程后才会报告成功，同时不会拆除 Codex 注入。清理和替换进程启动使用各自的时限。中断的请求可能已在上游执行，不会自动重放；请参阅[重启时限与重试安全](/reference/management-api/#system-lifecycle)（英文）。 +- **`GET /api/system/memory`** — 通过已认证的管理 API 提供同样的数据，便于仪表板或脚本使用。除了 RSS/heap/external 计数之外，它还会报告一个标量的 `responseState` 块（条目数、序列化总字节数/最大字节数、最老条目的年龄），对应代理内存中的 `previous_response_id` 续接存储。这能进一步归因增长：在观测到的内存上升时，如果 `responseState.totalBytes` 也在上升，说明是对话保留在增长（较长的 `store:false` 链在每轮中重新扩张）；而在观测到的内存上升时，如果 `responseState` 保持平稳，则更像不是这个存储造成的。返回值只包含标量，不包含请求正文、token、路径或账户标识，而且读取没有副作用（不会执行 prune，也不会 evict）。仪表板中的 **Memory observability** 卡片会渲染相同字段，并提供一个需要确认的 **Drain & restart** 操作：它会显示当前活动轮次数量，最多等待 60 秒让活动轮次结束（复用现有的 503 + `Retry-After` 排空机制），然后中止剩余轮次。运行中的代理负责重启授权和排空协调并退出；如果由服务管理器托管，则由已安装的服务管理器启动替换进程。只有确认同一端口上另一个经过身份验证且健康的进程后才会报告成功，同时不会拆除 Codex 注入。这是一种比 `POST /api/stop` 的短排空更长、更知情的回收方式。 - **有门控的替代流路径** — 一种有界的单读者中继，能彻底消除无界缓冲的形态。在 Windows 上，一旦某个捆绑的 Bun 版本可验证地包含 #32111 的修复，它就会自动成为默认路径；目前它仅支持显式启用（见下文）。在 macOS 上，即使到了那样的版本，它也仍然保持显式启用状态；切换 macOS 的 `auto` 是另一项独立决定。 这些改动带来的真实世界 RSS 改善，仍在等待 Windows 用户验证，我们并不宣称泄漏已经修复。 diff --git a/gui/src/components/MemoryObservabilityCard.tsx b/gui/src/components/MemoryObservabilityCard.tsx index ca079e7d1cd..8bea5857ca1 100644 --- a/gui/src/components/MemoryObservabilityCard.tsx +++ b/gui/src/components/MemoryObservabilityCard.tsx @@ -1,5 +1,4 @@ import { useEffect, useState } from "react"; -import { RESTART_DRAIN_GRACE_MS } from "../../../src/lib/system-restart-timing"; import { formatUptime } from "../formatUptime"; import { IconActivity } from "../icons"; import { useI18n, type Locale, type TFn } from "../i18n/shared"; @@ -10,8 +9,8 @@ import { startVisibilityPoll } from "../visibility-poll"; /** * Memory observability card. Polls GET /api/system/memory (#314 WP3) every 5s * and renders scalar diagnostics. Also hosts the confirm-gated Drain & restart - * action (#563): short in-flight grace, then restart with a separate cleanup - * budget, preserving client injection rather than the /api/stop teardown. + * action (#563): longer 60s drain, then respawn via ensure/service — not the + * short /api/stop teardown path. */ interface MemorySample { @@ -199,7 +198,7 @@ function MemoryPressure({ ); } -const DRAIN_TIMEOUT_S = RESTART_DRAIN_GRACE_MS / 1_000; +const DRAIN_TIMEOUT_S = 60; const RECONNECT_POLL_MS = 1500; const RECONNECT_GIVE_UP_MS = 120_000; diff --git a/gui/src/remote-link-api.ts b/gui/src/remote-link-api.ts index 13f96f9a15b..9dec2e1cea6 100644 --- a/gui/src/remote-link-api.ts +++ b/gui/src/remote-link-api.ts @@ -170,7 +170,7 @@ export const CHILD_RESTART_POLL_MS = 1_000; export const CHILD_RESTART_SLOW_POLL_MS = 5_000; /** * The server's handoff budget plus a margin, the window `ocx restart` also observes: up to 60 s of - * terminal cleanup (`MEMORY_DRAIN_RESTART_MS`), then up to 70 s for the replacement to answer + * drain (`MEMORY_DRAIN_RESTART_MS`), then up to 70 s for the replacement to answer * (`REPLACEMENT_READY_TIMEOUT_MS`), plus 15 s. Past it the page says the restart is slow and keeps * checking, so a Child that comes up late still reloads the page. */ diff --git a/gui/tests/memory-observability-card.test.tsx b/gui/tests/memory-observability-card.test.tsx index 83137fe9f68..6c134c161ef 100644 --- a/gui/tests/memory-observability-card.test.tsx +++ b/gui/tests/memory-observability-card.test.tsx @@ -158,7 +158,7 @@ test("Drain & restart posts /api/system/restart only after the in-page dialog is return Response.json({ success: true, activeTurnCount: 2, - drainTimeoutMs: 2_000, + drainTimeoutMs: 60_000, alreadyDraining: false, }, { status: 202 }); } @@ -178,7 +178,6 @@ test("Drain & restart posts /api/system/restart only after the in-page dialog is // The click alone must not restart anything: the consent is a real dialog now, and // inside the app the platform one drew nothing, so nothing could answer it. expect(actionDialogOpen(testWindow.document as unknown as Document)).toBe(true); - expect(testWindow.document.body.textContent).toContain("up to 2s; remaining requests are cut on timeout"); expect(restartPosts).toBe(0); await act(async () => { diff --git a/src/lib/system-restart-contract.ts b/src/lib/system-restart-contract.ts index ba7fec0c271..963b721abba 100644 --- a/src/lib/system-restart-contract.ts +++ b/src/lib/system-restart-contract.ts @@ -8,7 +8,9 @@ export const SYSTEM_RESTART_EXPECTED_PID_HEADER = "x-opencodex-restart-expected- export const SYSTEM_RESTART_NONCE_HEADER = "x-opencodex-restart-nonce"; export const SYSTEM_RESTART_CAPABILITY_HEADER = "x-opencodex-restart-capability"; -export { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS } from "./system-restart-timing"; +/** Fixed drain and replacement budgets shared by the server, CLI, and tray. */ +export const MEMORY_DRAIN_RESTART_MS = 60_000; +export const REPLACEMENT_READY_TIMEOUT_MS = 70_000; /** * The env var a restarting process hands its replacement `ocx start`: the restarting (parent) pid. diff --git a/src/lib/system-restart-timing.ts b/src/lib/system-restart-timing.ts deleted file mode 100644 index 9eba6fb7004..00000000000 --- a/src/lib/system-restart-timing.ts +++ /dev/null @@ -1,6 +0,0 @@ -/** Active turns and pre-existing scoped drains get this grace from restart acceptance. */ -export const RESTART_DRAIN_GRACE_MS = 2_000; -/** Terminal cleanup watchdog; also part of the CLI/tray restart observation budget. */ -export const MEMORY_DRAIN_RESTART_MS = 60_000; -/** Replacement startup has its own budget; short drain grace does not guarantee short downtime. */ -export const REPLACEMENT_READY_TIMEOUT_MS = 70_000; diff --git a/src/server/management/system-restart.ts b/src/server/management/system-restart.ts index 655970043dd..780fbc94e3f 100644 --- a/src/server/management/system-restart.ts +++ b/src/server/management/system-restart.ts @@ -1,8 +1,8 @@ /** * Dashboard memory-card drain-and-restart (#563). * - * Gives active turns/scoped drains a short grace, with a separate 60s cleanup - * watchdog, then respawns. Never runs restoreNativeCodex / stripGrokConfig — this is a + * Waits up to 60s for active turns unless a manual API caller explicitly opts + * into a shorter grace, with the existing 60s cleanup watchdog, then respawns. Never runs restoreNativeCodex / stripGrokConfig — this is a * recycle to reclaim RSS, not a teardown. * * Respawn policy (matches real supervisor configs in src/service.ts): @@ -59,7 +59,6 @@ import { MEMORY_DRAIN_RESTART_MS, isDesktopSupervised, } from "../../lib/system-restart-contract"; -import { RESTART_DRAIN_GRACE_MS } from "../../lib/system-restart-timing"; import { spawnReplacementStart } from "../restart-replacement"; export { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS } from "../../lib/system-restart-contract"; @@ -93,6 +92,11 @@ export interface SystemRestartIo { now?: () => number; } +export interface SystemRestartOptions { + /** Explicit API opt-in; omission preserves the existing 60s drain. */ + drainGraceMs?: number; +} + export interface SystemRestartAdmission { /** Only the caller that created this pending restart receives its veto. */ onAccepted?: (veto: () => void) => void; @@ -103,6 +107,7 @@ export interface SystemRestartAdmission { let restartIo: SystemRestartIo = {}; /** Prevents double-scheduling in the 200ms window before drainAndShutdown sets draining. */ let restartAccepted = false; +let acceptedDrainGraceMs = MEMORY_DRAIN_RESTART_MS; type RestartDrainOutcome = "completed" | "failed" | "rejected" | "deadline"; type BoundedSettlementOutcome = "completed" | "rejected" | "deadline"; @@ -174,6 +179,7 @@ export function noteExplicitShutdownRequested(): void { export function setSystemRestartIoForTests(io: SystemRestartIo = {}): void { restartIo = io; restartAccepted = false; + acceptedDrainGraceMs = MEMORY_DRAIN_RESTART_MS; explicitShutdownRequested = false; } @@ -328,7 +334,7 @@ async function completeDeadlineRestartHandoff( * respawn runs on a short timer so the HTTP response can flush first. * Idempotent while already draining: returns the accepted shape again. */ -export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: SystemRestartAdmission = {}): { +export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: SystemRestartAdmission = {}, options: SystemRestartOptions = {}): { accepted: true; alreadyDraining: boolean; activeTurnCount: number; @@ -338,11 +344,15 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: ?? io.isDraining ?? isShutdownDraining; const alreadyDraining = restartAccepted || shutdownActive(); + const drainGraceMs = alreadyDraining + ? (restartAccepted ? acceptedDrainGraceMs : MEMORY_DRAIN_RESTART_MS) + : (options.drainGraceMs ?? MEMORY_DRAIN_RESTART_MS); const activeTurnCount = (io.getActiveTurnCount ?? getActiveTurnCount)(); const schedule = io.schedule ?? ((fn, ms) => { setTimeout(() => { void fn(); }, ms); }); if (!alreadyDraining) { restartAccepted = true; + acceptedDrainGraceMs = drainGraceMs; const automatic = Boolean(admission.onAccepted); const temporaryDrain = automatic ? (io.acquireTemporaryDrain ?? (() => acquireTemporaryDrain("automatic-restart")))() @@ -353,7 +363,7 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: }; if (automatic && !temporaryDrain) { restartAccepted = false; - return { accepted: true, alreadyDraining: true, activeTurnCount, drainTimeoutMs: RESTART_DRAIN_GRACE_MS }; + return { accepted: true, alreadyDraining: true, activeTurnCount, drainTimeoutMs: MEMORY_DRAIN_RESTART_MS }; } let pending = true; let vetoed = false; @@ -366,7 +376,7 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: const now = io.now ?? Date.now; const acceptedAtMs = now(); const restartDeadlineMs = acceptedAtMs + MEMORY_DRAIN_RESTART_MS; - const drainDeadlineMs = acceptedAtMs + RESTART_DRAIN_GRACE_MS; + const drainDeadlineMs = acceptedAtMs + drainGraceMs; // Reject new data-plane traffic immediately (503), before the 200ms response-flush delay. if (!automatic) { if (io.beginShutdownDrain) io.beginShutdownDrain(); @@ -464,6 +474,6 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: accepted: true, alreadyDraining, activeTurnCount, - drainTimeoutMs: RESTART_DRAIN_GRACE_MS, + drainTimeoutMs: drainGraceMs, }; } diff --git a/src/server/management/system-routes.ts b/src/server/management/system-routes.ts index 455432115ff..7ee0376aa72 100644 --- a/src/server/management/system-routes.ts +++ b/src/server/management/system-routes.ts @@ -30,6 +30,7 @@ import { inspectResponseSpillStorage, responseStateMetrics, type ResponseSpillDi import { appOwnedBytesSnapshot } from "../../lib/app-owned-memory"; import { readWindowsReplaceRetryCounters } from "../../lib/windows-atomic-replace"; import { + MEMORY_DRAIN_RESTART_MS, SYSTEM_RESTART_EXPECTED_PID_HEADER, parseExpectedSystemRestartPid, } from "../../lib/system-restart-contract"; @@ -46,6 +47,7 @@ import type { } from "../../codex/app-server-restart-service"; import type { ManagementContext } from "./context"; import { acceptSystemRestart } from "./system-restart"; +import { readOptionalManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; const ENDPOINT_SAMPLE_LIMIT = 60; // The spill report reads the snapshot and walks the spill directory synchronously, @@ -179,8 +181,28 @@ export async function handleSystemRoutes(ctx: ManagementContext): Promise).drainGraceMs; + if (drainGraceMs !== undefined && ( + typeof drainGraceMs !== "number" || !Number.isSafeInteger(drainGraceMs) + || drainGraceMs < 1 || drainGraceMs > MEMORY_DRAIN_RESTART_MS + )) { + return jsonResponse({ success: false, error: "drainGraceMs must be an integer from 1 to 60000." }, 400, req, config); + } + // The process capability signs the restart target, not this new body option. + if (drainGraceMs !== undefined && ctx.principal === "system-restart-capability") { + return jsonResponse({ success: false, error: "Short grace requires management authentication." }, 403, req, config); + } + // Omission retains the informed 60s drain; only an explicit API request shortens it. + const result = acceptSystemRestart(undefined, {}, { drainGraceMs }); return jsonResponse({ success: true, message: result.alreadyDraining diff --git a/src/tray/windows-tray.ps1 b/src/tray/windows-tray.ps1 index 0b0cde354a3..3112ee6fdc9 100644 --- a/src/tray/windows-tray.ps1 +++ b/src/tray/windows-tray.ps1 @@ -656,7 +656,7 @@ $stopItem.add_Click({ $restartItem.add_Click({ if (-not (Set-PendingAction "Restart Proxy" 160)) { return } $statusItem.Text = (Get-TrayText "Proxy: Restarting..." "代理: 重启中...") - # /api/system/restart may await terminal cleanup for 60s, then spend up to 70s + # /api/system/restart may drain active work for 60s and then spend up to 70s # handing off to an identity-verified replacement. The tray observes health/PID # rather than the detached CLI exit, so keep a watchdog margin around that shared # lifecycle budget. The CLI remains the lifecycle owner; the tray never kills. diff --git a/structure/ops/service-and-sidecars.md b/structure/ops/service-and-sidecars.md index fd8fdd1dec6..acde839bc93 100644 --- a/structure/ops/service-and-sidecars.md +++ b/structure/ops/service-and-sidecars.md @@ -466,17 +466,14 @@ clients do not retry it at all. `server_restarting` falls through to retryable response bypasses the shared provider-overload mapping in `src/lib/errors.ts`. `tests/codex-integration/issue-452-empty-503.test.ts` pins its body and both listeners' CORS. -Restart acceptance immediately fences new work, but active turns and pre-existing scoped drains -have a two-second grace measured from acceptance, including the 200ms response-flush delay. -`src/lib/system-restart-timing.ts` keeps that grace separate from the unchanged 60-second terminal -cleanup watchdog and 70-second replacement-readiness budget. `drainTimeoutMs` in the management -response and the dashboard confirmation report the two-second grace. Completed turns are not -resent; a still-active turn is aborted when grace expires, even when its upstream execution outcome -is unknown. Successful cleanup or replacement health does not establish that outcome, and restart -never authorizes automatic replay. This trades in-flight completion time for a shorter admission -fence; cleanup and replacement startup can still outlast a client's retries. The old process never -reopens admission. Synthetic-clock coverage with the real lifecycle lives in -`tests/server/system-restart-admission.test.ts`. +`src/server/management/system-routes.ts` accepts an optional JSON `drainGraceMs` on the authenticated +restart API: an integer from 1 to 60000, with omission retaining the 60-second default. Invalid +input starts no drain. `src/server/management/system-restart.ts` fixes the selected grace at first +acceptance, including the response-flush delay; repeated requests cannot shorten it. The cleanup +watchdog stays 60 seconds and replacement readiness stays 70 seconds. Short grace is an explicit +API opt-in; dashboard, CLI, tray, join and automatic restarts keep their default. An interrupted +turn may have an unknown upstream outcome and is not automatically replayed. +`tests/server/system-restart-admission.test.ts` covers this contract and active-turn preservation. A dashboard drain-and-restart (`src/server/management/system-restart.ts`, which is also the restart after a join into a Child) and the client runtime's standalone recycle (`src/client/runtime.ts`) diff --git a/structure/remote-link.md b/structure/remote-link.md index efc58342c6d..181bda3364f 100644 --- a/structure/remote-link.md +++ b/structure/remote-link.md @@ -38,7 +38,7 @@ The client tunnel pidfile is `/link/client-tunnel.pid` with `{ versio The dashboard link routes (`GET /api/link/status`, `GET /api/link/candidates`, `POST /api/link/probe`, `POST /api/link/confirm-host`, `POST /api/link/apply` and `DELETE /api/link/{id}`) admit a paired GUI session, or, on a standalone runtime only, the current loopback-issued GUI session that reached the public listener bound to a loopback hostname. `POST /api/link/join` is stricter and requires the paired session because it durably redirects this machine's client traffic. The hub-link, hub-management and claude-intercept ingresses are never trusted loopback ingress, a stale session is refused, and a hub keeps the paired-only rule. The Tailscale identity refusal runs before either check. A loopback session is minted by the loopback dashboard bootstrap without a credential, so it proves possession, not user presence: any local process can fetch the bootstrap and replay its token and CSRF value, which is why it cannot authorize joining and why `src/client/machine-listener.ts` keeps durable machine changes on the connected listener away from that session. The remaining link routes accept this casual-path trade, the same as `POST /api/github/star` in `src/server/management/sidebar-routes.ts`; it is not a secret-backed boundary like an operator-created pairing grant or the admin token. -A successful join restarts this proxy (a 503 fence with a two-second active-turn grace, followed by cleanup and a closed listener; see [restart handoff](ops/service-and-sidecars.md#restart-handoff)) into the client runtime on the configured port. `GET /api/link/status` tells a GUI-session caller whether it may join: the response gains `joinAvailable`, true only for a paired dashboard session on a standalone runtime whose live port is its configured port, and `joinDenied`, which is `null` when allowed or the first failed gate (`pairing_required`, `standalone_required`, then `join_port_mismatch`). An admin-token caller gets the exact K16 document without either field, because `ocx link status` validates it key by key. The dashboard reads an absent availability field as false and an absent or unknown denial as generic unavailability; while unavailable the Child role card cannot be selected by pointer or keyboard. A pairing refusal says to pair this machine first and offers code entry only on the existing same-origin literal-loopback transport. A port refusal instead asks the operator to restart on the configured port; neither status reads nor displaying this guidance mint a grant or change pairing, keys, routing, or runtime state. Before **Connect as Child** the confirmation panel says that the restart briefly interrupts Codex and that Codex keeps its local address. The dashboard reads the standalone's pid from same-origin `/healthz`, sends the join, then reads `/healthz` once a second, skipping status polls meanwhile, and reloads only when it reports `role: "client"` under another pid, so the reloaded document carries the client role and a fresh session. It never gives up by itself: past the server's own handoff budget (60 s cleanup watchdog plus 70 s replacement readiness, plus a margin: 145 seconds) it says the restart is slow and keeps reading `/healthz` every 5 seconds until the Child answers or the page is left. +A successful join restarts this proxy (a 503 drain of up to a minute while running turns finish, then a closed listener) into the client runtime on the configured port. `GET /api/link/status` tells a GUI-session caller whether it may join: the response gains `joinAvailable`, true only for a paired dashboard session on a standalone runtime whose live port is its configured port, and `joinDenied`, which is `null` when allowed or the first failed gate (`pairing_required`, `standalone_required`, then `join_port_mismatch`). An admin-token caller gets the exact K16 document without either field, because `ocx link status` validates it key by key. The dashboard reads an absent availability field as false and an absent or unknown denial as generic unavailability; while unavailable the Child role card cannot be selected by pointer or keyboard. A pairing refusal says to pair this machine first and offers code entry only on the existing same-origin literal-loopback transport. A port refusal instead asks the operator to restart on the configured port; neither status reads nor displaying this guidance mint a grant or change pairing, keys, routing, or runtime state. Before **Connect as Child** the confirmation panel says that the restart briefly interrupts Codex and that Codex keeps its local address. The dashboard reads the standalone's pid from same-origin `/healthz`, sends the join, then reads `/healthz` once a second, skipping status polls meanwhile, and reloads only when it reports `role: "client"` under another pid, so the reloaded document carries the client role and a fresh session. It never gives up by itself: past the server's own handoff budget (60 s drain plus 70 s replacement readiness, plus a margin: 145 seconds) it says the restart is slow and keeps reading `/healthz` every 5 seconds until the Child answers or the page is left. A connected Child answers `GET` and `HEAD /api/link/status` on its own listener for a GUI session: `src/client/link-status.ts` projects the client sidecar and the tunnel supervisor into the K16 document with `role: "child"`, the listener off, no links and the child row, plus `joinAvailable: false`. A Home-initiated Child has no sidecar and reports `child: null`. In link mode `/api/machine/status` advertises the machine origin as the shared plane, because the tunnel's hub-link ingress serves no `/api/*` and no session bootstrap. diff --git a/structure/runtime.md b/structure/runtime.md index 3e67f412435..d3eac545039 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -193,7 +193,7 @@ until shutdown. Normal shutdown restores native Codex; a sibling instance beside `OCX_SERVICE=1`, so managed restarts do not repeatedly restore/reinject; explicit service stop and uninstall still restore. `src/service/cli.ts` removes the service token on uninstall only when persisted client state is disconnected and no pending connect marker owns the newly issued key. `src/client/connect.ts` publishes that fingerprint marker before the key, then clears it with the connection commit or rollback under the client lifecycle and config mutation locks. Connected, invalid, or mismatched client state retains an existing token. A valid pending marker retains only its matching fingerprint; an older marker does not own a replacement service key. An absent token is reported as absent; unsafe, malformed, or unreadable markers and lock, state-read, or deletion failures leave cleanup unverified. `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart timing and replacement ownership follow the [restart handoff contract](ops/service-and-sidecars.md#restart-handoff). +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/server/system-restart-admission.test.ts b/tests/server/system-restart-admission.test.ts index d3529714517..72c0f037a80 100644 --- a/tests/server/system-restart-admission.test.ts +++ b/tests/server/system-restart-admission.test.ts @@ -1,4 +1,4 @@ -/** #6643: active-turn grace must not consume the whole restart cleanup budget. */ +/** #6643: an explicit short grace preserves the default drain and cleanup budget. */ import { afterEach, expect, spyOn, test } from "bun:test"; import { acceptSystemRestart, @@ -17,9 +17,94 @@ import { stopServerListener, tryAdmitTurn, } from "../../src/server/lifecycle"; +import { handleManagementAPI } from "../../src/server/management-api"; +import { handleSystemRoutes } from "../../src/server/management/system-routes"; +import { ManagementRequest } from "../helpers/management-auth"; +import type { OcxConfig } from "../../src/types"; afterEach(() => setSystemRestartIoForTests()); +const apiConfig: OcxConfig = { + port: 10100, + defaultProvider: "openai", + providers: { openai: { adapter: "openai-chat", baseUrl: "https://api.example.test/v1", apiKey: "fixture", defaultModel: "fixture" } }, +}; + +function apiRequest(body?: string) { + return new ManagementRequest("http://127.0.0.1:10100/api/system/restart", { + method: "POST", body, headers: body === undefined ? undefined : { "Content-Type": "application/json" }, + }); +} + +test("empty API requests and empty objects retain the 60-second default", async () => { + for (const payload of [undefined, "{}"]) { + setSystemRestartIoForTests({ isDraining: () => false, schedule: () => {}, setDraining: () => {} }); + const req = apiRequest(payload); + const res = await handleManagementAPI(req, new URL(req.url), apiConfig); + expect(res?.status).toBe(202); + expect((await res!.json()).drainTimeoutMs).toBe(60_000); + } +}); + +test("the API explicitly opts into short grace and repeated requests retain the accepted grace", async () => { + let scheduled = 0; + setSystemRestartIoForTests({ isDraining: () => false, schedule: () => { scheduled += 1; }, setDraining: () => {} }); + for (const payload of ['{"drainGraceMs":2000}', undefined, '{"drainGraceMs":1000}']) { + const req = apiRequest(payload); + const res = await handleManagementAPI(req, new URL(req.url), apiConfig); + expect(res?.status).toBe(202); + expect((await res!.json()).drainTimeoutMs).toBe(2_000); + } + expect(scheduled).toBe(1); +}); + +test("a repeated API opt-in cannot shorten an existing default restart", async () => { + setSystemRestartIoForTests({ isDraining: () => false, schedule: () => {}, setDraining: () => {} }); + for (const payload of [undefined, '{"drainGraceMs":2000}']) { + const req = apiRequest(payload); + const res = await handleManagementAPI(req, new URL(req.url), apiConfig); + expect((await res!.json()).drainTimeoutMs).toBe(60_000); + } +}); + +test("invalid API grace or malformed JSON is rejected without beginning a drain", async () => { + let scheduled = 0; + let drains = 0; + setSystemRestartIoForTests({ isDraining: () => false, schedule: () => { scheduled += 1; }, setDraining: () => { drains += 1; } }); + const invalid = [0, -1, 60_001, 1.5, null, "2000", true]; + for (const payload of [...invalid.map(drainGraceMs => JSON.stringify({ drainGraceMs })), "{", "null", "[]", '"2000"']) { + const req = apiRequest(payload); + const res = await handleManagementAPI(req, new URL(req.url), apiConfig); + expect(res?.status).toBe(400); + } + expect(scheduled).toBe(0); + expect(drains).toBe(0); +}); + +test("the API accepts both supported grace boundaries", async () => { + for (const drainGraceMs of [1, 60_000]) { + setSystemRestartIoForTests({ isDraining: () => false, schedule: () => {}, setDraining: () => {} }); + const req = apiRequest(JSON.stringify({ drainGraceMs })); + const res = await handleManagementAPI(req, new URL(req.url), apiConfig); + expect(res?.status).toBe(202); + expect((await res!.json()).drainTimeoutMs).toBe(drainGraceMs); + } +}); + +test("a target-bound restart capability cannot opt into an unsigned grace option", async () => { + let scheduled = 0; + setSystemRestartIoForTests({ isDraining: () => false, schedule: () => { scheduled += 1; }, setDraining: () => {} }); + const req = apiRequest('{"drainGraceMs":2000}'); + const res = await handleSystemRoutes({ req, url: new URL(req.url), config: apiConfig, deps: {}, version: "fixture", principal: "system-restart-capability" }); + expect(res?.status).toBe(403); + expect(scheduled).toBe(0); + const defaultReq = apiRequest(); + const accepted = await handleSystemRoutes({ req: defaultReq, url: new URL(defaultReq.url), config: apiConfig, deps: {}, version: "fixture", principal: "system-restart-capability" }); + expect(accepted?.status).toBe(202); + expect((await accepted!.json()).drainTimeoutMs).toBe(60_000); + expect(scheduled).toBe(1); +}); + test("restart spends at most two seconds on active turns, including the response-flush delay", async () => { let now = 10_000; let scheduled!: () => void | Promise; @@ -40,7 +125,7 @@ test("restart spends at most two seconds on active turns, including the response spawnStart: () => {}, markRecycling: () => {}, exitProcess: () => {}, - }); + }, {}, { drainGraceMs: 2_000 }); now += 200; await scheduled(); expect(turnWaitMs).toBe(1_800); @@ -83,7 +168,7 @@ afterEach(() => { resetLifecycleDrainStateForTests(); }); -function restartFixture(options: { automatic?: boolean; holdCleanup?: boolean; holdReadiness?: boolean; failSpawn?: boolean; flushDelayMs?: number } = {}) { +function restartFixture(options: { automatic?: boolean; holdCleanup?: boolean; holdReadiness?: boolean; failSpawn?: boolean; flushDelayMs?: number; useDefaultGrace?: boolean } = {}) { const clock = new RestartClock(); const date = spyOn(Date, "now").mockImplementation(() => clock.now); const sleep = spyOn(Bun, "sleep").mockImplementation((ms) => new Promise(resolve => { @@ -123,7 +208,7 @@ function restartFixture(options: { automatic?: boolean; holdCleanup?: boolean; h markRecycling: () => { calls.push("recycle"); }, isClientConnected: () => false, exitProcess: code => { calls.push(`exit:${code}`); }, - }, options.automatic ? { onAccepted: callback => { veto = callback; } } : {}); + }, options.automatic ? { onAccepted: callback => { veto = callback; } } : {}, options.useDefaultGrace ? {} : { drainGraceMs: 2_000 }); return { clock, calls, accept, finishCleanup, finishReadiness, veto: () => veto(), running: () => running }; } @@ -138,6 +223,19 @@ function upstreamTurn(bindController = true) { for (const automatic of [false, true]) { const mode = automatic ? "automatic" : "manual"; + test(`${mode} default restart preserves active work beyond two seconds`, async () => { + const fixture = restartFixture({ automatic, useDefaultGrace: true }); + const turn = upstreamTurn(); + expect(fixture.accept().drainTimeoutMs).toBe(60_000); + await fixture.clock.advance(2_000); + expect(turn.controller.signal.aborted).toBe(false); + expect(getActiveTurnCount()).toBe(1); + turn.lease.release(); + await fixture.clock.advance(100); + await fixture.running(); + expect(turn.controller.signal.aborted).toBe(false); + expect(fixture.calls).toContain("drain:59800"); + }); test(`${mode} restart preserves a completed turn and hands off without spending the grace`, async () => { const fixture = restartFixture({ automatic }); const turn = upstreamTurn(); @@ -288,7 +386,7 @@ test("waiting to enter the drain microtask does not renew the acceptance-based g spawnStart: () => {}, markRecycling: () => {}, exitProcess: () => {}, - }); + }, {}, { drainGraceMs: 2_000 }); now += 200; const running = scheduled(); now += 350; diff --git a/tests/server/system-restart.test.ts b/tests/server/system-restart.test.ts index ced98f461d0..1a7c67ff046 100644 --- a/tests/server/system-restart.test.ts +++ b/tests/server/system-restart.test.ts @@ -20,7 +20,6 @@ import { waitForReplacementReady, } from "../../src/server/management/system-restart"; import { SYSTEM_RESTART_EXPECTED_PID_HEADER } from "../../src/lib/system-restart-contract"; -import { RESTART_DRAIN_GRACE_MS } from "../../src/lib/system-restart-timing"; import type { OcxConfig } from "../../src/types"; function config(): OcxConfig { @@ -102,7 +101,7 @@ describe("acceptSystemRestart", () => { accepted: true, alreadyDraining: false, activeTurnCount: 3, - drainTimeoutMs: RESTART_DRAIN_GRACE_MS, + drainTimeoutMs: MEMORY_DRAIN_RESTART_MS, }); // Data-plane reject must arm before the 200ms flush delay runs. expect(calls).toEqual(["draining:true"]); @@ -110,11 +109,11 @@ describe("acceptSystemRestart", () => { expect(scheduled).not.toBeNull(); now += 15_000; await scheduled!(); - expect(calls).toEqual(["draining:true", "drain:0", "stop", "start:10123:ready", "recycle", "exit:0"]); + expect(calls).toEqual(["draining:true", "drain:45000", "stop", "start:10123:ready", "recycle", "exit:0"]); expect(deadlineDelay).toBe(45_000); expect(deadlineCancellations).toBe(1); deadlineCallback?.(); - expect(calls).toEqual(["draining:true", "drain:0", "stop", "start:10123:ready", "recycle", "exit:0"]); + expect(calls).toEqual(["draining:true", "drain:45000", "stop", "start:10123:ready", "recycle", "exit:0"]); }); test("snapshots the restart port before normal drain invalidates listener metadata", async () => { @@ -922,7 +921,7 @@ describe("POST /api/system/restart", () => { }; expect(body.success).toBe(true); expect(body.activeTurnCount).toBe(1); - expect(body.drainTimeoutMs).toBe(RESTART_DRAIN_GRACE_MS); + expect(body.drainTimeoutMs).toBe(60_000); expect(body.alreadyDraining).toBe(false); expect(body.message.toLowerCase()).toContain("drain"); }); From 54589dc86ff1d8945d3720ece0a66f48e9a8b30a Mon Sep 17 00:00:00 2001 From: BigHulk Date: Wed, 7 Oct 2026 23:18:30 +0800 Subject: [PATCH 3/5] docs: describe explicit restart grace in localized API pages Signed-off-by: BigHulk --- docs-site/src/content/docs/fr/reference/management-api.md | 2 ++ docs-site/src/content/docs/ja/reference/management-api.md | 2 ++ docs-site/src/content/docs/ko/reference/management-api.md | 2 ++ docs-site/src/content/docs/ru/reference/management-api.md | 2 ++ docs-site/src/content/docs/tr/reference/management-api.md | 2 ++ docs-site/src/content/docs/zh-cn/reference/management-api.md | 2 ++ docs-site/src/content/docs/zh-tw/reference/management-api.md | 2 ++ 7 files changed, 14 insertions(+) diff --git a/docs-site/src/content/docs/fr/reference/management-api.md b/docs-site/src/content/docs/fr/reference/management-api.md index 9e7a6ec106f..dd9a5d843e4 100644 --- a/docs-site/src/content/docs/fr/reference/management-api.md +++ b/docs-site/src/content/docs/fr/reference/management-api.md @@ -345,6 +345,8 @@ lui-même s'il souhaite ajouter une étoile au dépôt. ### Cycle de vie du système +`POST /api/system/restart` conserve par défaut un délai de 60 secondes lorsque le corps est absent ou vaut `{}`. Une session de gestion ou un jeton administrateur peut choisir explicitement un délai plus court avec `{"drainGraceMs":2000}` : la valeur doit être un entier compris entre 1 et 60000 millisecondes. Un corps ou une valeur invalide renvoie 400 sans lancer le redémarrage ; la capacité locale liée à la cible ne permet pas cette option (403). `drainTimeoutMs` indique le délai accepté, fixé par le premier appel et inchangé lors des appels suivants. Il inclut le temps de transmission de la réponse ; les budgets de nettoyage et de disponibilité restent respectivement de 60 et 70 secondes. Une requête interrompue peut déjà avoir été exécutée : vérifiez son résultat avant de la renvoyer, sans répétition automatique. + | Méthode et chemin | Objectif | Erreurs notables | | --- | --- | --- | | `GET /api/system/memory` | Renvoyer les mesures scalaires du processus, du tas, des flux, de l'état des réponses, du mécanisme de surveillance et des tours actifs | — | diff --git a/docs-site/src/content/docs/ja/reference/management-api.md b/docs-site/src/content/docs/ja/reference/management-api.md index 25eb7807eab..733c910157c 100644 --- a/docs-site/src/content/docs/ja/reference/management-api.md +++ b/docs-site/src/content/docs/ja/reference/management-api.md @@ -301,6 +301,8 @@ CLI コマンドは Anthropic OAuth アカウントを id または一意の別 ### システムのライフサイクル +`POST /api/system/restart` は、本文を省略した場合や `{}` の場合、既定の 60 秒のドレイン猶予を維持します。管理セッションまたは管理者トークンを使う呼び出し元は、`{"drainGraceMs":2000}` で短い猶予を明示的に選択できます。値は 1～60000 ミリ秒の整数です。不正な本文や値は再起動を開始せずに 400 を返します。対象プロセスに限定されたローカル再起動 capability では、このオプションを設定できません（403）。応答の `drainTimeoutMs` は最初に受け入れた猶予を示し、繰り返しの呼び出しでも期限は変わりません。猶予には応答送信までの遅延を含み、クリーンアップと置換プロセスの準備には別々の 60 秒・70 秒の予算があります。中断された処理が既に実行された可能性があるため、再送前に結果を確認してください。自動再送は追加されません。 + |メソッドとパス |目的 |注目すべきエラー | | --- | --- | --- | | `GET /api/system/memory` |スカラー プロセス、ヒープ、ストリーム、応答状態、ウォッチドッグ、およびアクティブ ターン メトリックを返します。 — | diff --git a/docs-site/src/content/docs/ko/reference/management-api.md b/docs-site/src/content/docs/ko/reference/management-api.md index ed7d0292fa8..f9dd67d9f89 100644 --- a/docs-site/src/content/docs/ko/reference/management-api.md +++ b/docs-site/src/content/docs/ko/reference/management-api.md @@ -313,6 +313,8 @@ OpenAI도 같은 규칙을 따르며, 스위치를 켠다고 별도의 922k 모 ### 시스템 수명 주기 +`POST /api/system/restart`는 본문이 없거나 `{}`이면 기본 60초 drain 유예 시간을 유지합니다. 관리 세션 또는 관리자 토큰을 사용하는 호출자는 `{"drainGraceMs":2000}`으로 짧은 유예 시간을 명시적으로 선택할 수 있습니다. 값은 1~60000밀리초의 정수여야 합니다. 잘못된 본문이나 값은 재시작을 시작하지 않고 400을 반환하며, 대상 프로세스에 한정된 로컬 재시작 capability는 이 옵션을 설정할 수 없습니다(403). 응답의 `drainTimeoutMs`는 처음 수락한 유예 시간을 나타내며 반복 호출은 기존 기한을 바꾸지 않습니다. 유예 시간에는 응답 전송 지연이 포함됩니다. 정리와 대체 프로세스 준비에는 각각 별도의 60초와 70초 예산이 유지됩니다. 중단된 요청이 이미 실행되었을 수 있으므로 재전송 전에 결과를 확인하세요. 자동 재전송은 추가되지 않습니다. + | HTTP 메서드와 경로 | 목적 | 주요 오류 | | --- | --- | --- | | `GET /api/system/memory` | 프로세스, heap, stream, response-state, watchdog, active-turn의 스칼라 메트릭을 반환합니다 | — | diff --git a/docs-site/src/content/docs/ru/reference/management-api.md b/docs-site/src/content/docs/ru/reference/management-api.md index 77b456ab70c..209c7ffdf05 100644 --- a/docs-site/src/content/docs/ru/reference/management-api.md +++ b/docs-site/src/content/docs/ru/reference/management-api.md @@ -338,6 +338,8 @@ Management-аутентификация доказывает доступ к п ### Жизненный цикл системы +`POST /api/system/restart` сохраняет стандартный период drain в 60 секунд, если тело отсутствует или равно `{}`. Сессия управления или токен администратора позволяют явно выбрать более короткий период через `{"drainGraceMs":2000}`. Значение должно быть целым числом от 1 до 60000 миллисекунд. Некорректное тело или значение возвращает 400 без запуска перезапуска; локальная capability, привязанная к целевому процессу, не разрешает этот параметр (403). Поле ответа `drainTimeoutMs` сообщает первый принятый период, который повторные вызовы не меняют. Отсчёт включает задержку отправки ответа; отдельные бюджеты очистки и готовности замены остаются равными 60 и 70 секундам. Прерванный запрос мог уже выполниться: проверьте результат перед повторной отправкой. Автоматическая повторная отправка не добавляется. + | Метод и путь | Назначение | Особые ошибки | | --- | --- | --- | | `GET /api/system/memory` | Вернуть скалярные метрики процесса, heap, stream, response-state, watchdog и active-turn | — | diff --git a/docs-site/src/content/docs/tr/reference/management-api.md b/docs-site/src/content/docs/tr/reference/management-api.md index 2199bce119f..5233d4ce458 100644 --- a/docs-site/src/content/docs/tr/reference/management-api.md +++ b/docs-site/src/content/docs/tr/reference/management-api.md @@ -363,6 +363,8 @@ dolaşmamalıdır. Depoya yıldız verip vermeyeceğini kullanıcı seçmelidir. ### Sistem yaşam döngüsü +`POST /api/system/restart`, gövde yoksa veya `{}` ise varsayılan 60 saniyelik boşaltma süresini korur. Bir yönetim oturumu veya yönetici token'ı, `{"drainGraceMs":2000}` ile daha kısa süreyi açıkça seçebilir. Değer 1 ile 60000 milisaniye arasında bir tam sayı olmalıdır. Geçersiz gövde veya değer, yeniden başlatmayı başlatmadan 400 döndürür; hedef sürece bağlı yerel yeniden başlatma capability'si bu seçeneğe izin vermez (403). Yanıttaki `drainTimeoutMs`, ilk kabul edilen süreyi bildirir; tekrarlanan çağrılar mevcut süreyi veya son tarihi değiştirmez. Yanıt gönderme gecikmesi bu süreye dahildir; temizleme ve yeni sürecin hazır olması için ayrı 60 ve 70 saniyelik bütçeler korunur. Kesilen bir istek zaten yürütülmüş olabilir: yeniden göndermeden önce sonucunu kontrol edin. Otomatik yeniden gönderme eklenmez. + | Yöntem ve yol | Amaç | Dikkate değer hatalar | | --- | --- | --- | | `GET /api/system/memory` | Skaler süreç, yığın (heap), akış, yanıt durumu, denetleyici ve aktif tur metriklerini döndürün | — | diff --git a/docs-site/src/content/docs/zh-cn/reference/management-api.md b/docs-site/src/content/docs/zh-cn/reference/management-api.md index abcdb26b4f9..a590806cd01 100644 --- a/docs-site/src/content/docs/zh-cn/reference/management-api.md +++ b/docs-site/src/content/docs/zh-cn/reference/management-api.md @@ -292,6 +292,8 @@ OpenAI 也遵循此规则：开关不会选择特殊的 922k 模式。有效上 ### 系统生命周期 +`POST /api/system/restart` 在没有请求体或请求体为 `{}` 时保留默认的 60 秒排空等待。使用管理会话或管理员令牌的调用方可通过 `{"drainGraceMs":2000}` 显式选择短等待，取值须为 1–60000 毫秒的整数。无效 JSON 或参数返回 400，且不会开始重启；绑定目标进程的本机 restart capability 不允许设置该参数（403）。响应中的 `drainTimeoutMs` 表示首次接受的等待时间，重复调用不会改变已接受的时间或截止点。等待时间包含发送接受响应前的延迟，清理和替代进程就绪仍分别保留独立的 60 秒、70 秒预算。被中断的请求可能已经执行，重发前应核对结果；此选项不会增加自动重发。 + | 方法和路径 | 用途 | 典型错误 | | --- | --- | --- | | `GET /api/system/memory` | 返回标量级的进程、堆、流、响应状态、看门狗和活跃回合指标 | — | diff --git a/docs-site/src/content/docs/zh-tw/reference/management-api.md b/docs-site/src/content/docs/zh-tw/reference/management-api.md index 8bbfae3db3c..a67def0e48a 100644 --- a/docs-site/src/content/docs/zh-tw/reference/management-api.md +++ b/docs-site/src/content/docs/zh-tw/reference/management-api.md @@ -289,6 +289,8 @@ OpenAI 也遵循此規則：開關不會選擇特殊的 922k 模式。生效中 ### 系統生命週期 +`POST /api/system/restart` 在沒有請求本文或本文為 `{}` 時，保留預設的 60 秒排空等待。使用管理工作階段或管理員權杖的呼叫端可透過 `{"drainGraceMs":2000}` 明確選擇短等待；值必須是 1–60000 毫秒的整數。無效 JSON 或參數回傳 400，且不會開始重新啟動；綁定目標行程的本機 restart capability 不允許設定此參數（403）。回應中的 `drainTimeoutMs` 表示第一次接受的等待時間，重複呼叫不會變更已接受的時間或截止點。等待時間包含傳送接受回應前的延遲；清理與替代行程就緒仍分別保留獨立的 60 秒及 70 秒預算。被中斷的請求可能已經執行，重新傳送前應核對結果；此選項不會增加自動重新傳送。 + | 方法與路徑 | 用途 | 主要錯誤 | | --- | --- | --- | | `GET /api/system/memory` | 回傳純量行程、heap、串流、回應狀態、看門狗與活躍回合指標 | — | From cc7cae71b56f255a0dc84fa3cd845d2e254e491f Mon Sep 17 00:00:00 2001 From: BigHulk Date: Wed, 7 Oct 2026 23:44:02 +0800 Subject: [PATCH 4/5] docs(api): clarify replacement readiness in French --- docs-site/src/content/docs/fr/reference/management-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-site/src/content/docs/fr/reference/management-api.md b/docs-site/src/content/docs/fr/reference/management-api.md index dd9a5d843e4..1ee13d85e11 100644 --- a/docs-site/src/content/docs/fr/reference/management-api.md +++ b/docs-site/src/content/docs/fr/reference/management-api.md @@ -345,7 +345,7 @@ lui-même s'il souhaite ajouter une étoile au dépôt. ### Cycle de vie du système -`POST /api/system/restart` conserve par défaut un délai de 60 secondes lorsque le corps est absent ou vaut `{}`. Une session de gestion ou un jeton administrateur peut choisir explicitement un délai plus court avec `{"drainGraceMs":2000}` : la valeur doit être un entier compris entre 1 et 60000 millisecondes. Un corps ou une valeur invalide renvoie 400 sans lancer le redémarrage ; la capacité locale liée à la cible ne permet pas cette option (403). `drainTimeoutMs` indique le délai accepté, fixé par le premier appel et inchangé lors des appels suivants. Il inclut le temps de transmission de la réponse ; les budgets de nettoyage et de disponibilité restent respectivement de 60 et 70 secondes. Une requête interrompue peut déjà avoir été exécutée : vérifiez son résultat avant de la renvoyer, sans répétition automatique. +`POST /api/system/restart` conserve par défaut un délai de 60 secondes lorsque le corps est absent ou vaut `{}`. Une session de gestion ou un jeton administrateur peut choisir explicitement un délai plus court avec `{"drainGraceMs":2000}` : la valeur doit être un entier compris entre 1 et 60000 millisecondes. Un corps ou une valeur invalide renvoie 400 sans lancer le redémarrage ; la capacité locale liée à la cible ne permet pas cette option (403). `drainTimeoutMs` indique le délai accepté, fixé par le premier appel et inchangé lors des appels suivants. Il inclut le temps de transmission de la réponse ; les budgets de nettoyage et de préparation du processus de remplacement restent respectivement de 60 et 70 secondes. Une requête interrompue peut déjà avoir été exécutée : vérifiez son résultat avant de la renvoyer, sans répétition automatique. | Méthode et chemin | Objectif | Erreurs notables | | --- | --- | --- | From c7ebc4f14209dc22e009df29759bfdb1b91956ac Mon Sep 17 00:00:00 2001 From: JUN Date: Thu, 8 Oct 2026 09:50:38 +0900 Subject: [PATCH 5/5] fix(lifecycle): cut turns at a sub-200ms restart grace without shortening the flush window The restart drain always started 200ms after acceptance so the 202 response could flush, which meant an explicit drainGraceMs below 200 cancelled active turns late. Arm a separate early-cut timer at acceptance + grace for those values; cleanup, listener stop and process exit still wait for the 200ms flush window, and a pending veto cancels the timer. Default and >=200ms grace arm nothing new. Co-authored-by: Hulk --- src/server/management/system-restart.ts | 28 +++++-- structure/ops/service-and-sidecars.md | 5 +- tests/server/system-restart-admission.test.ts | 79 +++++++++++++++++-- 3 files changed, 96 insertions(+), 16 deletions(-) diff --git a/src/server/management/system-restart.ts b/src/server/management/system-restart.ts index 780fbc94e3f..3060978d207 100644 --- a/src/server/management/system-restart.ts +++ b/src/server/management/system-restart.ts @@ -37,6 +37,7 @@ * Checked before the service rule: that app, not a service manager, is the parent. */ import { + abortAndReleaseAllTurns, acquireTemporaryDrain, beginShutdownDrain, drainAndShutdown, @@ -64,8 +65,11 @@ import { spawnReplacementStart } from "../restart-replacement"; export { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS } from "../../lib/system-restart-contract"; export { waitForReplacementReady, type ReplacementReadinessIo } from "../restart-replacement"; export const DEADLINE_LISTENER_STOP_TIMEOUT_MS = 5_000; +const RESTART_RESPONSE_FLUSH_DELAY_MS = 200; export interface SystemRestartIo { + /** Cancel and release admitted turns without stopping the listener or process. */ + abortActiveTurns?: () => void; acquireTemporaryDrain?: () => { release(): void } | null; drainAndShutdown?: typeof drainAndShutdown; /** True when a background service can actually respawn this process after exit(1). */ @@ -105,7 +109,7 @@ export interface SystemRestartAdmission { } let restartIo: SystemRestartIo = {}; -/** Prevents double-scheduling in the 200ms window before drainAndShutdown sets draining. */ +/** Prevents double-scheduling during the 200ms response-flush window. */ let restartAccepted = false; let acceptedDrainGraceMs = MEMORY_DRAIN_RESTART_MS; @@ -331,7 +335,7 @@ async function completeDeadlineRestartHandoff( /** * Accept a drain-and-restart request. Returns immediately; the drain + - * respawn runs on a short timer so the HTTP response can flush first. + * respawn waits 200ms for the HTTP response to flush; shorter grace cuts turns separately. * Idempotent while already draining: returns the accepted shape again. */ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: SystemRestartAdmission = {}, options: SystemRestartOptions = {}): { @@ -357,7 +361,9 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: const temporaryDrain = automatic ? (io.acquireTemporaryDrain ?? (() => acquireTemporaryDrain("automatic-restart")))() : null; + let cancelEarlyCut: (() => void) | undefined; const releasePending = () => { + cancelEarlyCut?.(); temporaryDrain?.release(); restartAccepted = false; }; @@ -377,12 +383,24 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: const acceptedAtMs = now(); const restartDeadlineMs = acceptedAtMs + MEMORY_DRAIN_RESTART_MS; const drainDeadlineMs = acceptedAtMs + drainGraceMs; + const scheduleDeadline = io.scheduleDeadline ?? ((fn, ms) => { + const timer = setTimeout(fn, ms); + return () => clearTimeout(timer); + }); // Reject new data-plane traffic immediately (503), before the 200ms response-flush delay. if (!automatic) { if (io.beginShutdownDrain) io.beginShutdownDrain(); else if (io.setDraining) io.setDraining(true); else beginShutdownDrain(); } + // A shorter grace cuts admitted turns at acceptance + grace, but listener stop + // and process exit must still wait for the full response-flush window. + if (!vetoed && drainGraceMs < RESTART_RESPONSE_FLUSH_DELAY_MS) { + cancelEarlyCut = scheduleDeadline(() => { + if (vetoed) return; + (io.abortActiveTurns ?? (() => abortAndReleaseAllTurns(new Error("server shutdown"))))(); + }, drainGraceMs); + } schedule(async () => { if (vetoed) return; pending = false; @@ -408,10 +426,6 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: const drainPromise = Promise.resolve().then( () => drain(undefined, Math.max(0, drainDeadlineMs - now())), ); - const scheduleDeadline = io.scheduleDeadline ?? ((fn, ms) => { - const timer = setTimeout(fn, ms); - return () => clearTimeout(timer); - }); const drainOutcome = await waitForRestartDrain(drainPromise, restartDeadlineMs, now, scheduleDeadline); if (!canHandoff()) return; const exitProcess = io.exitProcess ?? ((code: number) => { process.exit(code); }); @@ -467,7 +481,7 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: } (io.markRecycling ?? markRecyclingForExit)(); exitProcess(drainOutcome === "failed" || drainOutcome === "rejected" ? 1 : 0); - }, 200); + }, RESTART_RESPONSE_FLUSH_DELAY_MS); } return { diff --git a/structure/ops/service-and-sidecars.md b/structure/ops/service-and-sidecars.md index c78d77a120e..af0419f014b 100644 --- a/structure/ops/service-and-sidecars.md +++ b/structure/ops/service-and-sidecars.md @@ -469,7 +469,10 @@ response bypasses the shared provider-overload mapping in `src/lib/errors.ts`. `src/server/management/system-routes.ts` accepts an optional JSON `drainGraceMs` on the authenticated restart API: an integer from 1 to 60000, with omission retaining the 60-second default. Invalid input starts no drain. `src/server/management/system-restart.ts` fixes the selected grace at first -acceptance, including the response-flush delay; repeated requests cannot shorten it. The cleanup +acceptance; repeated requests cannot shorten it. The response-flush delay stays 200ms for every +caller. A grace shorter than 200ms cancels and releases active turns at the accepted deadline via +a separate timer, cancelled by a pending restart veto. Cleanup, listener stop and process exit +still wait for the response-flush delay; grace of at least 200ms arms no extra timer. The cleanup watchdog stays 60 seconds and replacement readiness stays 70 seconds. Short grace is an explicit API opt-in; dashboard, CLI, tray, join and automatic restarts keep their default. An interrupted turn may have an unknown upstream outcome and is not automatically replayed. diff --git a/tests/server/system-restart-admission.test.ts b/tests/server/system-restart-admission.test.ts index 72c0f037a80..5f50c91e658 100644 --- a/tests/server/system-restart-admission.test.ts +++ b/tests/server/system-restart-admission.test.ts @@ -38,11 +38,13 @@ function apiRequest(body?: string) { test("empty API requests and empty objects retain the 60-second default", async () => { for (const payload of [undefined, "{}"]) { - setSystemRestartIoForTests({ isDraining: () => false, schedule: () => {}, setDraining: () => {} }); + const delays: number[] = []; + setSystemRestartIoForTests({ isDraining: () => false, schedule: (_fn, ms) => { delays.push(ms); }, setDraining: () => {} }); const req = apiRequest(payload); const res = await handleManagementAPI(req, new URL(req.url), apiConfig); expect(res?.status).toBe(202); expect((await res!.json()).drainTimeoutMs).toBe(60_000); + expect(delays).toEqual([200]); } }); @@ -83,7 +85,7 @@ test("invalid API grace or malformed JSON is rejected without beginning a drain" test("the API accepts both supported grace boundaries", async () => { for (const drainGraceMs of [1, 60_000]) { - setSystemRestartIoForTests({ isDraining: () => false, schedule: () => {}, setDraining: () => {} }); + setSystemRestartIoForTests({ isDraining: () => false, schedule: () => {}, scheduleDeadline: () => () => {}, setDraining: () => {} }); const req = apiRequest(JSON.stringify({ drainGraceMs })); const res = await handleManagementAPI(req, new URL(req.url), apiConfig); expect(res?.status).toBe(202); @@ -168,7 +170,7 @@ afterEach(() => { resetLifecycleDrainStateForTests(); }); -function restartFixture(options: { automatic?: boolean; holdCleanup?: boolean; holdReadiness?: boolean; failSpawn?: boolean; flushDelayMs?: number; useDefaultGrace?: boolean } = {}) { +function restartFixture(options: { automatic?: boolean; supervised?: boolean; holdCleanup?: boolean; holdReadiness?: boolean; failSpawn?: boolean; flushDelayMs?: number; useDefaultGrace?: boolean; drainGraceMs?: number } = {}) { const clock = new RestartClock(); const date = spyOn(Date, "now").mockImplementation(() => clock.now); const sleep = spyOn(Bun, "sleep").mockImplementation((ms) => new Promise(resolve => { @@ -176,6 +178,9 @@ function restartFixture(options: { automatic?: boolean; holdCleanup?: boolean; h })); restore.push(() => date.mockRestore(), () => sleep.mockRestore()); const calls: string[] = []; + const scheduleDelays: number[] = []; + const deadlineDelays: number[] = []; + const cancelledDeadlines: number[] = []; let finishCleanup!: () => void; const cleanup = new Promise(resolve => { finishCleanup = resolve; }); let finishReadiness!: () => void; @@ -188,8 +193,12 @@ function restartFixture(options: { automatic?: boolean; holdCleanup?: boolean; h const accept = () => acceptSystemRestart({ now: () => clock.now, getActiveTurnCount, - schedule: (fn, ms) => { clock.schedule(() => { running = fn(); }, options.flushDelayMs ?? ms); }, - scheduleDeadline: clock.schedule, + schedule: (fn, ms) => { scheduleDelays.push(ms); clock.schedule(() => { running = fn(); }, options.flushDelayMs ?? ms); }, + scheduleDeadline: (fn, ms) => { + deadlineDelays.push(ms); + const cancel = clock.schedule(fn, ms); + return () => { cancelledDeadlines.push(ms); cancel(); }; + }, drainAndShutdown: async (_server, ms) => { calls.push(`drain:${ms}`); const result = await drainAndShutdown(server, ms); @@ -199,7 +208,7 @@ function restartFixture(options: { automatic?: boolean; holdCleanup?: boolean; h stopListener: () => stopServerListener(server), listenPort: () => 10123, isDesktopSupervised: () => false, - isSupervisedServiceChild: () => false, + isSupervisedServiceChild: () => options.supervised ?? false, spawnStart: async (port, waitForHealth) => { calls.push(`spawn:${port}:${waitForHealth}:${clock.now}`); if (options.failSpawn) throw Object.assign(new Error("fixture"), { code: "EACCES" }); @@ -208,8 +217,8 @@ function restartFixture(options: { automatic?: boolean; holdCleanup?: boolean; h markRecycling: () => { calls.push("recycle"); }, isClientConnected: () => false, exitProcess: code => { calls.push(`exit:${code}`); }, - }, options.automatic ? { onAccepted: callback => { veto = callback; } } : {}, options.useDefaultGrace ? {} : { drainGraceMs: 2_000 }); - return { clock, calls, accept, finishCleanup, finishReadiness, veto: () => veto(), running: () => running }; + }, options.automatic ? { onAccepted: callback => { veto = callback; } } : {}, options.useDefaultGrace ? {} : { drainGraceMs: options.drainGraceMs ?? 2_000 }); + return { clock, calls, scheduleDelays, deadlineDelays, cancelledDeadlines, accept, finishCleanup, finishReadiness, veto: () => veto(), running: () => running }; } function upstreamTurn(bindController = true) { @@ -221,6 +230,60 @@ function upstreamTurn(bindController = true) { return { lease: lease!, controller }; } +for (const drainGraceMs of [1, 199]) { + for (const supervised of [false, true]) { + test(`${drainGraceMs}ms ${supervised ? "supervised" : "standalone"} restart cuts turns by the accepted deadline but waits 200ms to shut down`, async () => { + const fixture = restartFixture({ drainGraceMs, supervised }); + const turn = upstreamTurn(); + const acceptedAtMs = fixture.clock.now; + expect(fixture.accept().drainTimeoutMs).toBe(drainGraceMs); + expect(fixture.accept().alreadyDraining).toBe(true); + await fixture.clock.advance(drainGraceMs - 1); + expect(turn.controller.signal.aborted).toBe(false); + expect(getActiveTurnCount()).toBe(1); + await fixture.clock.advance(1); + expect(fixture.clock.now - acceptedAtMs).toBe(drainGraceMs); + expect(turn.controller.signal.aborted).toBe(true); + expect(getActiveTurnCount()).toBe(0); + expect(fixture.scheduleDelays).toEqual([200]); + expect(fixture.deadlineDelays).toEqual([drainGraceMs]); + expect(fixture.calls).toEqual([]); + await fixture.clock.advance(199 - drainGraceMs); + expect(fixture.calls).toEqual([]); + await fixture.clock.advance(1); + await fixture.running(); + expect(fixture.calls).toContain("drain:0"); + expect(fixture.calls).toContain(`stop:${acceptedAtMs + 200}`); + expect(fixture.calls).toContain(`exit:${supervised ? 1 : 0}`); + expect(fixture.calls.filter(call => call.startsWith("spawn:"))).toEqual(supervised ? [] : [`spawn:10123:true:${acceptedAtMs + 200}`]); + expect(tryAdmitTurn()).toBeNull(); + }); + } +} + +for (const options of [{ drainGraceMs: 200 }, { drainGraceMs: 60_000 }, { useDefaultGrace: true }]) { + test(`restart ${JSON.stringify(options)} arms no early cut`, () => { + const fixture = restartFixture(options); + fixture.accept(); + expect(fixture.scheduleDelays).toEqual([200]); + expect(fixture.deadlineDelays).toEqual([]); + }); +} + +test("vetoing a pending short-grace automatic restart cancels its early cut", async () => { + const fixture = restartFixture({ automatic: true, drainGraceMs: 1 }); + const turn = upstreamTurn(); + fixture.accept(); + expect(fixture.deadlineDelays).toEqual([1]); + fixture.veto(); + expect(fixture.cancelledDeadlines).toEqual([1]); + await fixture.clock.advance(200); + expect(turn.controller.signal.aborted).toBe(false); + expect(getActiveTurnCount()).toBe(1); + expect(fixture.calls).toEqual([]); + expect(isDraining()).toBe(false); +}); + for (const automatic of [false, true]) { const mode = automatic ? "automatic" : "manual"; test(`${mode} default restart preserves active work beyond two seconds`, async () => {