Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 122 additions & 4 deletions gateway/slash_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,14 @@
# its worker thread. (#35994)
_RESET_CLEANUP_TIMEOUT_S = 30.0

# How long a /update reservation stays valid before another /update may take it
# over (see _claim_update_slot). An updater that is killed without writing its
# exit code — host reboot, OOM kill — leaves its marker behind forever, so the
# reservation needs a ceiling or /update wedges for the lifetime of the profile.
# Sized above _watch_update_progress's own 1800s watch timeout so a live update
# that is still being watched is never stolen.
_UPDATE_RESERVATION_TTL_S = 3600.0


def _clean_str(value: Any) -> str:
"""Strip and return a non-empty string value, or empty string."""
Expand Down Expand Up @@ -123,6 +131,78 @@ def _home_thread_from_source(source) -> Optional[str]:
return str(thread_id)


def _claim_update_slot(pending_path: Path, ttl_seconds: float) -> bool:
"""Atomically reserve the profile-wide ``/update`` slot.

``/update`` is profile-global: it rewrites the checkout and the virtualenv
every session on the host shares, and the ``.update_*`` marker files are a
single-slot mailbox holding one requester's routing metadata. Two updaters
must therefore never run at once.

A ``if pending_path.exists(): return`` guard cannot provide that. Two
handlers can both observe the marker missing and both go on to create it,
so the second one's metadata silently replaces the first one's and a second
updater is spawned against the same checkout. ``os.open`` with
``O_CREAT | O_EXCL`` collapses the observe-and-create into a single atomic
syscall, so exactly one caller can ever win.

Returns ``True`` when this caller now owns the slot (the marker exists and
is empty, ready for the caller to fill in with its routing metadata), and
``False`` when another update already holds it.
"""
def _create_exclusive() -> bool:
try:
fd = os.open(pending_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
except FileExistsError:
return False
os.close(fd)
return True

if _create_exclusive():
return True

# The slot is held. Take it over only when it is provably abandoned: a host
# reboot or a SIGKILLed updater leaves a marker that nothing will ever
# clean up, and a permanently wedged /update would be a worse bug than the
# collision this guard exists to close.
try:
age = time.time() - pending_path.stat().st_mtime
except OSError:
# Vanished between the failed create and the stat — the previous update
# finished in that window. Retry once; if we lose again, the loser path
# is still correct.
return _create_exclusive()

if age < ttl_seconds:
return False

logger.warning(
"Reclaiming abandoned /update reservation %s (age %.0fs > %.0fs)",
pending_path, age, ttl_seconds,
)
try:
pending_path.unlink()
except OSError:
return False
return _create_exclusive()


def _release_update_slot(pending_path: Path) -> None:
"""Give back a reservation taken by :func:`_claim_update_slot`.

Only removes the marker while it is still the empty file the claim created.
A non-empty marker means somebody else's routing metadata is now there —
the notifier restoring a live update's marker via
``claimed_path.replace(pending_path)`` — and deleting that would lose their
completion notice.
"""
try:
if pending_path.stat().st_size == 0:
pending_path.unlink()
except OSError:
pass


class GatewaySlashCommandsMixin:
"""In-session slash-command handlers for GatewayRunner."""

Expand Down Expand Up @@ -5659,8 +5739,37 @@ async def _handle_update_command(self, event: MessageEvent) -> str:
return t("gateway.update.hermes_cmd_not_found")

pending_path = _hermes_home / ".update_pending.json"
claimed_path = _hermes_home / ".update_pending.claimed.json"
output_path = _hermes_home / ".update_output.txt"
exit_code_path = _hermes_home / ".update_exit_code"

# Admission control. Reserve the profile-wide update slot BEFORE any
# routing metadata is written and BEFORE the updater is spawned, so two
# concurrent /update invocations (a double tap while the minutes-long
# update runs without an ack, or two platforms on one multiplexed
# gateway) cannot both start an updater against the same checkout and
# venv, with the second one's metadata clobbering the first requester's
# so they never learn their update finished.
#
# `.update_pending.claimed.json` is the notifier's transient rename of
# the marker (see _send_update_notification); while it exists an update
# is in flight. It is checked both BEFORE and AFTER the exclusive
# create, because the notifier can rename pending -> claimed between
# the two: that leaves pending momentarily absent, so the create would
# otherwise succeed and admit a second updater against a live one.
admitted = not claimed_path.exists() and _claim_update_slot(
pending_path, _UPDATE_RESERVATION_TTL_S
)
if admitted and claimed_path.exists():
_release_update_slot(pending_path)
admitted = False

if not admitted:
# The loser still gets the outcome: the watcher is profile-wide and
# reports the result into the chat that started the update.
self._schedule_update_notification_watch()
return t("gateway.update.already_running")

session_key = self._session_key_for_source(event.source)
pending = {
"platform": event.source.platform.value,
Expand All @@ -5674,10 +5783,6 @@ async def _handle_update_command(self, event: MessageEvent) -> str:
pending["thread_id"] = event.source.thread_id
if event.message_id:
pending["message_id"] = event.message_id
_tmp_pending = pending_path.with_suffix(".tmp")
_tmp_pending.write_text(json.dumps(pending), encoding="utf-8")
_tmp_pending.replace(pending_path)
exit_code_path.unlink(missing_ok=True)

# Spawn `hermes update --gateway` detached so it survives gateway restart.
# --gateway enables file-based IPC for interactive prompts (stash
Expand All @@ -5704,6 +5809,17 @@ async def _handle_update_command(self, event: MessageEvent) -> str:
# so the simplest correct thing is: launch an inline Python helper
# that runs the command and writes both outputs.
try:
# Fill the reservation in with this requester's routing metadata.
# Write-to-temp + rename keeps the marker a complete JSON document
# for the readers in gateway/run.py, which poll it concurrently.
# Everything from the claim onwards must release the slot on
# failure, or a /update that never actually started would block the
# next one until the reservation TTL expires.
_tmp_pending = pending_path.with_suffix(".tmp")
_tmp_pending.write_text(json.dumps(pending), encoding="utf-8")
_tmp_pending.replace(pending_path)
exit_code_path.unlink(missing_ok=True)

if sys.platform == "win32":
import textwrap
from hermes_cli._subprocess_compat import windows_detach_popen_kwargs
Expand Down Expand Up @@ -5764,6 +5880,8 @@ async def _handle_update_command(self, event: MessageEvent) -> str:
start_new_session=True,
)
except Exception as e:
# Release the reservation so the user can retry immediately.
pending_path.with_suffix(".tmp").unlink(missing_ok=True)
pending_path.unlink(missing_ok=True)
exit_code_path.unlink(missing_ok=True)
return t("gateway.update.start_failed", error=e)
Expand Down
1 change: 1 addition & 0 deletions locales/af.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ Future messages in this room will use that transcript until `/reset` or another
hermes_cmd_not_found: "✗ Kon nie die `hermes`-opdrag vind nie. Hermes loop, maar die opdateeropdrag kon nie die uitvoerbare lêer op PATH of via die huidige Python-vertolker vind nie. Probeer `hermes update` met die hand in jou terminale uitvoer."
start_failed: "✗ Kon nie opdatering begin nie: {error}"
starting: "⚕ Begin Hermes-opdatering… Ek sal vordering hier stroom."
already_running: "⚕ 'n Hermes-opdatering loop reeds vir hierdie profiel. Ek sal die resultaat rapporteer in die geselsie wat dit begin het."

usage:
rate_limits: "⏱️ **Tariefperke:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/ar.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@ gateway:
hermes_cmd_not_found: "✗ تعذّر تحديد موقع أمر `hermes`. Hermes يعمل، لكن أمر التحديث لم يجد الملف التنفيذي على PATH أو عبر مُفسّر Python الحالي. جرّب تشغيل `hermes update` يدويًا في طرفيتك."
start_failed: "✗ فشل بدء التحديث: {error}"
starting: "⚕ جارٍ بدء تحديث Hermes… سأبثّ التقدّم هنا."
already_running: "⚕ يوجد بالفعل تحديث Hermes قيد التشغيل لهذا الملف الشخصي. سأبلّغ بالنتيجة في المحادثة التي بدأته."

usage:
rate_limits: "⏱️ **حدود المعدّل:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/de.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ Future messages in this room will use that transcript until `/reset` or another
hermes_cmd_not_found: "✗ Der Befehl `hermes` konnte nicht gefunden werden. Hermes läuft, aber der Update-Befehl konnte das ausführbare Programm weder im PATH noch über den aktuellen Python-Interpreter finden. Versuchen Sie, `hermes update` manuell im Terminal auszuführen."
start_failed: "✗ Update konnte nicht gestartet werden: {error}"
starting: "⚕ Hermes-Update wird gestartet… Ich streame den Fortschritt hier."
already_running: "⚕ Für dieses Profil läuft bereits ein Hermes-Update. Ich melde das Ergebnis in dem Chat, der es gestartet hat."

usage:
rate_limits: "⏱️ **Ratenlimits:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/en.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,7 @@ gateway:
hermes_cmd_not_found: "✗ Could not locate the `hermes` command. Hermes is running, but the update command could not find the executable on PATH or via the current Python interpreter. Try running `hermes update` manually in your terminal."
start_failed: "✗ Failed to start update: {error}"
starting: "⚕ Starting Hermes update… I'll stream progress here."
already_running: "⚕ A Hermes update is already running for this profile. I'll report the result in the chat that started it."

usage:
rate_limits: "⏱️ **Rate Limits:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/es.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,7 @@ gateway:
hermes_cmd_not_found: "✗ No se pudo localizar el comando `hermes`. Hermes está en ejecución, pero el comando de actualización no encontró el ejecutable en PATH ni a través del intérprete de Python actual. Intenta ejecutar `hermes update` manualmente en tu terminal."
start_failed: "✗ No se pudo iniciar la actualización: {error}"
starting: "⚕ Iniciando la actualización de Hermes… Transmitiré el progreso aquí."
already_running: "⚕ Ya hay una actualización de Hermes en curso para este perfil. Informaré del resultado en el chat que la inició."

usage:
rate_limits: "⏱️ **Límites de tasa:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/fr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ Future messages in this room will use that transcript until `/reset` or another
hermes_cmd_not_found: "✗ Impossible de localiser la commande `hermes`. Hermes est en cours d'exécution, mais la commande de mise à jour n'a pas pu trouver l'exécutable dans le PATH ni via l'interpréteur Python actuel. Essayez d'exécuter `hermes update` manuellement dans votre terminal."
start_failed: "✗ Échec du démarrage de la mise à jour : {error}"
starting: "⚕ Démarrage de la mise à jour Hermes… Je diffuserai la progression ici."
already_running: "⚕ Une mise à jour Hermes est déjà en cours pour ce profil. Je signalerai le résultat dans la conversation qui l'a lancée."

usage:
rate_limits: "⏱️ **Limites de débit :** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/ga.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,7 @@ Future messages in this room will use that transcript until `/reset` or another
hermes_cmd_not_found: "✗ Níorbh fhéidir an t-ordú `hermes` a aimsiú. Tá Hermes ag rith, ach níorbh fhéidir leis an ordú nuashonraithe an inrite a aimsiú ar PATH ná tríd an léirmhínitheoir Python reatha. Bain triail as `hermes update` a rith de láimh i do theirminéal."
start_failed: "✗ Theip ar nuashonrú a thosú: {error}"
starting: "⚕ Ag tosú nuashonrú Hermes… Cuirfidh mé an dul chun cinn ar shruth anseo."
already_running: "⚕ Tá nuashonrú Hermes ar siúl cheana féin don phróifíl seo. Tuairisceoidh mé an toradh sa chomhrá a thosaigh é."

usage:
rate_limits: "⏱️ **Teorainneacha Ráta:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/hu.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ Future messages in this room will use that transcript until `/reset` or another
hermes_cmd_not_found: "✗ Nem sikerült megtalálni a `hermes` parancsot. A Hermes fut, de a frissítőparancs nem találta a futtatható fájlt a PATH-on vagy a jelenlegi Python interpreteren keresztül. Próbáld futtatni a `hermes update` parancsot manuálisan a terminálban."
start_failed: "✗ Nem sikerült elindítani a frissítést: {error}"
starting: "⚕ Hermes frissítés indítása… A folyamatot itt fogom közvetíteni."
already_running: "⚕ Ehhez a profilhoz már fut egy Hermes frissítés. Az eredményt abban a beszélgetésben jelentem, amelyik elindította."

usage:
rate_limits: "⏱️ **Sebességkorlátok:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/it.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ Future messages in this room will use that transcript until `/reset` or another
hermes_cmd_not_found: "✗ Impossibile localizzare il comando `hermes`. Hermes è in esecuzione, ma il comando di aggiornamento non ha trovato l'eseguibile nel PATH o tramite l'interprete Python attuale. Prova a eseguire `hermes update` manualmente nel terminale."
start_failed: "✗ Avvio dell'aggiornamento non riuscito: {error}"
starting: "⚕ Avvio dell'aggiornamento di Hermes… mostrerò qui i progressi in streaming."
already_running: "⚕ Un aggiornamento di Hermes è già in corso per questo profilo. Riporterò il risultato nella chat che lo ha avviato."

usage:
rate_limits: "⏱️ **Limiti di frequenza:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/ja.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ Future messages in this room will use that transcript until `/reset` or another
hermes_cmd_not_found: "✗ `hermes` コマンドが見つかりません。Hermes は実行中ですが、更新コマンドは PATH 上にも現在の Python インタープリタ経由でも実行可能ファイルを見つけられませんでした。ターミナルで `hermes update` を手動で実行してみてください。"
start_failed: "✗ 更新の開始に失敗しました: {error}"
starting: "⚕ Hermes の更新を開始しています… 進捗をここにストリーミングします。"
already_running: "⚕ このプロファイルでは Hermes の更新がすでに実行中です。結果は更新を開始したチャットに報告します。"

usage:
rate_limits: "⏱️ **レート制限:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/ko.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ Future messages in this room will use that transcript until `/reset` or another
hermes_cmd_not_found: "✗ `hermes` 명령을 찾을 수 없습니다. Hermes는 실행 중이지만 PATH나 현재 Python 인터프리터를 통해 실행 파일을 찾을 수 없습니다. 터미널에서 `hermes update`를 직접 실행해 보세요."
start_failed: "✗ 업데이트 시작 실패: {error}"
starting: "⚕ Hermes 업데이트를 시작합니다… 진행 상황을 여기에 스트리밍하겠습니다."
already_running: "⚕ 이 프로필에서는 이미 Hermes 업데이트가 실행 중입니다. 결과는 업데이트를 시작한 채팅에 보고하겠습니다."

usage:
rate_limits: "⏱️ **요청 제한:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/pt.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ Future messages in this room will use that transcript until `/reset` or another
hermes_cmd_not_found: "✗ Não foi possível localizar o comando `hermes`. O Hermes está em execução, mas o comando de atualização não conseguiu encontrar o executável no PATH nem através do interpretador Python atual. Tenta executar `hermes update` manualmente no teu terminal."
start_failed: "✗ Falha ao iniciar a atualização: {error}"
starting: "⚕ A iniciar a atualização do Hermes… Vou transmitir o progresso aqui."
already_running: "⚕ Já está em curso uma atualização do Hermes para este perfil. Vou comunicar o resultado na conversa que a iniciou."

usage:
rate_limits: "⏱️ **Limites de taxa:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/ru.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ Future messages in this room will use that transcript until `/reset` or another
hermes_cmd_not_found: "✗ Не удалось найти команду `hermes`. Hermes запущен, но команда обновления не нашла исполняемый файл в PATH или через текущий интерпретатор Python. Попробуйте выполнить `hermes update` вручную в терминале."
start_failed: "✗ Не удалось запустить обновление: {error}"
starting: "⚕ Запуск обновления Hermes… Я буду транслировать прогресс сюда."
already_running: "⚕ Для этого профиля уже выполняется обновление Hermes. Я сообщу результат в чат, из которого оно было запущено."

usage:
rate_limits: "⏱️ **Ограничения скорости:** {state}"
Expand Down
1 change: 1 addition & 0 deletions locales/tr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ Future messages in this room will use that transcript until `/reset` or another
hermes_cmd_not_found: "✗ `hermes` komutu bulunamadı. Hermes çalışıyor, ancak güncelleme komutu yürütülebilir dosyayı PATH'te veya mevcut Python yorumlayıcısı aracılığıyla bulamadı. Terminalde `hermes update` komutunu manuel olarak çalıştırmayı deneyin."
start_failed: "✗ Güncelleme başlatılamadı: {error}"
starting: "⚕ Hermes güncellemesi başlatılıyor… İlerlemeyi buraya akıtacağım."
already_running: "⚕ Bu profil için zaten bir Hermes güncellemesi çalışıyor. Sonucu, güncellemeyi başlatan sohbette bildireceğim."

usage:
rate_limits: "⏱️ **Hız Sınırları:** {state}"
Expand Down
Loading
Loading