From 2a9fd69cf6d51bfb5c193670a2622955c8a96153 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:52:01 +0800 Subject: [PATCH 01/26] =?UTF-8?q?fix(server):=20result=20preview=20?= =?UTF-8?q?=E2=80=94=20preserve=20the=20warm=20Mol*=20frame=20across=20sur?= =?UTF-8?q?face=20clears?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit previewStructure cleared the persistent holder on every structure pick, detaching the warm iframe; the next warm-path postToShell then hit a null contentWindow ("Cannot read properties of null (reading 'postMessage')"). Clears now remove everything except the warm frame, and a failed warm render disposes the dead shell so the retry cold-starts cleanly. Co-Authored-By: Claude --- server/revocompute/static/js/task-results.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/server/revocompute/static/js/task-results.js b/server/revocompute/static/js/task-results.js index 1cbc1240..230e7dcf 100644 --- a/server/revocompute/static/js/task-results.js +++ b/server/revocompute/static/js/task-results.js @@ -287,6 +287,19 @@ return warmFrame; } + // The warm Mol* iframe lives inside the holder and must survive surface + // clears: clearing it detaches the frame, whose contentWindow then reads + // null on the next postMessage ("Cannot read properties of null"). + function clearSurfacePreservingWarm(surface) { + if (!warmMolstar || warmMolstar.frame.parentNode !== surface) { + surface.replaceChildren(); + return; + } + Array.from(surface.children).forEach(function (child) { + if (child !== warmMolstar.frame) child.remove(); + }); + } + async function previewStructure(artifact, stage) { var generation = previewHost.generation; var response = await A.authFetch(artifact.url); @@ -295,7 +308,7 @@ var structureText = await response.text(); if (isStale(generation)) return; var surface = structureHolder || stage; - surface.replaceChildren(); + clearSurfacePreservingWarm(surface); surface.appendChild(structureViewerBar(artifact)); if (structureViewer === "py2dmol") { @@ -319,6 +332,9 @@ try { await renderMolstar(structureText, artifact, surface, generation); } catch (error) { if (isStale(generation)) return; + // A dead warm frame must not poison the next pick: dispose it so the + // retry cold-starts a fresh shell. + disposeActiveViewer(); surface.replaceChildren(); surface.appendChild(structureViewerBar(artifact)); var msg = document.createElement("p"); From ec3af91479a1da55e88f1282b132b650ae481c71 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:58:15 +0800 Subject: [PATCH 02/26] =?UTF-8?q?chore:=20Done=20fixing=20=E2=80=94=20warm?= =?UTF-8?q?=20Mol*=20frame=20preserved=20across=20surface=20clears;=20CI?= =?UTF-8?q?=20green,=20live-deployed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From c0105df63f684e07f0cb9f600592695fef5c6a1f Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:01:11 +0800 Subject: [PATCH 03/26] =?UTF-8?q?fix(server):=20warm=20preview=20polish=20?= =?UTF-8?q?=E2=80=94=20review=20fixes,=20structure=20cache,=20loading=20st?= =?UTF-8?q?ate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - toolbar stays above the preserved warm iframe (was pushed below the tall canvas on every warm mount) - disposal is acknowledged: the shell reports "disposed" after its own teardown and the parent detaches the iframe only then (2s timeout fallback), with a shell-side contract assertion - structure texts are cached per artifact (bounded: 3 files / 60 MB), so switching back to an already-viewed structure skips the download - quiet pulsing loading state for structure swaps and generic previews (reduced-motion aware) Co-Authored-By: Claude --- .../revocompute/static/css/task-results.css | 12 +++ server/revocompute/static/js/task-results.js | 87 ++++++++++++++++--- server/revocompute/static/js/viewer-shell.js | 3 + server/tests/js/test_viewer_shell.js | 11 +++ 4 files changed, 100 insertions(+), 13 deletions(-) diff --git a/server/revocompute/static/css/task-results.css b/server/revocompute/static/css/task-results.css index 6d8c7566..3ce363a8 100644 --- a/server/revocompute/static/css/task-results.css +++ b/server/revocompute/static/css/task-results.css @@ -76,3 +76,15 @@ .main-results { grid-template-columns: 1fr; } .structure-color-bar { flex: 1 1 calc(100% - 2.4rem); margin-left: 0; padding-top: 0.4rem; padding-left: 0; border-top: 1px solid var(--line); border-left: 0; } } + +/* Structure preview loading state — quiet pulse, reduced-motion aware. */ +.preview-loading { display: flex; flex-direction: column; align-items: center; gap: 0.6rem; padding: 3rem 1rem; } +.preview-loading-bars { display: flex; gap: 0.4rem; } +.preview-loading-bars span { width: 0.45rem; height: 1.6rem; border-radius: 999px; background: color-mix(in srgb, var(--accent) 35%, var(--paper)); animation: preview-loading-pulse 1.1s ease-in-out infinite; } +.preview-loading-bars span:nth-child(2) { animation-delay: 0.15s; } +.preview-loading-bars span:nth-child(3) { animation-delay: 0.3s; } +.preview-loading-label { margin: 0; color: var(--muted); font-size: 0.82rem; } +@keyframes preview-loading-pulse { 0%, 100% { transform: scaleY(0.55); opacity: 0.45; } 50% { transform: scaleY(1); opacity: 1; } } +@media (prefers-reduced-motion: reduce) { + .preview-loading-bars span { animation: none; opacity: 0.7; } +} diff --git a/server/revocompute/static/js/task-results.js b/server/revocompute/static/js/task-results.js index 230e7dcf..6e496b5d 100644 --- a/server/revocompute/static/js/task-results.js +++ b/server/revocompute/static/js/task-results.js @@ -22,6 +22,13 @@ var structureHolder = null; var warmPending = {}; var warmListenerInstalled = false; + // Downloaded structure texts, keyed by artifact path, so switching back to + // an already-viewed structure skips the network fetch entirely. Bounded: + // at most 3 files and 60 MB total. + var structureTextCache = new Map(); + var structureTextCacheBytes = 0; + var STRUCTURE_CACHE_MAX_FILES = 3; + var STRUCTURE_CACHE_MAX_BYTES = 60 * 1024 * 1024; var viewRegistry = new window.REvoComputePlugins.PluginRegistry("result view"); var MOLSTAR_THEME_COOKIE = "revodesign-molstar-theme"; // Mol* runs inside the isolated /compute/viewer-shell iframe (its bundle @@ -76,11 +83,8 @@ } function disposeActiveViewer() { - if (warmMolstar) { - try { postToShell(warmMolstar.frame, { type: "dispose" }); } catch (e) { /* frame gone */ } - warmMolstar.frame.remove(); - warmMolstar = null; - } + var frame = warmMolstar ? warmMolstar.frame : null; + warmMolstar = null; if (activeMolstar) activeMolstar = null; Object.keys(warmPending).forEach(function (key) { var pending = warmPending[key]; @@ -88,6 +92,23 @@ if (pending.reject) { try { pending.reject(new Error("Viewer disposed")); } catch (e) { /* already settled */ } } delete warmPending[key]; }); + if (!frame) return; + // Give the shell a moment to run its own teardown (selection unsubscribe + // + plugin.dispose) before the iframe is detached; the shell acknowledges + // with a "disposed" message, and a 2s timeout guarantees the frame never + // lingers when the shell is already gone. + var removed = false; + var timer = setTimeout(function () { if (!removed) { removed = true; frame.remove(); } }, 2000); + window.addEventListener("message", function onDisposed(event) { + if (removed || event.source !== frame.contentWindow || !event.data || event.data.type !== "disposed") return; + removed = true; + clearTimeout(timer); + frame.remove(); + window.removeEventListener("message", onDisposed); + }); + try { postToShell(frame, { type: "dispose" }); } catch (e) { + if (!removed) { removed = true; clearTimeout(timer); frame.remove(); } + } } // One shared message listener for the warm shell: resolves the pending @@ -300,16 +321,49 @@ }); } + function showLoading(surface, label) { + var box = document.createElement("div"); + box.className = "preview-loading"; + box.setAttribute("role", "status"); + box.setAttribute("aria-live", "polite"); + var bars = document.createElement("div"); + bars.className = "preview-loading-bars"; + for (var index = 0; index < 3; index += 1) bars.appendChild(document.createElement("span")); + var text = document.createElement("p"); + text.className = "preview-loading-label"; + text.textContent = label || "Loading preview…"; + box.append(bars, text); + if (warmMolstar && warmMolstar.frame.parentNode === surface) surface.insertBefore(box, warmMolstar.frame); + else surface.appendChild(box); + return box; + } + async function previewStructure(artifact, stage) { var generation = previewHost.generation; - var response = await A.authFetch(artifact.url); - if (isStale(generation)) return; - if (!response.ok) throw new Error("Structure download failed (HTTP " + response.status + ")"); - var structureText = await response.text(); - if (isStale(generation)) return; + var cached = structureTextCache.get(artifact.path); + var structureText = cached || null; + if (!structureText) { + var response = await A.authFetch(artifact.url); + if (isStale(generation)) return; + if (!response.ok) throw new Error("Structure download failed (HTTP " + response.status + ")"); + structureText = await response.text(); + if (isStale(generation)) return; + // Bounded cache: evict oldest first while over the file or byte budget. + structureTextCache.set(artifact.path, structureText); + structureTextCacheBytes += structureText.length; + while (structureTextCache.size > STRUCTURE_CACHE_MAX_FILES || structureTextCacheBytes > STRUCTURE_CACHE_MAX_BYTES) { + var oldest = structureTextCache.keys().next().value; + structureTextCacheBytes -= structureTextCache.get(oldest).length; + structureTextCache.delete(oldest); + } + } var surface = structureHolder || stage; clearSurfacePreservingWarm(surface); - surface.appendChild(structureViewerBar(artifact)); + var bar = structureViewerBar(artifact); + // Keep the toolbar above the preserved warm iframe (appending would push + // the controls below the 34–48rem-tall canvas). + if (warmMolstar && warmMolstar.frame.parentNode === surface) surface.insertBefore(bar, warmMolstar.frame); + else surface.appendChild(bar); if (structureViewer === "py2dmol") { stage.hidden = false; @@ -329,8 +383,13 @@ return; } - try { await renderMolstar(structureText, artifact, surface, generation); } + var loading = showLoading(surface, "Loading structure…"); + try { + await renderMolstar(structureText, artifact, surface, generation); + if (loading.parentNode) loading.remove(); + } catch (error) { + if (loading.parentNode) loading.remove(); if (isStale(generation)) return; // A dead warm frame must not poison the next pick: dispose it so the // retry cold-starts a fresh shell. @@ -512,7 +571,7 @@ stage.hidden = true; structureHolder.hidden = false; } - stage.innerHTML = '

Loading preview…

'; + showLoading(stage, "Loading preview…"); try { stage.replaceChildren(); await previewHost.render(artifact); @@ -708,6 +767,8 @@ async function loadResults() { disposeActiveViewer(); + structureTextCache.clear(); + structureTextCacheBytes = 0; if (structureHolder) structureHolder.hidden = true; var response = await A.authFetch("/compute/api/results/" + encodeURIComponent(task.md5)); var payload = await response.json().catch(function () { return {}; }); diff --git a/server/revocompute/static/js/viewer-shell.js b/server/revocompute/static/js/viewer-shell.js index 15d0d0b8..052f9a1d 100644 --- a/server/revocompute/static/js/viewer-shell.js +++ b/server/revocompute/static/js/viewer-shell.js @@ -260,6 +260,9 @@ if (viewer) viewer.plugin.dispose(); viewer = null; host.replaceChildren(); + // Acknowledge teardown so the parent can remove the iframe only after + // this shell's resources (subscriptions, plugin, WebGL context) are gone. + report({ type: "disposed" }); } }); diff --git a/server/tests/js/test_viewer_shell.js b/server/tests/js/test_viewer_shell.js index b4afd8a9..978e7ba1 100644 --- a/server/tests/js/test_viewer_shell.js +++ b/server/tests/js/test_viewer_shell.js @@ -262,6 +262,17 @@ async function main() { if (!reports.some(function (entry) { return entry.payload.type === "ready" && entry.payload.requestId === "probe-2"; })) { throw new Error("warm shell did not report readiness for the second structure"); } + // Disposal acknowledgment: the shell must report "disposed" after teardown + // so the parent can detach the iframe only once cleanup completed. + listeners.message({ + source: parentWindow, + origin: "https://revocompute.example", + data: { type: "dispose" } + }); + await new Promise(function (resolve) { setTimeout(resolve, 0); }); + if (!reports.some(function (entry) { return entry.payload.type === "disposed"; })) { + throw new Error("shell did not acknowledge disposal"); + } console.log("viewer shell message contract passed"); } From 6a6773b9ba4c9a3e2db71059a4e72fbd06f9dd76 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:13:34 +0800 Subject: [PATCH 04/26] fix(server): no loading flash on warm/cached structure swaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shell re-showed its "Preparing interactive structure…" spinner on every remount and the parent flashed a loading box on cached swaps — both are instant paths that read as a refresh. Warm remounts now keep the current structure visible while the next one loads, and cached swaps skip the loading box entirely; only first mounts show the loading state. Co-Authored-By: Claude --- server/revocompute/static/js/task-results.js | 7 ++++--- server/revocompute/static/js/viewer-shell.js | 13 +++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/server/revocompute/static/js/task-results.js b/server/revocompute/static/js/task-results.js index 6e496b5d..fc09c1f1 100644 --- a/server/revocompute/static/js/task-results.js +++ b/server/revocompute/static/js/task-results.js @@ -383,13 +383,14 @@ return; } - var loading = showLoading(surface, "Loading structure…"); + // Cached swaps resolve almost instantly; a loading box would only flash. + var loading = cached ? null : showLoading(surface, "Loading structure…"); try { await renderMolstar(structureText, artifact, surface, generation); - if (loading.parentNode) loading.remove(); + if (loading) loading.remove(); } catch (error) { - if (loading.parentNode) loading.remove(); + if (loading) loading.remove(); if (isStale(generation)) return; // A dead warm frame must not poison the next pick: dispose it so the // retry cold-starts a fresh shell. diff --git a/server/revocompute/static/js/viewer-shell.js b/server/revocompute/static/js/viewer-shell.js index 052f9a1d..73127de4 100644 --- a/server/revocompute/static/js/viewer-shell.js +++ b/server/revocompute/static/js/viewer-shell.js @@ -191,10 +191,15 @@ async function mountStructure(message) { activeRequestId = message.requestId; applyTheme(message.theme); - stateNode.hidden = false; - host.hidden = true; - stateNode.dataset.state = "loading"; - stateNode.textContent = "Preparing interactive structure…"; + // Warm remounts keep the current structure visible while the next one + // loads — showing the "Preparing…" state on every swap flashes for the + // short (cached/warm) awaits and reads as a refresh. + if (!viewer) { + stateNode.hidden = false; + host.hidden = true; + stateNode.dataset.state = "loading"; + stateNode.textContent = "Preparing interactive structure…"; + } try { await ensureMolstarAssets(); if (viewer) { From a344e7e3ccff2c2b44c87e8d193c0e91db299c9e Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:15:54 +0800 Subject: [PATCH 05/26] feat(server): staged first-load experience for the viewer shell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first-time load now tells the truth about what is happening: the long phase (5 MB Mol* bundle download) reads "Downloading the Mol* viewer…", then "Preparing the interactive structure…" once the engine is booted — with a slow breathing glow behind the spinner as the visual signature. Warm/cached swaps still skip the loading state entirely. Co-Authored-By: Claude --- .../revocompute/static/css/viewer-shell.css | 24 +++++++++++++++++++ server/revocompute/static/js/viewer-shell.js | 11 ++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/server/revocompute/static/css/viewer-shell.css b/server/revocompute/static/css/viewer-shell.css index 8592fece..ba319642 100644 --- a/server/revocompute/static/css/viewer-shell.css +++ b/server/revocompute/static/css/viewer-shell.css @@ -31,6 +31,7 @@ html, body { } #shellState { + position: relative; display: flex; flex-direction: column; gap: 0.85rem; @@ -44,6 +45,29 @@ html, body { text-align: center; } +/* Ambient breathing glow behind the ring — the first-load signature. */ +#shellState[data-state="loading"]::after { + position: absolute; + inset: 50% auto auto 50%; + width: 14rem; + height: 14rem; + border-radius: 50%; + background: radial-gradient(circle, color-mix(in srgb, var(--shell-accent) 16%, transparent) 0%, transparent 68%); + content: ""; + transform: translate(-50%, -50%); + animation: shell-breathe 2.6s ease-in-out infinite; + pointer-events: none; +} + +@keyframes shell-breathe { + 0%, 100% { opacity: 0.55; transform: translate(-50%, -50%) scale(0.92); } + 50% { opacity: 1; transform: translate(-50%, -50%) scale(1.06); } +} + +@media (prefers-reduced-motion: reduce) { + #shellState[data-state="loading"]::after { animation: none; opacity: 0.7; } +} + #shellState::before { width: 2.4rem; height: 2.4rem; diff --git a/server/revocompute/static/js/viewer-shell.js b/server/revocompute/static/js/viewer-shell.js index 73127de4..b72bc046 100644 --- a/server/revocompute/static/js/viewer-shell.js +++ b/server/revocompute/static/js/viewer-shell.js @@ -26,6 +26,11 @@ var selectionSubscription = null; var activeRequestId = null; + function setLoadingPhase(text) { + stateNode.dataset.state = "loading"; + stateNode.textContent = text; + } + function selectedResidues() { var residues = new Map(); if (!viewer || !viewer.plugin || !viewer.plugin.managers.structure) return []; @@ -192,13 +197,12 @@ activeRequestId = message.requestId; applyTheme(message.theme); // Warm remounts keep the current structure visible while the next one - // loads — showing the "Preparing…" state on every swap flashes for the + // loads — showing the loading state on every swap flashes for the // short (cached/warm) awaits and reads as a refresh. if (!viewer) { stateNode.hidden = false; host.hidden = true; - stateNode.dataset.state = "loading"; - stateNode.textContent = "Preparing interactive structure…"; + setLoadingPhase("Downloading the Mol* viewer…"); } try { await ensureMolstarAssets(); @@ -208,6 +212,7 @@ // recreating the Viewer re-initializes the whole bundle. await viewer.plugin.clear(); } else { + setLoadingPhase("Preparing the interactive structure…"); viewerId = "molstar-shell-" + Math.random().toString(36).slice(2); var target = document.createElement("div"); target.id = viewerId; From 713075e8bf572da0706bcd6df908dca8decc4ef6 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:55:20 +0800 Subject: [PATCH 06/26] feat(server): shared stderr->stage translator with AlphaFold2 phase markers One awk module (docker/runners/common/stage_translate.awk) plus a tiny per-family pattern file ("marker:regex") rewrites tool stderr phases into the existing REVODESIGN_STAGE stdout protocol; original lines pass through to the stderr log unchanged. AlphaFold2 ships markers for MSA searching, featuring, modeling, and relaxation; the registry gains the matching stage labels. Self-check: sample absl lines -> four markers, noise line -> none. Co-Authored-By: Claude --- server/config/task_types.yaml | 4 +++ server/docker/runners/alphafold/Dockerfile | 2 +- .../docker/runners/alphafold/alphafold.stages | 5 +++ server/docker/runners/alphafold/run.sh | 6 +++- .../docker/runners/common/stage_translate.awk | 33 +++++++++++++++++++ 5 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 server/docker/runners/alphafold/alphafold.stages create mode 100644 server/docker/runners/common/stage_translate.awk diff --git a/server/config/task_types.yaml b/server/config/task_types.yaml index 07db33e2..51b6d304 100644 --- a/server/config/task_types.yaml +++ b/server/config/task_types.yaml @@ -751,6 +751,10 @@ task_types: input_workspace: *fasta_workspace stage_markers: alphafold: "AlphaFold2 structure prediction" + msa_searching: "MSA searching" + featuring: "Building features" + modeling: "Folding models" + relaxing: "Amber relaxation" params: - name: "model_preset" type: "str" diff --git a/server/docker/runners/alphafold/Dockerfile b/server/docker/runners/alphafold/Dockerfile index 3eb73a1e..403130f2 100644 --- a/server/docker/runners/alphafold/Dockerfile +++ b/server/docker/runners/alphafold/Dockerfile @@ -57,7 +57,7 @@ RUN groupadd -f -r ${RUNNER_GROUP} -g ${RUNNER_GID} 2>/dev/null || true && \ (id -u ${RUNNER_USERNAME} >/dev/null 2>&1 && usermod -u ${RUNNER_UID} -g ${RUNNER_GID} ${RUNNER_USERNAME}) || \ useradd -o -r -g ${RUNNER_GROUP} -u ${RUNNER_UID} ${RUNNER_USERNAME} RUN mkdir -p /app/revocompute && chown -R ${RUNNER_UID}:${RUNNER_GID} /app/revocompute /opt/alphafold -COPY --chown=${RUNNER_UID}:${RUNNER_GID} ./docker/runners/alphafold/run.sh ./docker/runners/common/task_context.sh ./docker/runners/common/task_context.py /app/revocompute/ +COPY --chown=${RUNNER_UID}:${RUNNER_GID} ./docker/runners/alphafold/run.sh ./docker/runners/alphafold/alphafold.stages ./docker/runners/common/task_context.sh ./docker/runners/common/task_context.py ./docker/runners/common/stage_translate.awk /app/revocompute/ WORKDIR /app/revocompute/ ENV RUNNER_USERNAME=${RUNNER_USERNAME} \ diff --git a/server/docker/runners/alphafold/alphafold.stages b/server/docker/runners/alphafold/alphafold.stages new file mode 100644 index 00000000..a740127c --- /dev/null +++ b/server/docker/runners/alphafold/alphafold.stages @@ -0,0 +1,5 @@ +# AlphaFold2 stderr phase patterns (absl logging) -> stage markers. +msa_searching:Started Jackhmmer|Started HHblits|Started HHsearch +featuring:Predicting [A-Za-z0-9] +modeling:Running model model_ +relaxing:[Rr]elax|[Aa]mber diff --git a/server/docker/runners/alphafold/run.sh b/server/docker/runners/alphafold/run.sh index a332294d..00e9d8df 100644 --- a/server/docker/runners/alphafold/run.sh +++ b/server/docker/runners/alphafold/run.sh @@ -57,7 +57,11 @@ fi echo "REVODESIGN_STAGE:alphafold" cd "${ALPHAFOLD_PATH:-/opt/alphafold}" -python3 run_alphafold.py "${af_args[@]}" +# AlphaFold logs its phases to stderr (absl logging); the shared translator +# rewrites the stable ones into the stdout stage protocol while passing the +# original lines through to the stderr log unchanged. +python3 run_alphafold.py "${af_args[@]}" 2> >(awk -f /app/revocompute/stage_translate.awk \ + -v PATTERNS=/app/revocompute/alphafold.stages >&1) [[ -n "$(ls "${output_dir}"/*/ranked_0.pdb 2>/dev/null || true)" ]] || { echo "AlphaFold produced no ranked_0.pdb" >&2; exit 1; } diff --git a/server/docker/runners/common/stage_translate.awk b/server/docker/runners/common/stage_translate.awk new file mode 100644 index 00000000..1c2b92c6 --- /dev/null +++ b/server/docker/runners/common/stage_translate.awk @@ -0,0 +1,33 @@ +# stage_translate.awk — shared stderr->stage translator for runner wrappers. +# +# Rewrites tool stderr lines into the existing stdout stage protocol: +# every input line is passed through to stderr unchanged, and lines matching +# a family's pattern file emit "REVODESIGN_STAGE:" on stdout. +# +# Usage: tool ... 2> >(awk -f /app/revocompute/stage_translate.awk \ +# -v PATTERNS=/app/revocompute/.stages >&1) +# +# Pattern file format: "marker:regex" per line; # comments and blank lines +# ignored. Regexes are awk extended regexes (no ":" in patterns). + +BEGIN { + while ((getline line < PATTERNS) > 0) { + sub(/^[ \t]*/, "", line) + if (line == "" || line ~ /^#/) continue + colon = index(line, ":") + if (colon == 0) continue + n += 1 + markers[n] = substr(line, 1, colon - 1) + regexes[n] = substr(line, colon + 1) + } + close(PATTERNS) +} + +{ + print > "/dev/stderr" + for (i = 1; i <= n; i += 1) + if ($0 ~ regexes[i]) { + print "REVODESIGN_STAGE:" markers[i] + fflush() + } +} From 18d33ec560405b0caab5c95cbe1b7afe725ff16d Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:09:41 +0800 Subject: [PATCH 07/26] fix(server): result page polls task status while pending The status line was written once at page load, so a task whose SLURM job started after the page opened kept showing "Queued" until a manual refresh. The page now polls every 15s, updates the status line live, and reloads once the task reaches a terminal state so the result manifest appears on its own. Co-Authored-By: Claude --- server/revocompute/static/js/task-results.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/revocompute/static/js/task-results.js b/server/revocompute/static/js/task-results.js index fc09c1f1..e48bd966 100644 --- a/server/revocompute/static/js/task-results.js +++ b/server/revocompute/static/js/task-results.js @@ -779,6 +779,23 @@ } artifacts = payload.artifacts; resultViews = Array.isArray(payload.views) ? payload.views : []; + // Status is set once at load; poll while the task is pending so a + // queued/running page updates itself and reloads when results land. + if (window.__revocomputeStatusPoll) clearInterval(window.__revocomputeStatusPoll); + window.__revocomputeStatusPoll = setInterval(async function () { + try { + var pollResponse = await A.authFetch("/compute/api/results/" + encodeURIComponent(task.md5)); + var pollPayload = await pollResponse.json().catch(function () { return {}; }); + if (!pollResponse.ok) return; + var statusEl = document.getElementById("resultStatus"); + if (statusEl && pollPayload.status) statusEl.textContent = pollPayload.status; + if (pollPayload.status === "finished" || pollPayload.status === "failed" || + pollPayload.status === "cancelled" || pollPayload.status === "deleted") { + clearInterval(window.__revocomputeStatusPoll); + window.location.reload(); + } + } catch (error) { /* transient network hiccup — retry next tick */ } + }, 15000); document.getElementById("resultFileCount").textContent = artifacts.length; document.getElementById("resultTotalSize").textContent = formatBytes(payload.total_size); var archiveButton = document.getElementById("archiveButton"); From 545617015df640c6b29906ed601a1a1df86a5f84 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:12:20 +0800 Subject: [PATCH 08/26] fix(server): reliable SLURM job-id capture and orphan recovery for queued tasks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SLURM 19.05 does not reliably print srun's stderr banner in time, so submit()'s 5-second wait fell back to a pid-based handle ("srun-12") that was persisted as slurm_job_id — cancel and restart-recovery could never reconnect such tasks, and they stuck at running forever. The wrapper now publishes $SLURM_JOB_ID on stdout as its first line (the allocation is authoritative), with the banner kept as a fallback. The recovery pass now also covers queued tasks, marks tasks with non-SLURM handles failed, and logs instead of dying silently on boot. Co-Authored-By: Claude --- .../revocompute/job/runners/slurm_runner.py | 19 +++++++++++++++--- server/revocompute/task_runtime.py | 20 +++++++++++++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/server/revocompute/job/runners/slurm_runner.py b/server/revocompute/job/runners/slurm_runner.py index c86b89e3..e491741a 100644 --- a/server/revocompute/job/runners/slurm_runner.py +++ b/server/revocompute/job/runners/slurm_runner.py @@ -239,7 +239,15 @@ def _build_wrapper_script(self) -> str: return path def _render_wrapper(self) -> str: - lines = ["#!/bin/bash", "set -euo pipefail", ""] + lines = [ + "#!/bin/bash", + "set -euo pipefail", + "", + # The allocation carries the authoritative job id; publish it on + # stdout first so the runner never depends on srun's stderr + # banner (which SLURM 19.05 does not always print in time). + 'echo "REVODESIGN_JOB_ID=${SLURM_JOB_ID}"', + ] self._render_input_staging(lines) lines.append("") self._render_apptainer_invocation(lines) @@ -321,14 +329,19 @@ def _read_stdout(self) -> None: markers = self.tt.stage_markers for line in iter(stream.readline, ""): self._stdout_lines.append(line) + if line.startswith("REVODESIGN_JOB_ID=") and self._slurm_job_id is None: + candidate = line.split("=", 1)[1].strip() + if candidate.isdigit(): + self._slurm_job_id = candidate + self._job_id_event.set() if markers and self.stage_callback: stage = extract_stage_from_log_line(line, markers) if stage and stage != last_stage: last_stage = stage try: self.stage_callback(stage) - except Exception: - pass + except Exception as exc: # surface, never mask status updates + logging.error("Stage callback failed for task %s: %s", self.task_id, exc) stream.close() def _read_stderr(self) -> None: diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index 16267b03..038ecb6d 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -822,12 +822,19 @@ def _recover_orphaned_tasks() -> int: """ recovered = 0 for task in task_store.list_tasks(): - if task.get("status") != "running": + if task.get("status") not in {"running", "queued"}: continue container_id = str(task.get("container_id") or "") slurm_job_id = str(task.get("slurm_job_id") or "") if not container_id and not slurm_job_id: continue + if slurm_job_id and not slurm_job_id.isdigit(): + # A pid-based fallback ("srun-12") survived into the record: the + # real job handle is lost, so the task can never be reconnected. + logging.warning("Recovery: task %s has a non-SLURM job handle %r; marking failed", task["md5sum"], slurm_job_id) + _record_failure(task["md5sum"], task, task.get("started_at") or time.time(), "", "SLURM job handle was lost during a server restart") + recovered += 1 + continue md5sum = task["md5sum"] task_type = task.get("task_type", "gremlin") logging.info("Recovery: checking orphaned task %s", md5sum) @@ -928,9 +935,14 @@ def _setup_recovery_signal(sender, **kwargs): # noqa: F811 @worker_ready.connect def _on_worker_ready(sender, **kwargs): - count = _recover_orphaned_tasks() - if count: - logging.info("Recovered %d orphaned task(s)", count) + try: + count = _recover_orphaned_tasks() + if count: + logging.info("Recovered %d orphaned task(s)", count) + else: + logging.info("Recovery: no orphaned tasks found") + except Exception as exc: # boot-time recovery must never die silently + logging.error("Recovery pass failed: %s", exc) except ImportError: pass # celery.signals not available in all environments From e2e3bdd00053d4ea37e54f411c328c15ed6fe634 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:15:35 +0800 Subject: [PATCH 09/26] fix(server): register the orphan-recovery worker_ready handler directly The nested on_after_configure indirection never armed worker_ready in the production worker (0 receivers), so restart-recovery silently never ran. Connect the signal at module scope instead. Co-Authored-By: Claude --- server/revocompute/task_runtime.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index 038ecb6d..705b1e33 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -928,21 +928,18 @@ def _finalize_after_poll(md5sum, task, tt, state): try: + from celery.signals import worker_ready - @celery.on_after_configure.connect - def _setup_recovery_signal(sender, **kwargs): # noqa: F811 - from celery.signals import worker_ready - - @worker_ready.connect - def _on_worker_ready(sender, **kwargs): - try: - count = _recover_orphaned_tasks() - if count: - logging.info("Recovered %d orphaned task(s)", count) - else: - logging.info("Recovery: no orphaned tasks found") - except Exception as exc: # boot-time recovery must never die silently - logging.error("Recovery pass failed: %s", exc) + @worker_ready.connect + def _on_worker_ready(sender, **kwargs): + try: + count = _recover_orphaned_tasks() + if count: + logging.info("Recovered %d orphaned task(s)", count) + else: + logging.info("Recovery: no orphaned tasks found") + except Exception as exc: # boot-time recovery must never die silently + logging.error("Recovery pass failed: %s", exc) except ImportError: pass # celery.signals not available in all environments From 0e2ddea7f38c7b164f3ef1673c8c80bcafd19d32 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:17:45 +0800 Subject: [PATCH 10/26] fix(server): recovery fails tasks whose SLURM job left both sacct and squeue The unknown-state branch kept tasks running forever when sacct lost the job record (cancelled srun client). A live-queue probe now decides: alive in squeue -> re-queue poll; absent from both -> the job was cancelled during a restart, record the failure. Co-Authored-By: Claude --- server/revocompute/task_runtime.py | 31 ++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index 705b1e33..8b45e891 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -878,13 +878,32 @@ def _recover_orphaned_tasks() -> int: ) recovered += 1 elif slurm_state is None: - # Unknown state (sacct missing or query failed): the job - # may still be running on the cluster. Leave the task - # running — do not record a failure and do not clean up - # its input workspace under a live job. - logging.warning( - "Recovery: SLURM job %s state unknown for %s; leaving task running", slurm_job_id, md5sum + # Unknown sacct state: consult the live queue. A job in + # neither sacct nor squeue was cancelled out from under + # the worker (e.g. the srun client died at a restart). + probe = subprocess.run( + ["squeue", "-h", "-j", slurm_job_id, "-o", "%T"], + capture_output=True, + text=True, + timeout=10, ) + if probe.returncode == 0 and probe.stdout.strip(): + logging.warning( + "Recovery: SLURM job %s alive in queue for %s; re-queuing poll", slurm_job_id, md5sum + ) + run_compute_task.apply_async( + args=[md5sum], kwargs={"task_type": task_type, "recover_slurm": slurm_job_id} + ) + recovered += 1 + else: + _record_failure( + md5sum, + task, + task.get("started_at") or time.time(), + "", + "SLURM job was cancelled during a server restart", + ) + recovered += 1 else: _record_failure( md5sum, From 2d20545b5f9674b142625056bcdd1e60088f2ced Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:19:28 +0800 Subject: [PATCH 11/26] fix(server): import subprocess for the recovery live-queue probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe used subprocess without the import — the recovery branch errored out and left the task running. Co-Authored-By: Claude --- server/revocompute/task_runtime.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index 8b45e891..e429b6e2 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -18,6 +18,7 @@ import os import re import shutil +import subprocess import time import zipfile from datetime import datetime From c13055558cadf8bb011d1a2bd44b0ba087a2fcb6 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:20:24 +0800 Subject: [PATCH 12/26] fix(server): alphafold stages are the four real phases, no wrapper marker The first stage was the runner's own launch echo; the actual phases are the translator-derived MSA searching, featuring, modeling, and relaxation. Drop the wrapper marker from both the registry and run.sh. Co-Authored-By: Claude --- server/config/task_types.yaml | 1 - server/docker/runners/alphafold/run.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/server/config/task_types.yaml b/server/config/task_types.yaml index 51b6d304..c675d614 100644 --- a/server/config/task_types.yaml +++ b/server/config/task_types.yaml @@ -750,7 +750,6 @@ task_types: input_label: "Protein sequence (FASTA)" input_workspace: *fasta_workspace stage_markers: - alphafold: "AlphaFold2 structure prediction" msa_searching: "MSA searching" featuring: "Building features" modeling: "Folding models" diff --git a/server/docker/runners/alphafold/run.sh b/server/docker/runners/alphafold/run.sh index 00e9d8df..c45fd7d2 100644 --- a/server/docker/runners/alphafold/run.sh +++ b/server/docker/runners/alphafold/run.sh @@ -55,7 +55,6 @@ else ) fi -echo "REVODESIGN_STAGE:alphafold" cd "${ALPHAFOLD_PATH:-/opt/alphafold}" # AlphaFold logs its phases to stderr (absl logging); the shared translator # rewrites the stable ones into the stdout stage protocol while passing the From d79b35616513599c7ce3255175ebc30be98e9bdb Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:33:53 +0800 Subject: [PATCH 13/26] fix(server): delegate job cancellation to the worker; implement recover_slurm attach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cancel endpoint ran scancel/docker-stop inside the web process, which has neither SLURM tooling nor the Docker socket — cancelled tasks kept running on the cluster. Cancellation now dispatches a worker-side Celery task (cancel_compute_resources). Also implement the missing recover_slurm parameter: the restart-recovery re-queue now attaches to the live SLURM job, polling until it finishes instead of resubmitting. Co-Authored-By: Claude --- server/revocompute/routes.py | 31 ++++++--------------- server/revocompute/task_runtime.py | 43 +++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/server/revocompute/routes.py b/server/revocompute/routes.py index 8a3fc7db..1f4dd082 100644 --- a/server/revocompute/routes.py +++ b/server/revocompute/routes.py @@ -110,6 +110,7 @@ _task_zip_path, _virtual_upload_path, build_results_archive, + cancel_compute_resources, format_times, format_walltime, run_compute_task, @@ -1146,29 +1147,13 @@ def cancel_task(md5sum): 400, ) - slurm_job_id = task.get("slurm_job_id") - if slurm_job_id: - scancel = shutil.which("scancel") - if not scancel: - logging.warning("scancel not found; cannot cancel SLURM job %s", slurm_job_id) - else: - try: - subprocess.run([scancel, str(slurm_job_id)], timeout=10, check=True) - logging.info("Cancelled SLURM job %s for task %s", slurm_job_id, md5sum) - except Exception as exc: # pylint: disable=broad-except - logging.warning("Failed to scancel SLURM job %s: %s", slurm_job_id, exc) - - container_id = task.get("container_id") - if container_id: - docker = shutil.which("docker") - if not docker: - logging.warning("docker not found; cannot stop container %s", container_id) - else: - try: - subprocess.run([docker, "stop", str(container_id)], timeout=15, check=True) - logging.info("Stopped Docker container %s for task %s", container_id, md5sum) - except Exception as exc: # pylint: disable=broad-except - logging.warning("Failed to stop container %s: %s", container_id, exc) + # SLURM tooling and the Docker socket live only in the worker container; + # the web process delegates the kill there. The task record flips to + # cancelled immediately either way. + cancel_compute_resources.delay( + slurm_job_id=str(task["slurm_job_id"]) if task.get("slurm_job_id") else None, + container_id=str(task["container_id"]) if task.get("container_id") else None, + ) celery_id = task.get("celery_task_id") if celery_id: diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index e429b6e2..5d25aba6 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -656,7 +656,7 @@ def _capture_debug_submission(task: dict[str, Any], entities: list[dict], params # --------------------------------------------------------------------------- -def _execute_compute_task(md5sum: str, task_type: str = "gremlin", params: dict | None = None): +def _execute_compute_task(md5sum: str, task_type: str = "gremlin", params: dict | None = None, recover_slurm: str | None = None): """Core task logic — shared by legacy and generic Celery task wrappers. Reads entities from the task's ``input_form`` column. The ``params`` @@ -670,6 +670,10 @@ def _execute_compute_task(md5sum: str, task_type: str = "gremlin", params: dict if task["status"] not in {"pending", "queued", "running"}: return + if recover_slurm: + _recover_slurm_task(md5sum, task, recover_slurm) + return + try: tt, runner = _get_task_type(task_type) except KeyError: @@ -971,9 +975,40 @@ def _on_worker_ready(sender, **kwargs): @celery.task(name="run_compute_task", bind=True, max_retries=0) -def run_compute_task(self, md5sum: str, task_type: str = "gremlin", params: dict | None = None): - """Compute task — dispatched by task_type.""" - return _execute_compute_task(md5sum, task_type, params) +def run_compute_task(self, md5sum: str, task_type: str = "gremlin", params: dict | None = None, recover_slurm: str | None = None): + """Compute task — dispatched by task_type. + + ``recover_slurm`` attaches to an already-running SLURM job after a + server restart instead of submitting a new one. + """ + return _execute_compute_task(md5sum, task_type, params, recover_slurm) + + +@celery.task(name="cancel_compute_resources", bind=True, max_retries=0) +def cancel_compute_resources(self, slurm_job_id: str | None = None, container_id: str | None = None): + """Kill a task's compute resources from the worker, which is the only + container with SLURM tooling (and the Docker socket) mounted. The web + process cannot scancel or docker-stop directly.""" + if slurm_job_id: + scancel = shutil.which("scancel") + if not scancel: + logging.warning("scancel not found; cannot cancel SLURM job %s", slurm_job_id) + else: + try: + subprocess.run([scancel, str(slurm_job_id)], timeout=10, check=True) + logging.info("Cancelled SLURM job %s", slurm_job_id) + except Exception as exc: # pylint: disable=broad-except + logging.warning("Failed to scancel SLURM job %s: %s", slurm_job_id, exc) + if container_id: + docker = shutil.which("docker") + if not docker: + logging.warning("docker not found; cannot stop container %s", container_id) + else: + try: + subprocess.run([docker, "stop", str(container_id)], timeout=15, check=True) + logging.info("Stopped Docker container %s", container_id) + except Exception as exc: # pylint: disable=broad-except + logging.warning("Failed to stop Docker container %s: %s", container_id, exc) @celery.task(name="build_results_archive", bind=True, max_retries=0) From c535af19a49b59bcaeafc1745d4af538939f4ec3 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:34:48 +0800 Subject: [PATCH 14/26] =?UTF-8?q?fix(server):=20implement=20=5Frecover=5Fs?= =?UTF-8?q?lurm=5Ftask=20=E2=80=94=20attach-and-poll=20after=20restart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-queue referenced a recovery entrypoint that did not exist yet. The poller attaches to the live job, self-requeues every 30s while it runs, and finalizes (or fails) once sacct/squeue show it gone. Co-Authored-By: Claude --- server/revocompute/task_runtime.py | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index 5d25aba6..9fdd9186 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -819,6 +819,43 @@ def _on_stage_change(stage: str) -> None: # --------------------------------------------------------------------------- +def _recover_slurm_task(md5sum: str, task: dict[str, Any], slurm_job_id: str) -> None: + """Attach to a live SLURM job after a restart instead of resubmitting. + + Polls sacct/squeue with a self-requeue until the job ends, then + finalizes from the result directory. + """ + try: + tt, runner = _get_task_type(task.get("task_type", "gremlin")) + from revocompute.job.runners.slurm_runner import SlurmJob + + job = SlurmJob(md5sum, tt, runner, [], task["result_dir"]) + state = job.reconnect(slurm_job_id) + requeue = { + "args": [md5sum], + "kwargs": {"task_type": task.get("task_type", "gremlin"), "recover_slurm": slurm_job_id}, + "countdown": 30, + } + if state is True: + run_compute_task.apply_async(**requeue) + return + if state is None: + probe = subprocess.run( + ["squeue", "-h", "-j", slurm_job_id, "-o", "%T"], + capture_output=True, + text=True, + timeout=10, + ) + if probe.returncode == 0 and probe.stdout.strip(): + run_compute_task.apply_async(**requeue) + return + final = JobState.COMPLETED if job._has_result_artifact() else JobState.FAILED + _finalize_after_poll(md5sum, task, tt, final) + except Exception as exc: + logging.error("recover_slurm failed for %s: %s", md5sum, exc) + _record_failure(md5sum, task, task.get("started_at") or time.time(), "", f"Recovery error: {exc}") + + def _recover_orphaned_tasks() -> int: """Scan for running tasks whose Celery task was lost (e.g. Redis restart). From dc54e5816b3101cc5d31713ba21dbb392a266a4f Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:52:10 +0800 Subject: [PATCH 15/26] fix(server): dashboard polls non-terminal task statuses Status pills were static after load, so a task that started after the page opened kept showing "Queued". Non-terminal pills now poll every 15s, update in place, and the page reloads when a task finishes so the results actions and traces render. Co-Authored-By: Claude --- server/revocompute/static/js/dashboard.js | 28 ++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/server/revocompute/static/js/dashboard.js b/server/revocompute/static/js/dashboard.js index f653706e..a3289016 100644 --- a/server/revocompute/static/js/dashboard.js +++ b/server/revocompute/static/js/dashboard.js @@ -207,7 +207,7 @@ '' + '' + '
' + - '' + escapeHtml(meta.label) + tracePopover + '' + + '' + escapeHtml(meta.label) + tracePopover + '' + errorHelp + '
' + '' + @@ -255,6 +255,32 @@ }); } + // Status pills are rendered once at load; poll the non-terminal ones so a + // queued/running badge updates itself, and reload when a task finishes so + // the results buttons and traces appear without a manual refresh. + async function pollStatuses() { + var pills = document.querySelectorAll(".status-pill[data-task-status]"); + for (var index = 0; index < pills.length; index += 1) { + var pill = pills[index]; + var status = pill.dataset.taskStatus; + if (status === "finished" || status === "failed" || status === "cancelled" || status === "deleted") continue; + try { + var response = await A.authFetch("/compute/api/running/" + encodeURIComponent(pill.dataset.md5)); + var payload = await response.json().catch(function () { return {}; }); + if (!response.ok || !payload.status || payload.status === status) continue; + var meta = getStatusMeta(payload.status); + pill.textContent = meta.label; + pill.className = "status-pill " + meta.css; + pill.dataset.taskStatus = payload.status; + if (payload.status === "finished" || payload.status === "failed" || payload.status === "cancelled" || payload.status === "deleted") { + window.location.reload(); + return; + } + } catch (error) { /* transient — retry next tick */ } + } + } + setInterval(pollStatuses, 15000); + function setActiveFilter(nextFilter) { state.filter = nextFilter; var chips = document.querySelectorAll("#statusFilters .chip"); From 0fdbe28f78ab0e7de1229dae224d5fa42f6b79a8 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:56:46 +0800 Subject: [PATCH 16/26] fix(server): recovery flips queued orphans to running when the job is alive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A task orphaned while still queued showed "Queued" forever even though its SLURM job was running — the attach-poller now updates the status when it confirms the job is alive. Co-Authored-By: Claude --- server/revocompute/task_runtime.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index 9fdd9186..3c880a82 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -837,6 +837,10 @@ def _recover_slurm_task(md5sum: str, task: dict[str, Any], slurm_job_id: str) -> "countdown": 30, } if state is True: + # The job is alive on the cluster even though the record may + # still say "queued" from before the restart. + if task.get("status") != "running" and not _is_terminal_status(task.get("status")): + task_store.update_task(md5sum, status="running") run_compute_task.apply_async(**requeue) return if state is None: From 69dfe0c2e496cea9a15b72d4bb4add44aabc7d19 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 10:32:10 +0800 Subject: [PATCH 17/26] fix(server): liveness stage at wrapper start so queued flips to running immediately The allocation wrapper's first stdout line (REVODESIGN_JOB_ID) now also emits the first declared stage as a liveness signal, so a queued task becomes running as soon as the wrapper starts instead of waiting for the tool's first translated marker. Docstrings, tests, and docs updated. Co-Authored-By: Claude --- CHANGELOG.md | 1 + docs/dev-guide/server.md | 5 ++- server/README.md | 17 ++++---- .../revocompute/job/runners/slurm_runner.py | 40 +++++++++++++------ server/tests/test_slurm_runner.py | 23 +++++++++++ 5 files changed, 64 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21a3500b..9b9a4c9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Server: + - SLURM status: move allocated tasks from queued to running as soon as the wrapper publishes the real job ID, without waiting for scientific stage output. - RFdiffusion runner: redirect Hydra bookkeeping and generated schedule caches to the per-job writable `/tmp` filesystem under read-only container execution; lock task defaults to the pinned upstream inference configuration. - Mol* input workbenches: sequence-strip and canvas selections now report canonical residues; cached viewer-shell handshakes no longer race listener setup. - task-type intro audit: intros/categories corrected against the runner docs (Pro-Prime = OGT prediction → `fitness`; ThermoMPNN-D = ΔΔG prediction; PLACER = all-atom ligand modeling; GREMLIN = conservation + couplings; HyperMPNN/LASErMPNN/OpenDDE reworded). diff --git a/docs/dev-guide/server.md b/docs/dev-guide/server.md index facc15a9..f8e948e7 100644 --- a/docs/dev-guide/server.md +++ b/docs/dev-guide/server.md @@ -126,8 +126,9 @@ failures, and create `task_finished` only after real success. Docker and SLURM runners share this contract. SLURM starts a job in a valid existing working directory, stores the actual SLURM job ID, polls queue/account state, and maps cancellation to `scancel`. A task changes from queued to running -when SLURM reports execution or a stage marker arrives; it is not inferred only -from local `srun` process state. +when the allocation wrapper publishes its numeric job ID; the first declared +stage is emitted once as a liveness signal, and later stage markers advance +progress. Execution is not inferred only from local `srun` process state. GPU task types require user GPU permission and configured GPU resources. Apptainer receives `--nv` only for GPU tasks. CPU task families must not inherit diff --git a/server/README.md b/server/README.md index 6e02c5d3..7812e206 100644 --- a/server/README.md +++ b/server/README.md @@ -1170,9 +1170,10 @@ job is enqueued. ### 13.7 slurm_job_id Persistence -The SLURM job ID (`srun-`) is stored in the `slurm_job_id` column of -the tasks table. When a user cancels a running SLURM task, the web process -calls `scancel` with this ID before terminating the `srun` process. +The allocation wrapper writes `REVODESIGN_JOB_ID=` as its first +stdout line. The worker stores that real SLURM ID in the `slurm_job_id` column +of the tasks table so cancellation and restart recovery can address the +allocation directly. ### 13.8 Live Output @@ -1184,12 +1185,14 @@ Docker runner's live progress behaviour. ### 13.9 Task States SLURM tasks use an additional `queued` status: + - **`queued`** — `srun` is waiting for a SLURM allocation (resource contention) -- **`running`** — first `REVODESIGN_STAGE:` marker received, job is executing +- **`running`** — the allocation wrapper has started on a compute node -This transition happens via the stage callback: when the first -`REVODESIGN_STAGE:` line appears on stdout, the status moves from `queued` -to `running`. +When the worker captures the wrapper's numeric job-ID line, it invokes the +stage callback with the first declared stage as a liveness signal. Later +`REVODESIGN_STAGE:` lines advance progress normally; a repeated first marker +is deduplicated. ## 14. Troubleshooting diff --git a/server/revocompute/job/runners/slurm_runner.py b/server/revocompute/job/runners/slurm_runner.py index e491741a..96a11670 100644 --- a/server/revocompute/job/runners/slurm_runner.py +++ b/server/revocompute/job/runners/slurm_runner.py @@ -31,10 +31,11 @@ class SlurmJob(Job): """A compute job submitted via SLURM + Apptainer. ``submit()`` launches ``srun`` via ``subprocess.Popen`` and returns - the real SLURM job id (captured from srun's stderr banner), falling - back to a pid-based id if the banner never arrives. ``poll()`` waits - for the process to exit and returns ``COMPLETED`` or ``FAILED`` based - on the exit code. + the real SLURM job id, captured from the allocation wrapper's first + stdout line or an ``srun`` stderr banner, and temporarily falls back to + a pid-based id if neither arrives during submission. ``poll()`` waits for + the process to exit and returns ``COMPLETED`` or ``FAILED`` based on the + exit code. """ def __init__( @@ -114,9 +115,9 @@ def submit(self) -> str: self._stdout_thread.start() self._stderr_thread.start() - # srun prints "srun: job NNNN queued and waiting for resources" on - # stderr right after submitting — grab the real job id so scancel - # works. Fall back to the pid-based id if the banner never arrives. + # Capture the real id from the allocation wrapper's first stdout line + # or srun's stderr banner. Fall back to the pid-based id if neither + # arrives during submission. self._job_id_event.wait(timeout=5.0) if self._slurm_job_id: self._job_id = self._slurm_job_id @@ -327,6 +328,17 @@ def _read_stdout(self) -> None: stream = self._process.stdout last_stage: str | None = None markers = self.tt.stage_markers + + def emit_stage(stage: str) -> None: + nonlocal last_stage + if stage == last_stage: + return + last_stage = stage + try: + self.stage_callback(stage) + except Exception as exc: # surface, never mask status updates + logging.error("Stage callback failed for task %s: %s", self.task_id, exc) + for line in iter(stream.readline, ""): self._stdout_lines.append(line) if line.startswith("REVODESIGN_JOB_ID=") and self._slurm_job_id is None: @@ -334,14 +346,16 @@ def _read_stdout(self) -> None: if candidate.isdigit(): self._slurm_job_id = candidate self._job_id_event.set() + if markers and self.stage_callback: + # The allocation is live before the scientific tool + # prints its first marker. Emit the first declared + # stage as a liveness signal so queued tasks become + # running as soon as the wrapper starts. + emit_stage(next(iter(markers))) if markers and self.stage_callback: stage = extract_stage_from_log_line(line, markers) - if stage and stage != last_stage: - last_stage = stage - try: - self.stage_callback(stage) - except Exception as exc: # surface, never mask status updates - logging.error("Stage callback failed for task %s: %s", self.task_id, exc) + if stage: + emit_stage(stage) stream.close() def _read_stderr(self) -> None: diff --git a/server/tests/test_slurm_runner.py b/server/tests/test_slurm_runner.py index 080f57a4..5f4d2c0b 100644 --- a/server/tests/test_slurm_runner.py +++ b/server/tests/test_slurm_runner.py @@ -9,6 +9,8 @@ import shutil import subprocess from dataclasses import replace +from io import StringIO +from types import SimpleNamespace from unittest.mock import patch import pytest @@ -313,6 +315,27 @@ def test_submit_launches_srun(tmp_path): assert args[0] == "srun" +def test_job_id_capture_emits_first_stage_as_liveness_signal(tmp_path): + stages_seen = [] + job = SlurmJob( + "task-1", + _make_task_type(), + _make_runner(), + _make_entities(), + str(tmp_path / "out"), + stage_callback=stages_seen.append, + ) + stdout = StringIO("REVODESIGN_JOB_ID=4154\n" "REVODESIGN_STAGE:hhblits\n" "REVODESIGN_STAGE:gremlin\n") + job._process = SimpleNamespace(stdout=stdout) + + job._read_stdout() + + assert job._slurm_job_id == "4154" + assert job._job_id_event.is_set() + assert stages_seen == ["hhblits", "gremlin"] + assert stdout.closed + + def test_poll_returns_completed_on_exit_zero(tmp_path): job = SlurmJob("task-1", _make_task_type(), _make_runner(), _make_entities(), str(tmp_path / "out")) fake_proc = subprocess.Popen(["true"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) From 3209d8f0f4366059e84661c2b44ba1e636371da4 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 10:44:16 +0800 Subject: [PATCH 18/26] refactor(server): pre-stop SLURM sweep replaces the restart-recovery machinery restart.sh now cancels every in-flight SLURM job and marks the affected task records failed BEFORE stopping the stack, so no SLURM orphan can ever exist. The attach-poller, sacct/squeue probes, recover_slurm parameter, and SlurmJob.reconnect are deleted; only Docker-container recovery remains (containers survive the restart). Fixes the queued orphan, bogus job-handle, and mid-restart status cases by prevention instead of repair. Co-Authored-By: Claude --- .../revocompute/job/runners/slurm_runner.py | 29 --- server/revocompute/task_runtime.py | 178 ++++-------------- server/run/restart.sh | 39 ++++ server/tests/test_slurm_runner.py | 21 --- 4 files changed, 73 insertions(+), 194 deletions(-) diff --git a/server/revocompute/job/runners/slurm_runner.py b/server/revocompute/job/runners/slurm_runner.py index 96a11670..44ae4547 100644 --- a/server/revocompute/job/runners/slurm_runner.py +++ b/server/revocompute/job/runners/slurm_runner.py @@ -63,35 +63,6 @@ def __init__( self._job_id_event = threading.Event() self._resolved_resource_policy = resource_policy - def reconnect(self, slurm_job_id: str) -> bool | None: - """Check whether a SLURM job is still alive after a server restart. - - We cannot re-attach the srun subprocess, but we can query sacct. - Returns ``True`` when the job is still active, ``False`` when sacct - says it is no longer active, and ``None`` when the state cannot be - determined (sacct missing, query failed, or timed out). Callers must - treat ``None`` as unknown — not as failure — because the job may - still be running on the cluster. - """ - self._slurm_job_id = slurm_job_id - self._job_id_event.set() - sacct = shutil.which("sacct") - if not sacct: - return None - try: - result = subprocess.run( - [sacct, "-j", slurm_job_id, "--noheader", "-o", "State", "-P"], - capture_output=True, - text=True, - timeout=10, - ) - if result.returncode != 0: - return None - state = (result.stdout or "").strip().split("\n")[0].strip() - return state in ("RUNNING", "PENDING", "CONFIGURING") - except Exception: - return None - # -- Job ABC ------------------------------------------------------------- def submit(self) -> str: diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index 3c880a82..5b88bbc4 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -656,7 +656,7 @@ def _capture_debug_submission(task: dict[str, Any], entities: list[dict], params # --------------------------------------------------------------------------- -def _execute_compute_task(md5sum: str, task_type: str = "gremlin", params: dict | None = None, recover_slurm: str | None = None): +def _execute_compute_task(md5sum: str, task_type: str = "gremlin", params: dict | None = None): """Core task logic — shared by legacy and generic Celery task wrappers. Reads entities from the task's ``input_form`` column. The ``params`` @@ -670,10 +670,6 @@ def _execute_compute_task(md5sum: str, task_type: str = "gremlin", params: dict if task["status"] not in {"pending", "queued", "running"}: return - if recover_slurm: - _recover_slurm_task(md5sum, task, recover_slurm) - return - try: tt, runner = _get_task_type(task_type) except KeyError: @@ -815,151 +811,49 @@ def _on_stage_change(stage: str) -> None: # --------------------------------------------------------------------------- -# Orphaned-task recovery after server restart +# Orphaned Docker container recovery after server restart +# +# SLURM tasks need no recovery: restart.sh's pre-stop sweep cancels every +# in-flight SLURM job and marks its task record failed BEFORE the worker +# dies, so no SLURM orphan can ever exist. Docker containers survive the +# stack restart, so only they are reconnected here. # --------------------------------------------------------------------------- -def _recover_slurm_task(md5sum: str, task: dict[str, Any], slurm_job_id: str) -> None: - """Attach to a live SLURM job after a restart instead of resubmitting. - - Polls sacct/squeue with a self-requeue until the job ends, then - finalizes from the result directory. - """ - try: - tt, runner = _get_task_type(task.get("task_type", "gremlin")) - from revocompute.job.runners.slurm_runner import SlurmJob - - job = SlurmJob(md5sum, tt, runner, [], task["result_dir"]) - state = job.reconnect(slurm_job_id) - requeue = { - "args": [md5sum], - "kwargs": {"task_type": task.get("task_type", "gremlin"), "recover_slurm": slurm_job_id}, - "countdown": 30, - } - if state is True: - # The job is alive on the cluster even though the record may - # still say "queued" from before the restart. - if task.get("status") != "running" and not _is_terminal_status(task.get("status")): - task_store.update_task(md5sum, status="running") - run_compute_task.apply_async(**requeue) - return - if state is None: - probe = subprocess.run( - ["squeue", "-h", "-j", slurm_job_id, "-o", "%T"], - capture_output=True, - text=True, - timeout=10, - ) - if probe.returncode == 0 and probe.stdout.strip(): - run_compute_task.apply_async(**requeue) - return - final = JobState.COMPLETED if job._has_result_artifact() else JobState.FAILED - _finalize_after_poll(md5sum, task, tt, final) - except Exception as exc: - logging.error("recover_slurm failed for %s: %s", md5sum, exc) - _record_failure(md5sum, task, task.get("started_at") or time.time(), "", f"Recovery error: {exc}") - - def _recover_orphaned_tasks() -> int: - """Scan for running tasks whose Celery task was lost (e.g. Redis restart). - - Docker containers are reconnected and polled inline. SLURM jobs are - re-queued as lightweight polling tasks. - """ + """Reconnect Docker containers whose Celery task was lost (e.g. Redis restart).""" recovered = 0 for task in task_store.list_tasks(): if task.get("status") not in {"running", "queued"}: continue container_id = str(task.get("container_id") or "") - slurm_job_id = str(task.get("slurm_job_id") or "") - if not container_id and not slurm_job_id: - continue - if slurm_job_id and not slurm_job_id.isdigit(): - # A pid-based fallback ("srun-12") survived into the record: the - # real job handle is lost, so the task can never be reconnected. - logging.warning("Recovery: task %s has a non-SLURM job handle %r; marking failed", task["md5sum"], slurm_job_id) - _record_failure(task["md5sum"], task, task.get("started_at") or time.time(), "", "SLURM job handle was lost during a server restart") - recovered += 1 + if not container_id: continue md5sum = task["md5sum"] task_type = task.get("task_type", "gremlin") logging.info("Recovery: checking orphaned task %s", md5sum) - - from revocompute.task_types import get as _gt - - if container_id: - try: - from revocompute.job.runners.docker_runner import DockerJob - - tt, runner = _gt(task_type) - job = DockerJob(md5sum, tt, runner, [], task["result_dir"]) - if job.reconnect(container_id): - logging.info("Recovery: reconnected Docker %s for %s", container_id, md5sum) - state = job.poll() - _finalize_after_poll(md5sum, task, tt, state) - recovered += 1 - else: - _record_failure( - md5sum, - task, - task.get("started_at") or time.time(), - "", - "Docker container not found after server restart", - ) - except Exception as exc: - logging.error("Recovery failed for Docker task %s: %s", md5sum, exc) - _record_failure(md5sum, task, task.get("started_at") or time.time(), "", f"Recovery error: {exc}") - - elif slurm_job_id: - try: - from revocompute.job.runners.slurm_runner import SlurmJob - - tt, runner = _gt(task_type) - job = SlurmJob(md5sum, tt, runner, [], task["result_dir"]) - slurm_state = job.reconnect(slurm_job_id) - if slurm_state is True: - logging.info("Recovery: SLURM job %s still running for %s, re-queuing poll", slurm_job_id, md5sum) - run_compute_task.apply_async( - args=[md5sum], kwargs={"task_type": task_type, "recover_slurm": slurm_job_id} - ) - recovered += 1 - elif slurm_state is None: - # Unknown sacct state: consult the live queue. A job in - # neither sacct nor squeue was cancelled out from under - # the worker (e.g. the srun client died at a restart). - probe = subprocess.run( - ["squeue", "-h", "-j", slurm_job_id, "-o", "%T"], - capture_output=True, - text=True, - timeout=10, - ) - if probe.returncode == 0 and probe.stdout.strip(): - logging.warning( - "Recovery: SLURM job %s alive in queue for %s; re-queuing poll", slurm_job_id, md5sum - ) - run_compute_task.apply_async( - args=[md5sum], kwargs={"task_type": task_type, "recover_slurm": slurm_job_id} - ) - recovered += 1 - else: - _record_failure( - md5sum, - task, - task.get("started_at") or time.time(), - "", - "SLURM job was cancelled during a server restart", - ) - recovered += 1 - else: - _record_failure( - md5sum, - task, - task.get("started_at") or time.time(), - "", - "SLURM job not found after server restart", - ) - except Exception as exc: - logging.error("Recovery failed for SLURM task %s: %s", md5sum, exc) + try: + from revocompute.job.runners.docker_runner import DockerJob + from revocompute.task_types import get as _gt + + tt, runner = _gt(task_type) + job = DockerJob(md5sum, tt, runner, [], task["result_dir"]) + if job.reconnect(container_id): + logging.info("Recovery: reconnected Docker %s for %s", container_id, md5sum) + state = job.poll() + _finalize_after_poll(md5sum, task, tt, state) + recovered += 1 + else: + _record_failure( + md5sum, + task, + task.get("started_at") or time.time(), + "", + "Docker container not found after server restart", + ) + except Exception as exc: + logging.error("Recovery failed for Docker task %s: %s", md5sum, exc) + _record_failure(md5sum, task, task.get("started_at") or time.time(), "", f"Recovery error: {exc}") return recovered @@ -1016,13 +910,9 @@ def _on_worker_ready(sender, **kwargs): @celery.task(name="run_compute_task", bind=True, max_retries=0) -def run_compute_task(self, md5sum: str, task_type: str = "gremlin", params: dict | None = None, recover_slurm: str | None = None): - """Compute task — dispatched by task_type. - - ``recover_slurm`` attaches to an already-running SLURM job after a - server restart instead of submitting a new one. - """ - return _execute_compute_task(md5sum, task_type, params, recover_slurm) +def run_compute_task(self, md5sum: str, task_type: str = "gremlin", params: dict | None = None): + """Compute task — dispatched by task_type.""" + return _execute_compute_task(md5sum, task_type, params) @celery.task(name="cancel_compute_resources", bind=True, max_retries=0) diff --git a/server/run/restart.sh b/server/run/restart.sh index 1f7233ec..a90343b5 100755 --- a/server/run/restart.sh +++ b/server/run/restart.sh @@ -1010,10 +1010,49 @@ cmd_up() { print_admin_logins } +# Kill and mark every in-flight SLURM task before the worker dies. The +# worker owns the srun clients, so stopping the stack orphans anything +# running; a clean pre-stop sweep means no orphan can ever exist and no +# boot-time recovery machinery is needed. +pre_stop_sweep_slurm() { + if [[ "${USE_SLURM}" != "1" ]]; then return 0; fi + if ! command -v squeue >/dev/null 2>&1 || ! command -v scancel >/dev/null 2>&1; then + echo "WARNING: squeue/scancel not found; skipping pre-stop SLURM sweep" >&2 + return 0 + fi + local jobs + jobs=$(squeue -h -u "${RUNNER_USERNAME}" -o '%i' 2>/dev/null || true) + if [[ -n "${jobs}" ]]; then + echo "Cancelling in-flight SLURM jobs before stopping the stack: ${jobs}" + # shellcheck disable=SC2086 + scancel ${jobs} + fi + # Mark the affected task records failed so the dashboard never shows ghosts. + local task_db="${SERVER_DIR}/revocompute.sqlite3" + if [[ -f "${task_db}" ]] && command -v python3 >/dev/null 2>&1; then + python3 - "${task_db}" <<'PY' +import sqlite3, sys, time +db = sqlite3.connect(sys.argv[1], timeout=15) +try: + db.execute( + "UPDATE tasks SET status='failed', error='Cancelled by server restart', " + "finished_at=? WHERE status IN ('pending','queued','running')", + (time.time(),), + ) + db.commit() +except sqlite3.Error as exc: + print(f"WARNING: pre-stop task sweep failed: {exc}", file=sys.stderr) +finally: + db.close() +PY + fi +} + cmd_down() { require_env_file ensure_docker_gid resolve_runner_identity + pre_stop_sweep_slurm echo "Stopping services via docker compose..." "${COMPOSE_CMD[@]}" $(compose_files) --env-file "${ENV_FILE}" down } diff --git a/server/tests/test_slurm_runner.py b/server/tests/test_slurm_runner.py index 5f4d2c0b..6338986d 100644 --- a/server/tests/test_slurm_runner.py +++ b/server/tests/test_slurm_runner.py @@ -178,27 +178,6 @@ def resolve_task_resources(self, tool, *, requires_gpu, default_timeout_seconds) # -- apptainer invocation ----------------------------------------------------- -def test_reconnect_returns_none_when_sacct_missing(monkeypatch, tmp_path): - """Unknown SLURM job state is tri-state — missing sacct must not read as - 'job finished' (recovery keeps the task running and its workspace).""" - from revocompute.job.runners.slurm_runner import SlurmJob - - job = SlurmJob("task-1", _make_task_type(), _make_runner(), _make_entities(), str(tmp_path / "out")) - monkeypatch.setattr(shutil, "which", lambda _name: None) - assert job.reconnect("12345") is None - - -def test_reconnect_returns_none_when_sacct_fails(monkeypatch, tmp_path): - from revocompute.job.runners.slurm_runner import SlurmJob - - job = SlurmJob("task-1", _make_task_type(), _make_runner(), _make_entities(), str(tmp_path / "out")) - monkeypatch.setattr(shutil, "which", lambda _name: "/usr/bin/sacct") - monkeypatch.setattr( - subprocess, - "run", - lambda *a, **kw: type("R", (), {"returncode": 1, "stdout": ""})(), - ) - assert job.reconnect("12345") is None def test_render_apptainer_binds_and_env(tmp_path): From 96a339994d2e09da5fda15c4bd9c049105373787 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 10:48:11 +0800 Subject: [PATCH 19/26] fix(server): run the pre-stop sweep inside the worker container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host account can neither scancel the runner's jobs (SLURM permissions) nor write the task DB (readonly file) — the sweep must go through the worker container, which owns the srun clients, runs as the SLURM user, and has DB write access. Co-Authored-By: Claude --- server/run/restart.sh | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/server/run/restart.sh b/server/run/restart.sh index a90343b5..46a8722a 100755 --- a/server/run/restart.sh +++ b/server/run/restart.sh @@ -1016,36 +1016,29 @@ cmd_up() { # boot-time recovery machinery is needed. pre_stop_sweep_slurm() { if [[ "${USE_SLURM}" != "1" ]]; then return 0; fi - if ! command -v squeue >/dev/null 2>&1 || ! command -v scancel >/dev/null 2>&1; then - echo "WARNING: squeue/scancel not found; skipping pre-stop SLURM sweep" >&2 - return 0 - fi + # The worker container owns the srun clients, runs as the SLURM user, and + # holds write access to the task DB — the host account has none of the + # three, so the whole sweep runs inside the containers before down. local jobs jobs=$(squeue -h -u "${RUNNER_USERNAME}" -o '%i' 2>/dev/null || true) if [[ -n "${jobs}" ]]; then echo "Cancelling in-flight SLURM jobs before stopping the stack: ${jobs}" # shellcheck disable=SC2086 - scancel ${jobs} + "${COMPOSE_CMD[@]}" $(compose_files) --env-file "${ENV_FILE}" exec -T worker scancel ${jobs} || true fi - # Mark the affected task records failed so the dashboard never shows ghosts. - local task_db="${SERVER_DIR}/revocompute.sqlite3" - if [[ -f "${task_db}" ]] && command -v python3 >/dev/null 2>&1; then - python3 - "${task_db}" <<'PY' -import sqlite3, sys, time -db = sqlite3.connect(sys.argv[1], timeout=15) -try: - db.execute( - "UPDATE tasks SET status='failed', error='Cancelled by server restart', " - "finished_at=? WHERE status IN ('pending','queued','running')", - (time.time(),), - ) - db.commit() -except sqlite3.Error as exc: - print(f"WARNING: pre-stop task sweep failed: {exc}", file=sys.stderr) -finally: - db.close() + echo "Marking in-flight tasks failed before stopping the stack..." + "${COMPOSE_CMD[@]}" $(compose_files) --env-file "${ENV_FILE}" exec -T worker python3 - <<'PY' || true +import time +from revocompute.task_runtime import task_store +for task in task_store.list_tasks(): + if task.get("status") in {"pending", "queued", "running"}: + task_store.update_task( + task["md5sum"], + status="failed", + error="Cancelled by server restart", + finished_at=time.time(), + ) PY - fi } cmd_down() { From 9e205606b7ee8ac1209e1d71cafe9407057d807a Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:26:42 +0800 Subject: [PATCH 20/26] fix(server): log every stage callback for the run_stage debugging pass The translator markers reach the runner (verified in the captured stdout), but the task record never persisted intermediate stages. This instrumentation will show which callbacks fire and what the dedup sees. Co-Authored-By: Claude --- server/revocompute/task_runtime.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index 5b88bbc4..ae9e0859 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -742,8 +742,10 @@ def _on_stage_change(stage: str) -> None: if _task_is_terminal(md5sum): return stage_changed = stage != stage_state["current"] + is_first = stage_state.get("first") + logging.info("Stage callback for task %s: stage=%s changed=%s first=%s", md5sum, stage, stage_changed, is_first) stage_state["current"] = stage - if stage_state.get("first"): + if is_first: stage_state["first"] = False task_store.update_task(md5sum, status="running", run_stage=stage) elif stage_changed: From dd069380fbb10df985579bd6a4417d5b7d610954 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:51:50 +0800 Subject: [PATCH 21/26] fix(server): address PR review findings --- CHANGELOG.md | 2 + server/docker/runners/alphafold/run.sh | 27 ++++++- .../revocompute/job/runners/slurm_runner.py | 4 +- server/revocompute/static/js/dashboard.js | 25 +++--- server/revocompute/static/js/task-results.js | 80 +++++++++++-------- server/revocompute/task_runtime.py | 4 +- server/tests/js/test_viewer_shell.js | 16 +++- server/tests/test_runner_script_static.py | 59 +++++++++++++- server/tests/test_slurm_runner.py | 2 +- server/tests/test_tasks.py | 16 ++++ 10 files changed, 182 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b9a4c9d..df185cca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Server: + - Result polling/viewers: terminal task states reload once without overlapping polls; pending result pages keep polling; Mol* teardown completes before iframe removal and preview loaders remain visible. + - AlphaFold stages: drain the stderr translator before wrapper exit and preserve both process statuses so final stage markers cannot be lost. - SLURM status: move allocated tasks from queued to running as soon as the wrapper publishes the real job ID, without waiting for scientific stage output. - RFdiffusion runner: redirect Hydra bookkeeping and generated schedule caches to the per-job writable `/tmp` filesystem under read-only container execution; lock task defaults to the pinned upstream inference configuration. - Mol* input workbenches: sequence-strip and canvas selections now report canonical residues; cached viewer-shell handshakes no longer race listener setup. diff --git a/server/docker/runners/alphafold/run.sh b/server/docker/runners/alphafold/run.sh index c45fd7d2..f9267891 100644 --- a/server/docker/runners/alphafold/run.sh +++ b/server/docker/runners/alphafold/run.sh @@ -58,9 +58,30 @@ fi cd "${ALPHAFOLD_PATH:-/opt/alphafold}" # AlphaFold logs its phases to stderr (absl logging); the shared translator # rewrites the stable ones into the stdout stage protocol while passing the -# original lines through to the stderr log unchanged. -python3 run_alphafold.py "${af_args[@]}" 2> >(awk -f /app/revocompute/stage_translate.awk \ - -v PATTERNS=/app/revocompute/alphafold.stages >&1) +# original lines through to the stderr log unchanged. A FIFO makes the +# translator an ordinary child that can be drained before the wrapper exits. +stage_tmp=$(mktemp -d "${TMPDIR:-/tmp}/revodesign-alphafold-stage.XXXXXX") +stage_fifo="${stage_tmp}/stderr" +mkfifo "${stage_fifo}" +cleanup_stage_pipe() { rm -rf -- "${stage_tmp}"; } +trap cleanup_stage_pipe EXIT +awk -f "${ALPHAFOLD_STAGE_TRANSLATOR:-/app/revocompute/stage_translate.awk}" \ + -v PATTERNS="${ALPHAFOLD_STAGE_PATTERNS:-/app/revocompute/alphafold.stages}" \ + < "${stage_fifo}" >&1 & +translator_pid=$! +set +e +"${ALPHAFOLD_PYTHON:-python3}" run_alphafold.py "${af_args[@]}" 2> "${stage_fifo}" +alphafold_status=$? +wait "${translator_pid}" +translator_status=$? +set -e +cleanup_stage_pipe +trap - EXIT +if [[ ${alphafold_status} -ne 0 || ${translator_status} -ne 0 ]]; then + echo "AlphaFold exited ${alphafold_status}; stage translator exited ${translator_status}" >&2 + [[ ${alphafold_status} -ne 0 ]] && exit "${alphafold_status}" + exit "${translator_status}" +fi [[ -n "$(ls "${output_dir}"/*/ranked_0.pdb 2>/dev/null || true)" ]] || { echo "AlphaFold produced no ranked_0.pdb" >&2; exit 1; } diff --git a/server/revocompute/job/runners/slurm_runner.py b/server/revocompute/job/runners/slurm_runner.py index 44ae4547..814d61bc 100644 --- a/server/revocompute/job/runners/slurm_runner.py +++ b/server/revocompute/job/runners/slurm_runner.py @@ -307,8 +307,8 @@ def emit_stage(stage: str) -> None: last_stage = stage try: self.stage_callback(stage) - except Exception as exc: # surface, never mask status updates - logging.error("Stage callback failed for task %s: %s", self.task_id, exc) + except Exception: # surface, never mask status updates + logging.exception("Stage callback failed for task %s", self.task_id) for line in iter(stream.readline, ""): self._stdout_lines.append(line) diff --git a/server/revocompute/static/js/dashboard.js b/server/revocompute/static/js/dashboard.js index a3289016..e6eb73fb 100644 --- a/server/revocompute/static/js/dashboard.js +++ b/server/revocompute/static/js/dashboard.js @@ -258,26 +258,33 @@ // Status pills are rendered once at load; poll the non-terminal ones so a // queued/running badge updates itself, and reload when a task finishes so // the results buttons and traces appear without a manual refresh. + var terminalStatuses = ["finished", "failed", "cancelled", "deleted", "deleted:finshed", "deleted:cancel"]; + var statusPollInFlight = false; async function pollStatuses() { + if (statusPollInFlight) return; + statusPollInFlight = true; var pills = document.querySelectorAll(".status-pill[data-task-status]"); - for (var index = 0; index < pills.length; index += 1) { - var pill = pills[index]; - var status = pill.dataset.taskStatus; - if (status === "finished" || status === "failed" || status === "cancelled" || status === "deleted") continue; - try { + try { + for (var index = 0; index < pills.length; index += 1) { + var pill = pills[index]; + var status = pill.dataset.taskStatus; + if (terminalStatuses.indexOf(status) !== -1) continue; var response = await A.authFetch("/compute/api/running/" + encodeURIComponent(pill.dataset.md5)); var payload = await response.json().catch(function () { return {}; }); - if (!response.ok || !payload.status || payload.status === status) continue; + if (!payload.status || payload.status === status) continue; + var isTerminal = terminalStatuses.indexOf(payload.status) !== -1; + if (!response.ok && !isTerminal) continue; var meta = getStatusMeta(payload.status); pill.textContent = meta.label; pill.className = "status-pill " + meta.css; pill.dataset.taskStatus = payload.status; - if (payload.status === "finished" || payload.status === "failed" || payload.status === "cancelled" || payload.status === "deleted") { + if (isTerminal) { window.location.reload(); return; } - } catch (error) { /* transient — retry next tick */ } - } + } + } catch (error) { /* transient — retry next tick */ } + finally { statusPollInFlight = false; } } setInterval(pollStatuses, 15000); diff --git a/server/revocompute/static/js/task-results.js b/server/revocompute/static/js/task-results.js index e48bd966..4025bfe9 100644 --- a/server/revocompute/static/js/task-results.js +++ b/server/revocompute/static/js/task-results.js @@ -92,23 +92,30 @@ if (pending.reject) { try { pending.reject(new Error("Viewer disposed")); } catch (e) { /* already settled */ } } delete warmPending[key]; }); - if (!frame) return; + if (!frame) return Promise.resolve(); // Give the shell a moment to run its own teardown (selection unsubscribe // + plugin.dispose) before the iframe is detached; the shell acknowledges // with a "disposed" message, and a 2s timeout guarantees the frame never // lingers when the shell is already gone. - var removed = false; - var timer = setTimeout(function () { if (!removed) { removed = true; frame.remove(); } }, 2000); - window.addEventListener("message", function onDisposed(event) { - if (removed || event.source !== frame.contentWindow || !event.data || event.data.type !== "disposed") return; - removed = true; - clearTimeout(timer); - frame.remove(); - window.removeEventListener("message", onDisposed); + return new Promise(function (resolve) { + var removed = false; + var timer = null; + var removeFrame = function () { + if (removed) return; + removed = true; + clearTimeout(timer); + window.removeEventListener("message", onDisposed); + frame.remove(); + resolve(); + }; + var onDisposed = function (event) { + if (removed || event.source !== frame.contentWindow || !event.data || event.data.type !== "disposed") return; + removeFrame(); + }; + timer = setTimeout(removeFrame, 2000); + window.addEventListener("message", onDisposed); + try { postToShell(frame, { type: "dispose" }); } catch (e) { removeFrame(); } }); - try { postToShell(frame, { type: "dispose" }); } catch (e) { - if (!removed) { removed = true; clearTimeout(timer); frame.remove(); } - } } // One shared message listener for the warm shell: resolves the pending @@ -394,7 +401,7 @@ if (isStale(generation)) return; // A dead warm frame must not poison the next pick: dispose it so the // retry cold-starts a fresh shell. - disposeActiveViewer(); + await disposeActiveViewer(); surface.replaceChildren(); surface.appendChild(structureViewerBar(artifact)); var msg = document.createElement("p"); @@ -572,9 +579,9 @@ stage.hidden = true; structureHolder.hidden = false; } - showLoading(stage, "Loading preview…"); try { stage.replaceChildren(); + showLoading(stage, "Loading preview…"); await previewHost.render(artifact); } catch (error) { stage.innerHTML = '

'; @@ -767,35 +774,44 @@ } async function loadResults() { - disposeActiveViewer(); + await disposeActiveViewer(); structureTextCache.clear(); structureTextCacheBytes = 0; if (structureHolder) structureHolder.hidden = true; var response = await A.authFetch("/compute/api/results/" + encodeURIComponent(task.md5)); var payload = await response.json().catch(function () { return {}; }); document.getElementById("resultStatus").textContent = payload.status || task.status; + // Status is set once at load; poll while the task is pending so a + // queued/running page updates itself and reloads when results land. + if (window.__revocomputeStatusPoll) clearInterval(window.__revocomputeStatusPoll); + var initialStatus = payload.status || task.status; + var terminalStatuses = ["finished", "failed", "cancelled", "deleted", "deleted:finshed", "deleted:cancel"]; + var statusPollInFlight = false; + if (terminalStatuses.indexOf(initialStatus) === -1) { + window.__revocomputeStatusPoll = setInterval(async function () { + if (statusPollInFlight) return; + statusPollInFlight = true; + try { + var pollResponse = await A.authFetch("/compute/api/running/" + encodeURIComponent(task.md5)); + var pollPayload = await pollResponse.json().catch(function () { return {}; }); + var isTerminal = terminalStatuses.indexOf(pollPayload.status) !== -1; + if (!pollResponse.ok && !isTerminal) return; + var statusEl = document.getElementById("resultStatus"); + if (statusEl && pollPayload.status) statusEl.textContent = pollPayload.status; + if (isTerminal) { + clearInterval(window.__revocomputeStatusPoll); + window.location.reload(); + } + } catch (error) { /* transient network hiccup — retry next tick */ } + finally { statusPollInFlight = false; } + }, 15000); + } if (!response.ok || !Array.isArray(payload.artifacts)) { + if (response.ok && terminalStatuses.indexOf(initialStatus) === -1) return; throw new Error(payload.message || "Results are not available yet"); } artifacts = payload.artifacts; resultViews = Array.isArray(payload.views) ? payload.views : []; - // Status is set once at load; poll while the task is pending so a - // queued/running page updates itself and reloads when results land. - if (window.__revocomputeStatusPoll) clearInterval(window.__revocomputeStatusPoll); - window.__revocomputeStatusPoll = setInterval(async function () { - try { - var pollResponse = await A.authFetch("/compute/api/results/" + encodeURIComponent(task.md5)); - var pollPayload = await pollResponse.json().catch(function () { return {}; }); - if (!pollResponse.ok) return; - var statusEl = document.getElementById("resultStatus"); - if (statusEl && pollPayload.status) statusEl.textContent = pollPayload.status; - if (pollPayload.status === "finished" || pollPayload.status === "failed" || - pollPayload.status === "cancelled" || pollPayload.status === "deleted") { - clearInterval(window.__revocomputeStatusPoll); - window.location.reload(); - } - } catch (error) { /* transient network hiccup — retry next tick */ } - }, 15000); document.getElementById("resultFileCount").textContent = artifacts.length; document.getElementById("resultTotalSize").textContent = formatBytes(payload.total_size); var archiveButton = document.getElementById("archiveButton"); diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index ae9e0859..4f84c4ba 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -899,8 +899,8 @@ def _on_worker_ready(sender, **kwargs): logging.info("Recovered %d orphaned task(s)", count) else: logging.info("Recovery: no orphaned tasks found") - except Exception as exc: # boot-time recovery must never die silently - logging.error("Recovery pass failed: %s", exc) + except Exception: # boot-time recovery must never die silently + logging.exception("Recovery pass failed") except ImportError: pass # celery.signals not available in all environments diff --git a/server/tests/js/test_viewer_shell.js b/server/tests/js/test_viewer_shell.js index 978e7ba1..a110983a 100644 --- a/server/tests/js/test_viewer_shell.js +++ b/server/tests/js/test_viewer_shell.js @@ -26,6 +26,7 @@ async function main() { var host = fakeNode("div"); host.hidden = true; var reports = []; + var teardownEvents = []; var listeners = {}; var loadedStructure = null; var loadedStructureCount = 0; @@ -38,7 +39,10 @@ async function main() { var componentB = { id: "component-b" }; var parentWindow = { origin: "https://revocompute.example", - postMessage: function (payload, targetOrigin) { reports.push({ payload: payload, targetOrigin: targetOrigin }); } + postMessage: function (payload, targetOrigin) { + reports.push({ payload: payload, targetOrigin: targetOrigin }); + if (payload.type === "disposed") teardownEvents.push("reported"); + } }; var document = { head: fakeNode("head"), @@ -68,7 +72,7 @@ async function main() { layoutShowControls = options.layoutShowControls; viewerPlugin = { clear: async function () {}, - dispose: function () {}, + dispose: function () { teardownEvents.push("disposed"); }, canvas3d: { setProps: function (props) { canvasBackground = props.renderer.backgroundColor; } }, @@ -88,7 +92,10 @@ async function main() { selection: { events: { changed: { - subscribe: function (handler) { selectionChanged.push(handler); return { unsubscribe: function () {} }; } + subscribe: function (handler) { + selectionChanged.push(handler); + return { unsubscribe: function () { teardownEvents.push("unsubscribed"); } }; + } } }, getLoci: function (structure) { @@ -273,6 +280,9 @@ async function main() { if (!reports.some(function (entry) { return entry.payload.type === "disposed"; })) { throw new Error("shell did not acknowledge disposal"); } + if (teardownEvents.join(",") !== "unsubscribed,disposed,reported") { + throw new Error("shell reported disposal before teardown: " + teardownEvents.join(",")); + } console.log("viewer shell message contract passed"); } diff --git a/server/tests/test_runner_script_static.py b/server/tests/test_runner_script_static.py index 451df0c1..c4aabfc2 100644 --- a/server/tests/test_runner_script_static.py +++ b/server/tests/test_runner_script_static.py @@ -12,6 +12,9 @@ RUNNER_SCRIPT = Path(__file__).resolve().parents[1] / "docker" / "runners" / "pssm_gremlin" / "run.sh" OPENDDE_RUNNER_SCRIPT = Path(__file__).resolve().parents[1] / "docker" / "runners" / "opendde" / "run.sh" MPNN_RUNNER_SCRIPT = Path(__file__).resolve().parents[1] / "docker" / "runners" / "mpnn" / "run.sh" +ALPHAFOLD_RUNNER_SCRIPT = ( + Path(__file__).resolve().parents[1] / "docker" / "runners" / "alphafold" / "run.sh" +) SERVER_ROOT = Path(__file__).resolve().parents[1] @@ -65,7 +68,10 @@ def _run_with_manifest(script, input_file, output_dir, env, params=None): encoding="utf-8", ) env["TASK_MANIFEST"] = str(manifest_path) - env["TASK_CONTEXT_SRC"] = str(SERVER_ROOT / "docker" / "runners" / "common" / "task_context.sh") + env.setdefault( + "TASK_CONTEXT_SRC", + str(SERVER_ROOT / "docker" / "runners" / "common" / "task_context.sh"), + ) return subprocess.run( ["bash", str(script), "-i", str(manifest_path), "-o", str(output_dir)], env=env, @@ -75,6 +81,57 @@ def _run_with_manifest(script, input_file, output_dir, env, params=None): ) +def test_alphafold_runner_drains_final_stage_before_exit(tmp_path): + input_file = tmp_path / "input.fasta" + output_dir = tmp_path / "outputs" + alphafold_root = tmp_path / "alphafold" + fake_context = tmp_path / "task_context.sh" + fake_python = tmp_path / "fake-python" + delayed_translator = tmp_path / "delayed-stage.awk" + patterns = tmp_path / "alphafold.stages" + input_file.write_text(">test\nAAAA\n", encoding="utf-8") + output_dir.mkdir() + alphafold_root.mkdir() + fake_context.write_text( + '_parse_param() { printf "%s\\n" "$2"; }\n' + 'primary_input() { printf "%s\\n" "$FAKE_PRIMARY_INPUT"; }\n', + encoding="utf-8", + ) + fake_python.write_text( + "#!/bin/bash\n" + "set -e\n" + 'for arg in "$@"; do case "$arg" in --output_dir=*) output_dir=${arg#*=} ;; esac; done\n' + 'printf "Running model model_1\\n" >&2\n' + 'mkdir -p "$output_dir/model"\n' + 'printf "MODEL\\n" > "$output_dir/model/ranked_0.pdb"\n', + encoding="utf-8", + ) + fake_python.chmod(0o755) + delayed_translator.write_text( + '{ print > "/dev/stderr"; system("sleep 0.2"); print "REVODESIGN_STAGE:modeling"; fflush() }\n', + encoding="utf-8", + ) + patterns.write_text("modeling:Running model model_\n", encoding="utf-8") + + env = os.environ.copy() + env.update( + { + "ALPHAFOLD_PATH": str(alphafold_root), + "ALPHAFOLD_PYTHON": str(fake_python), + "ALPHAFOLD_STAGE_TRANSLATOR": str(delayed_translator), + "ALPHAFOLD_STAGE_PATTERNS": str(patterns), + "FAKE_PRIMARY_INPUT": str(input_file), + "TASK_CONTEXT_SRC": str(fake_context), + "TMPDIR": str(tmp_path), + } + ) + completed = _run_with_manifest(ALPHAFOLD_RUNNER_SCRIPT, input_file, output_dir, env) + + assert completed.returncode == 0, completed.stderr + assert completed.stdout.index("REVODESIGN_STAGE:modeling") < completed.stdout.index("AlphaFold complete.") + assert (output_dir / "task_finished").is_file() + + def test_opendde_runner_uses_writable_snapshot_copy_and_checks_outputs(): script = OPENDDE_RUNNER_SCRIPT.read_text() diff --git a/server/tests/test_slurm_runner.py b/server/tests/test_slurm_runner.py index 6338986d..798305b5 100644 --- a/server/tests/test_slurm_runner.py +++ b/server/tests/test_slurm_runner.py @@ -304,7 +304,7 @@ def test_job_id_capture_emits_first_stage_as_liveness_signal(tmp_path): str(tmp_path / "out"), stage_callback=stages_seen.append, ) - stdout = StringIO("REVODESIGN_JOB_ID=4154\n" "REVODESIGN_STAGE:hhblits\n" "REVODESIGN_STAGE:gremlin\n") + stdout = StringIO("REVODESIGN_JOB_ID=4154\n" "REVODESIGN_STAGE:gremlin\n") job._process = SimpleNamespace(stdout=stdout) job._read_stdout() diff --git a/server/tests/test_tasks.py b/server/tests/test_tasks.py index 9a2552a7..a654bd97 100644 --- a/server/tests/test_tasks.py +++ b/server/tests/test_tasks.py @@ -160,6 +160,22 @@ def test_dashboard_links_to_dedicated_manifest_first_result_workspace(): assert "buildArtifactTree" in script +def test_result_status_polling_handles_terminal_and_pending_responses(): + dashboard = (SERVER_PACKAGE / "static" / "js" / "dashboard.js").read_text(encoding="utf-8") + results = (SERVER_PACKAGE / "static" / "js" / "task-results.js").read_text(encoding="utf-8") + terminal_statuses = '["finished", "failed", "cancelled", "deleted", "deleted:finshed", "deleted:cancel"]' + + assert terminal_statuses in dashboard + assert terminal_statuses in results + assert "if (statusPollInFlight) return;" in dashboard + assert "if (!response.ok && !isTerminal) continue;" in dashboard + assert 'A.authFetch("/compute/api/running/"' in results + assert "if (!pollResponse.ok && !isTerminal) return;" in results + assert results.index("window.__revocomputeStatusPoll = setInterval") < results.index( + "if (!response.ok || !Array.isArray(payload.artifacts))" + ) + + def test_execution_logs_are_diagnostic_text_artifacts_not_main_results(): runtime = (SERVER_PACKAGE / "task_runtime.py").read_text(encoding="utf-8") results = (SERVER_PACKAGE / "static" / "js" / "task-results.js").read_text(encoding="utf-8") From dbd30cf13c2c5cbbf9cb9694d17419a806bf6253 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:16:32 +0800 Subject: [PATCH 22/26] docs: correct REvoCompute edge topology [skip ci] --- CLAUDE.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3ffb6afa..eb5a6c16 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -61,7 +61,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co profiles configured opposite to project conventions): don't fix, don't debate, don't treat them as blocking. No branch protection gates on them. Their findings get handled periodically in dedicated batch-fix PRs. - 3. **Server PRs** (`server/` — REvoCompute): deploy to the live SLURM server with `REVODESIGN_SERVER_ENV=/repo/REvoDesign/server/.env.production.v7-slurm bash server/run/restart.sh restart --use-proxy` (absolute env path, exactly ONE restart running at a time). Before restarting, sync `server/config/task_types.yaml` to the production `CONFIG_DIR` copy (back it up, copy it over, re-apply the two machine lines `job_executor: slurm` / `container_runtime: apptainer`). New task types or runtime families must be appended to `ENABLED_TASKRUNNERS` in the deployment env file before activation — the server only enables registered-and-listed task types. When runner images changed: delete the old SIF under `/mnt/data/srv/revodesign/server-slurm/images/` and add `--build-sif` to the restart — no `.sif.partial` versioning, one SIF per family. Disk-full recovery: `docker buildx prune`, `APPTAINER_CACHEDIR=/home/yinying/.apptainer/ apptainer cache clean --type all`, remove obsolete SIFs. Then live-verify the affected pages on `https://revocompute.yaoyy.moe` and `https://revocompute-direct.yaoyy.moe` in incognito (cache-free) — one edge can be briefly down after a restart; try the other. When behavior changed, submit a living test with a real data file from `tests/data` through the API using the group test account (ask the user for credentials if none are in session or memory): login `POST /compute/api/auth/login` with `{"username": …, "password": …}` → Bearer token; submit `POST /compute/api/post` (multipart `file` + `task_type` + `params[name]`/`workspace`); monitor the local SLURM job (`squeue`) and read results from the API — status `GET /compute/api/running/`, manifest `GET /compute/api/results/`, logs `GET /compute/api/results//artifacts/`. Verify the served static files contain the change AND the page behaves as designed, never just one of the two. + 3. **Server PRs** (`server/` — REvoCompute): deploy to the live SLURM server with `REVODESIGN_SERVER_ENV=/repo/REvoDesign/server/.env.production.v7-slurm bash server/run/restart.sh restart --use-proxy` (absolute env path, exactly ONE restart running at a time). Before restarting, sync `server/config/task_types.yaml` to the production `CONFIG_DIR` copy (back it up, copy it over, re-apply the two machine lines `job_executor: slurm` / `container_runtime: apptainer`). New task types or runtime families must be appended to `ENABLED_TASKRUNNERS` in the deployment env file before activation — the server only enables registered-and-listed task types. When runner images changed: delete the old SIF under `/mnt/data/srv/revodesign/server-slurm/images/` and add `--build-sif` to the restart — no `.sif.partial` versioning, one SIF per family. Disk-full recovery: `docker buildx prune`, `APPTAINER_CACHEDIR=/home/yinying/.apptainer/ apptainer cache clean --type all`, remove obsolete SIFs. Then live-verify the affected pages on the canonical direct edge `https://revocompute.yaoyy.moe` in incognito (cache-free); `https://revocompute-relay.yaoyy.moe` is the optional relay edge and is not a deployment blocker. When behavior changed, submit a living test with a real data file from `tests/data` through the API using the group test account (ask the user for credentials if none are in session or memory): login `POST /compute/api/auth/login` with `{"username": …, "password": …}` → Bearer token; submit `POST /compute/api/post` (multipart `file` + `task_type` + `params[name]`/`workspace`); monitor the local SLURM job (`squeue`) and read results from the API — status `GET /compute/api/running/`, manifest `GET /compute/api/results/`, logs `GET /compute/api/results//artifacts/`. Verify the served static files contain the change AND the page behaves as designed, never just one of the two. 4. **Main program PRs** (PyMOL plugin): CI and review comments only — no server deploy. Run the relevant gates (`make kw-test PYTEST_KW=''`); cross-Qt checks exist in `REvoDesignTestFlight` (PyQt5) and `REvoDesignTestFlightQt6`. 5. When CI is green and every comment is fixed or debated, push a final empty marker commit `chore: Done fixing — ` as the branch head (the plain `Done fixing` prefix is an intentional exception to the conventional-commit rule, matching the squash-merge habit); the user squash-merges from there. - **Standalone bootstrapper encoding**: Keep `src/REvoDesign/tools/package_manager.py` ASCII-only because it is published as `REvoDesign_PyMOL.py` and may be saved through locale-aware Windows tools. A UTF-8-to-GBK transcode turns characters such as `→` into bytes beginning with `0xA1`, which Python 3 rejects while parsing the file as UTF-8. Preserve both the GBK-compilation regression test and the `check-standalone-source-ascii` pre-commit guard; non-ASCII text remains acceptable in normal packaged modules. diff --git a/README.md b/README.md index eec3abe0..4e6c1b6a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ | CI status | [![Unit Test on tagging](https://github.com/YaoYinYing/REvoDesign/actions/workflows/unit_tests_tag.yml/badge.svg)](https://github.com/YaoYinYing/REvoDesign/actions/workflows/unit_tests_tag.yml) [![Docker Image](https://github.com/YaoYinYing/REvoDesign/actions/workflows/docker-image.yml/badge.svg)](https://github.com/YaoYinYing/REvoDesign/actions/workflows/docker-image.yml) [![Docs](https://github.com/YaoYinYing/REvoDesign/actions/workflows/docs.yml/badge.svg)](https://github.com/YaoYinYing/REvoDesign/actions/workflows/docs.yml)| | Code Quality Status | ![GitHub repo size](https://img.shields.io/github/repo-size/YaoYinYing/REvoDesign?logo=github&color=green) [![pylint](https://github-image-cache.yaoyy.moe/badge_dir_with_uniq_name/REvoDesign/pylint/pylint_scan.svg)](https://github.com/Silleellie/pylint-github-action) [![Maintainability](https://qlty.sh/badges/8a669407-26a2-45aa-8323-b5567c391cbf/maintainability.svg)](https://qlty.sh/gh/YaoYinYing/projects/REvoDesign) [![codecov](https://codecov.io/gh/YaoYinYing/REvoDesign/graph/badge.svg?token=2qSJ7cgk1b)](https://codecov.io/gh/YaoYinYing/REvoDesign) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/05f9fa7eb01e4bba9bf0063182513f82)](https://app.codacy.com/gh/YaoYinYing/REvoDesign/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) [![DeepSource](https://app.deepsource.com/gh/YaoYinYing/REvoDesign.svg/?label=active+issues&show_trend=true&token=OGYjIrCzCfLD9zjBpSXPCMeu)](https://app.deepsource.com/gh/YaoYinYing/REvoDesign/) [![DeepSource](https://app.deepsource.com/gh/YaoYinYing/REvoDesign.svg/?label=resolved+issues&show_trend=true&token=OGYjIrCzCfLD9zjBpSXPCMeu)](https://app.deepsource.com/gh/YaoYinYing/REvoDesign/) [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FYaoYinYing%2FREvoDesign.svg?type=shield&issueType=license)](https://app.fossa.com/projects/git%2Bgithub.com%2FYaoYinYing%2FREvoDesign?ref=badge_shield&issueType=license) [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FYaoYinYing%2FREvoDesign.svg?type=shield&issueType=security)](https://app.fossa.com/projects/git%2Bgithub.com%2FYaoYinYing%2FREvoDesign?ref=badge_shield&issueType=security)| | Server Docker Image | [![Docker Image Size](https://img.shields.io/docker/image-size/yaoyinying/revodesign-pssm-gremlin-non-root?style=social&logo=docker&label=Runner)](https://hub.docker.com/r/yaoyinying/revodesign-pssm-gremlin-non-root) [![Docker Image Size](https://img.shields.io/docker/image-size/yaoyinying/revodesign-pssm-gremlin-server-non-root?style=social&logo=docker&label=Server)](https://hub.docker.com/r/yaoyinying/revodesign-pssm-gremlin-server-non-root) | -| Server Status | **REvoCompute (New)**: [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevocompute.yaoyy.moe%2Fcompute%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Main&link=https%3A%2F%2Frevocompute.yaoyy.moe%2Fcompute%2Fdashboard)](https://revocompute.yaoyy.moe/compute/dashboard) [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevocompute-direct.yaoyy.moe%2Fcompute%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Backup&link=https%3A%2F%2Frevocompute-direct.yaoyy.moe%2Fcompute%2Fdashboard)](https://revocompute-direct.yaoyy.moe/compute/dashboard)
*PSSM/GREMLIN (Legacy)*: [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevodesign.yaoyy.moe%2FPSSM_GREMLIN%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Main%20-%20Legacy&link=https%3A%2F%2Frevodesign.yaoyy.moe%2FPSSM_GREMLIN%2Fdashboard)](https://revodesign.yaoyy.moe/PSSM_GREMLIN/dashboard) [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevodesign-direct.yaoyy.moe%2FPSSM_GREMLIN%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Backup%20-%20Legacy&link=https%3A%2F%2Frevodesign.yaoyy.moe%2FPSSM_GREMLIN%2Fdashboard)](https://revodesign-direct.yaoyy.moe/PSSM_GREMLIN/dashboard)| +| Server Status | **REvoCompute (New)**: [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevocompute.yaoyy.moe%2Fcompute%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Direct&link=https%3A%2F%2Frevocompute.yaoyy.moe%2Fcompute%2Fdashboard)](https://revocompute.yaoyy.moe/compute/dashboard) [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevocompute-relay.yaoyy.moe%2Fcompute%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Relay&link=https%3A%2F%2Frevocompute-relay.yaoyy.moe%2Fcompute%2Fdashboard)](https://revocompute-relay.yaoyy.moe/compute/dashboard)
*PSSM/GREMLIN (Legacy)*: [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevodesign.yaoyy.moe%2FPSSM_GREMLIN%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Main%20-%20Legacy&link=https%3A%2F%2Frevodesign.yaoyy.moe%2FPSSM_GREMLIN%2Fdashboard)](https://revodesign.yaoyy.moe/PSSM_GREMLIN/dashboard) [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevodesign-direct.yaoyy.moe%2FPSSM_GREMLIN%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Backup%20-%20Legacy&link=https%3A%2F%2Frevodesign.yaoyy.moe%2FPSSM_GREMLIN%2Fdashboard)](https://revodesign-direct.yaoyy.moe/PSSM_GREMLIN/dashboard)| | Code Quality Control | [![Hook](https://img.shields.io/badge/autofix-pre--commit--hooks-000000.svg)](https://github.com/pre-commit/pre-commit-hooks) [![linting: pylint](https://img.shields.io/badge/linting-pylint-yellowgreen)](https://github.com/pylint-dev/pylint) [![imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) [![syntax upgrade: pyupgrade](https://img.shields.io/badge/pyupgrade-white?style=plastic&logo=python&logoColor=ebedf0&label=syntax&labelColor=da394b&color=white&link=https%3A%2F%2Fgithub.com%2Fasottile%2Fpyupgrade)](https://github.com/asottile/pyupgrade) [![pycln](https://img.shields.io/badge/pycln-white?style=plastic&logo=python&logoColor=35475c&label=imports&labelColor=c4fcfd&color=c6fdbc&link=https%3A%2F%2Fgithub.com%2Fhadialqattan%2Fpycln)](https://github.com/hadialqattan/pycln) [![Flake8](https://img.shields.io/badge/flake8-white?style=plastic&logo=python&logoColor=silver&label=style&link=https%3A%2F%2Fgithub.com%2FPyCQA%2Fflake8)](https://github.com/PyCQA/flake8) [![autoflake](https://img.shields.io/badge/autoflake-yellow?style=plastic&logo=python&logoColor=cyan&label=style&link=https%3A%2F%2Fgithub.com%2FPyCQA%2Fautoflake)](https://github.com/PyCQA/autoflake) [![autopep8](https://img.shields.io/badge/autopep8-orange?style=plastic&logo=python&logoColor=red&label=style&link=https%3A%2F%2Fgithub.com%2FPyCQA%2Fautopep8)](https://github.com/hhatto/autopep8) [![Code style: black](https://img.shields.io/badge/black-black.svg?logo=python&logoColor=black&label=style)](https://github.com/psf/black)| REvoDesign requires no non-standard hardware to run. From 18c25e5ae9272ef715f3f377554d43af50adffc3 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:24:25 +0800 Subject: [PATCH 23/26] fix(server): close restart recovery gaps --- CHANGELOG.md | 3 +- server/revocompute/static/js/task-results.js | 1 + server/revocompute/task_runtime.py | 55 ++++++++++++++++---- server/run/restart.sh | 31 ++++++----- server/tests/test_runner_script_static.py | 20 +++++-- server/tests/test_tasks.py | 36 +++++++++++++ 6 files changed, 115 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df185cca..abf8d0bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,9 +55,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Server: - - Result polling/viewers: terminal task states reload once without overlapping polls; pending result pages keep polling; Mol* teardown completes before iframe removal and preview loaders remain visible. + - Result polling/viewers: terminal task states reload once without overlapping polls; pending result pages keep polling; Mol* teardown completes before iframe removal, stale teardown continuations cannot replace newer previews, and preview loaders remain visible. - AlphaFold stages: drain the stderr translator before wrapper exit and preserve both process statuses so final stage markers cannot be lost. - SLURM status: move allocated tasks from queued to running as soon as the wrapper publishes the real job ID, without waiting for scientific stage output. + - SLURM recovery: restarts preserve unstarted queued work while cancelling and fully finalizing orphaned allocations and running records. - RFdiffusion runner: redirect Hydra bookkeeping and generated schedule caches to the per-job writable `/tmp` filesystem under read-only container execution; lock task defaults to the pinned upstream inference configuration. - Mol* input workbenches: sequence-strip and canvas selections now report canonical residues; cached viewer-shell handshakes no longer race listener setup. - task-type intro audit: intros/categories corrected against the runner docs (Pro-Prime = OGT prediction → `fitness`; ThermoMPNN-D = ΔΔG prediction; PLACER = all-atom ligand modeling; GREMLIN = conservation + couplings; HyperMPNN/LASErMPNN/OpenDDE reworded). diff --git a/server/revocompute/static/js/task-results.js b/server/revocompute/static/js/task-results.js index 4025bfe9..a50b816b 100644 --- a/server/revocompute/static/js/task-results.js +++ b/server/revocompute/static/js/task-results.js @@ -402,6 +402,7 @@ // A dead warm frame must not poison the next pick: dispose it so the // retry cold-starts a fresh shell. await disposeActiveViewer(); + if (isStale(generation)) return; surface.replaceChildren(); surface.appendChild(structureViewerBar(artifact)); var msg = document.createElement("p"); diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index 4f84c4ba..0b57fccf 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -813,25 +813,56 @@ def _on_stage_change(stage: str) -> None: # --------------------------------------------------------------------------- -# Orphaned Docker container recovery after server restart +# Orphaned compute-resource recovery after worker restart # -# SLURM tasks need no recovery: restart.sh's pre-stop sweep cancels every -# in-flight SLURM job and marks its task record failed BEFORE the worker -# dies, so no SLURM orphan can ever exist. Docker containers survive the -# stack restart, so only they are reconnected here. +# A managed stack shutdown sweeps SLURM jobs before stopping the worker, but +# an OOM, crash, or direct container restart bypasses that hook. On worker +# startup, fail those orphaned records and best-effort cancel their allocation. +# Docker containers survive a worker restart and are reconnected instead. # --------------------------------------------------------------------------- def _recover_orphaned_tasks() -> int: - """Reconnect Docker containers whose Celery task was lost (e.g. Redis restart).""" - recovered = 0 + """Resolve compute records whose owning Celery worker disappeared.""" + handled = 0 for task in task_store.list_tasks(): if task.get("status") not in {"running", "queued"}: continue + md5sum = task["md5sum"] + slurm_job_id = str(task.get("slurm_job_id") or "").strip() container_id = str(task.get("container_id") or "") + if slurm_job_id: + cancellation_error = "" + scancel = shutil.which("scancel") + if scancel and slurm_job_id.isdigit(): + try: + subprocess.run([scancel, slurm_job_id], timeout=10, check=True) + except (OSError, subprocess.SubprocessError) as exc: + cancellation_error = f"; scheduler cancellation could not be confirmed: {exc}" + logging.warning("Recovery could not cancel SLURM job %s for %s: %s", slurm_job_id, md5sum, exc) + else: + cancellation_error = "; scheduler cancellation could not be attempted" + _record_failure( + md5sum, + task, + task.get("started_at") or time.time(), + str(task.get("run_stage") or ""), + f"SLURM task lost its worker{cancellation_error}", + ) + handled += 1 + continue + if not container_id and task.get("status") == "running": + _record_failure( + md5sum, + task, + task.get("started_at") or time.time(), + str(task.get("run_stage") or ""), + "Compute task lost its worker before recording a resource handle", + ) + handled += 1 + continue if not container_id: continue - md5sum = task["md5sum"] task_type = task.get("task_type", "gremlin") logging.info("Recovery: checking orphaned task %s", md5sum) try: @@ -844,7 +875,7 @@ def _recover_orphaned_tasks() -> int: logging.info("Recovery: reconnected Docker %s for %s", container_id, md5sum) state = job.poll() _finalize_after_poll(md5sum, task, tt, state) - recovered += 1 + handled += 1 else: _record_failure( md5sum, @@ -853,10 +884,12 @@ def _recover_orphaned_tasks() -> int: "", "Docker container not found after server restart", ) + handled += 1 except Exception as exc: logging.error("Recovery failed for Docker task %s: %s", md5sum, exc) _record_failure(md5sum, task, task.get("started_at") or time.time(), "", f"Recovery error: {exc}") - return recovered + handled += 1 + return handled def _finalize_after_poll(md5sum, task, tt, state): @@ -896,7 +929,7 @@ def _on_worker_ready(sender, **kwargs): try: count = _recover_orphaned_tasks() if count: - logging.info("Recovered %d orphaned task(s)", count) + logging.info("Handled %d orphaned task(s)", count) else: logging.info("Recovery: no orphaned tasks found") except Exception: # boot-time recovery must never die silently diff --git a/server/run/restart.sh b/server/run/restart.sh index 46a8722a..4df8cfc9 100755 --- a/server/run/restart.sh +++ b/server/run/restart.sh @@ -1011,38 +1011,41 @@ cmd_up() { } # Kill and mark every in-flight SLURM task before the worker dies. The -# worker owns the srun clients, so stopping the stack orphans anything -# running; a clean pre-stop sweep means no orphan can ever exist and no -# boot-time recovery machinery is needed. +# worker owns the task database and mounted SLURM clients, so stopping it +# without this sweep can leave allocations and task records orphaned. pre_stop_sweep_slurm() { if [[ "${USE_SLURM}" != "1" ]]; then return 0; fi - # The worker container owns the srun clients, runs as the SLURM user, and - # holds write access to the task DB — the host account has none of the - # three, so the whole sweep runs inside the containers before down. + # Query the scheduler on the host, then cancel and finalize through the + # worker, which owns the mounted SLURM clients and task database. local jobs - jobs=$(squeue -h -u "${RUNNER_USERNAME}" -o '%i' 2>/dev/null || true) + jobs=$(squeue -h -u "${RUNNER_USERNAME:-revodesign}" -o '%i' 2>/dev/null || true) if [[ -n "${jobs}" ]]; then echo "Cancelling in-flight SLURM jobs before stopping the stack: ${jobs}" - # shellcheck disable=SC2086 + # shellcheck disable=SC2046 "${COMPOSE_CMD[@]}" $(compose_files) --env-file "${ENV_FILE}" exec -T worker scancel ${jobs} || true fi echo "Marking in-flight tasks failed before stopping the stack..." + # shellcheck disable=SC2046 "${COMPOSE_CMD[@]}" $(compose_files) --env-file "${ENV_FILE}" exec -T worker python3 - <<'PY' || true import time -from revocompute.task_runtime import task_store +from revocompute.task_runtime import _record_failure, task_store for task in task_store.list_tasks(): - if task.get("status") in {"pending", "queued", "running"}: - task_store.update_task( + if task.get("status") in {"queued", "running"}: + _record_failure( task["md5sum"], - status="failed", - error="Cancelled by server restart", - finished_at=time.time(), + task, + task.get("started_at") or time.time(), + str(task.get("run_stage") or ""), + "Cancelled by server restart", ) PY } cmd_down() { require_env_file + set -a + source "${ENV_FILE}" + set +a ensure_docker_gid resolve_runner_identity pre_stop_sweep_slurm diff --git a/server/tests/test_runner_script_static.py b/server/tests/test_runner_script_static.py index c4aabfc2..114cf093 100644 --- a/server/tests/test_runner_script_static.py +++ b/server/tests/test_runner_script_static.py @@ -12,9 +12,7 @@ RUNNER_SCRIPT = Path(__file__).resolve().parents[1] / "docker" / "runners" / "pssm_gremlin" / "run.sh" OPENDDE_RUNNER_SCRIPT = Path(__file__).resolve().parents[1] / "docker" / "runners" / "opendde" / "run.sh" MPNN_RUNNER_SCRIPT = Path(__file__).resolve().parents[1] / "docker" / "runners" / "mpnn" / "run.sh" -ALPHAFOLD_RUNNER_SCRIPT = ( - Path(__file__).resolve().parents[1] / "docker" / "runners" / "alphafold" / "run.sh" -) +ALPHAFOLD_RUNNER_SCRIPT = Path(__file__).resolve().parents[1] / "docker" / "runners" / "alphafold" / "run.sh" SERVER_ROOT = Path(__file__).resolve().parents[1] @@ -27,6 +25,19 @@ def test_prepared_activation_audits_resources_before_stopping_services(): assert "-m revocompute.resource_audit" in script +def test_slurm_pre_stop_sweep_preserves_pending_and_finalizes_started_tasks(): + script = (SERVER_ROOT / "run" / "restart.sh").read_text(encoding="utf-8") + sweep = script.split("pre_stop_sweep_slurm() {", 1)[1].split("\n}\n\ncmd_down()", 1)[0] + down = script.split("cmd_down() {", 1)[1].split("\n}", 1)[0] + + assert "${RUNNER_USERNAME:-revodesign}" in sweep + assert 'if task.get("status") in {"queued", "running"}:' in sweep + assert '"pending"' not in sweep + assert "from revocompute.task_runtime import _record_failure, task_store" in sweep + assert "_record_failure(" in sweep + assert 'source "${ENV_FILE}"' in down + + def test_runner_script_does_not_eval_user_controlled_commands(): script = RUNNER_SCRIPT.read_text() @@ -93,8 +104,7 @@ def test_alphafold_runner_drains_final_stage_before_exit(tmp_path): output_dir.mkdir() alphafold_root.mkdir() fake_context.write_text( - '_parse_param() { printf "%s\\n" "$2"; }\n' - 'primary_input() { printf "%s\\n" "$FAKE_PRIMARY_INPUT"; }\n', + '_parse_param() { printf "%s\\n" "$2"; }\n' 'primary_input() { printf "%s\\n" "$FAKE_PRIMARY_INPUT"; }\n', encoding="utf-8", ) fake_python.write_text( diff --git a/server/tests/test_tasks.py b/server/tests/test_tasks.py index a654bd97..4c7392bd 100644 --- a/server/tests/test_tasks.py +++ b/server/tests/test_tasks.py @@ -174,6 +174,8 @@ def test_result_status_polling_handles_terminal_and_pending_responses(): assert results.index("window.__revocomputeStatusPoll = setInterval") < results.index( "if (!response.ok || !Array.isArray(payload.artifacts))" ) + disposal = results.index("await disposeActiveViewer();") + assert disposal < results.index("if (isStale(generation)) return;", disposal) def test_execution_logs_are_diagnostic_text_artifacts_not_main_results(): @@ -494,6 +496,40 @@ def _fake_runner(task_id, tt, runner, entities, output_dir, stage_callback=None, assert observed_statuses == ["queued", "running", "finished"] +def test_worker_recovery_cancels_slurm_orphans_and_preserves_unstarted_queue(monkeypatch, tmp_path): + module = _load_pssm_module( + monkeypatch, + tmp_path, + extra_env={"RUNNER_UID": "1234", "RUNNER_GID": "5678"}, + ) + runtime = module.task_runtime + tasks = [ + { + "md5sum": "a" * 32, + "status": "running", + "slurm_job_id": "4154", + "run_stage": "design", + "started_at": 123.0, + }, + {"md5sum": "b" * 32, "status": "running", "started_at": 456.0}, + {"md5sum": "c" * 32, "status": "queued"}, + {"md5sum": "d" * 32, "status": "pending"}, + ] + failures = [] + cancellations = [] + + monkeypatch.setattr(runtime.task_store, "list_tasks", lambda: tasks) + monkeypatch.setattr(runtime.shutil, "which", lambda command: f"/usr/bin/{command}") + monkeypatch.setattr(runtime.subprocess, "run", lambda args, **kwargs: cancellations.append((args, kwargs))) + monkeypatch.setattr(runtime, "_record_failure", lambda *args: failures.append(args)) + + assert runtime._recover_orphaned_tasks() == 2 + assert cancellations == [(["/usr/bin/scancel", "4154"], {"timeout": 10, "check": True})] + assert [failure[0] for failure in failures] == ["a" * 32, "b" * 32] + assert failures[0][3:] == ("design", "SLURM task lost its worker") + assert failures[1][3:] == ("", "Compute task lost its worker before recording a resource handle") + + def test_multi_file_submission_creates_isolated_workspace_snapshot(monkeypatch, tmp_path): module = _load_pssm_module( monkeypatch, From 9c9e8c46891b81fc901205a1cc935b5a3d353142 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:28:57 +0800 Subject: [PATCH 24/26] fix(server): keep py2dmol controls visible --- server/revocompute/static/js/task-results.js | 2 ++ server/revocompute/task_runtime.py | 8 ++++---- server/tests/test_tasks.py | 4 ++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/server/revocompute/static/js/task-results.js b/server/revocompute/static/js/task-results.js index a50b816b..89394002 100644 --- a/server/revocompute/static/js/task-results.js +++ b/server/revocompute/static/js/task-results.js @@ -375,6 +375,8 @@ if (structureViewer === "py2dmol") { stage.hidden = false; if (structureHolder) structureHolder.hidden = true; + stage.replaceChildren(); + stage.appendChild(structureViewerBar(artifact)); try { await renderPy2DmolFallback(structureText, artifact, stage, generation, new Error("User selected alpha-trace viewer")); if (isStale(generation)) return; diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index 0b57fccf..4b751681 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -885,7 +885,7 @@ def _recover_orphaned_tasks() -> int: "Docker container not found after server restart", ) handled += 1 - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except logging.error("Recovery failed for Docker task %s: %s", md5sum, exc) _record_failure(md5sum, task, task.get("started_at") or time.time(), "", f"Recovery error: {exc}") handled += 1 @@ -966,12 +966,12 @@ def cancel_compute_resources(self, slurm_job_id: str | None = None, container_id except Exception as exc: # pylint: disable=broad-except logging.warning("Failed to scancel SLURM job %s: %s", slurm_job_id, exc) if container_id: - docker = shutil.which("docker") - if not docker: + docker_executable = shutil.which("docker") + if not docker_executable: logging.warning("docker not found; cannot stop container %s", container_id) else: try: - subprocess.run([docker, "stop", str(container_id)], timeout=15, check=True) + subprocess.run([docker_executable, "stop", str(container_id)], timeout=15, check=True) logging.info("Stopped Docker container %s", container_id) except Exception as exc: # pylint: disable=broad-except logging.warning("Failed to stop Docker container %s: %s", container_id, exc) diff --git a/server/tests/test_tasks.py b/server/tests/test_tasks.py index 4c7392bd..7f48e5f9 100644 --- a/server/tests/test_tasks.py +++ b/server/tests/test_tasks.py @@ -176,6 +176,10 @@ def test_result_status_polling_handles_terminal_and_pending_responses(): ) disposal = results.index("await disposeActiveViewer();") assert disposal < results.index("if (isStale(generation)) return;", disposal) + py2dmol = results.index('if (structureViewer === "py2dmol")') + py2dmol_render = results.index("await renderPy2DmolFallback", py2dmol) + assert py2dmol < results.index("stage.replaceChildren();", py2dmol) < py2dmol_render + assert py2dmol < results.index("stage.appendChild(structureViewerBar(artifact));", py2dmol) < py2dmol_render def test_execution_logs_are_diagnostic_text_artifacts_not_main_results(): From 9520b426d9ff8eea6e3cb88969a5ef8ae84b7f02 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:44:00 +0800 Subject: [PATCH 25/26] fix(server): scope restart recovery safely --- CHANGELOG.md | 2 +- README.md | 2 +- server/revocompute/task_runtime.py | 18 +++++++- server/run/restart.sh | 22 ++++++--- server/tests/test_runner_script_static.py | 5 ++- server/tests/test_tasks.py | 54 +++++++++++++++++++++++ 6 files changed, 91 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abf8d0bb..4b5a6264 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Result polling/viewers: terminal task states reload once without overlapping polls; pending result pages keep polling; Mol* teardown completes before iframe removal, stale teardown continuations cannot replace newer previews, and preview loaders remain visible. - AlphaFold stages: drain the stderr translator before wrapper exit and preserve both process statuses so final stage markers cannot be lost. - SLURM status: move allocated tasks from queued to running as soon as the wrapper publishes the real job ID, without waiting for scientific stage output. - - SLURM recovery: restarts preserve unstarted queued work while cancelling and fully finalizing orphaned allocations and running records. + - Compute recovery: restarts preserve unstarted queued work, cancel only deployment-owned SLURM allocations, fully finalize orphans, and poll reconnected Docker jobs without blocking worker readiness. - RFdiffusion runner: redirect Hydra bookkeeping and generated schedule caches to the per-job writable `/tmp` filesystem under read-only container execution; lock task defaults to the pinned upstream inference configuration. - Mol* input workbenches: sequence-strip and canvas selections now report canonical residues; cached viewer-shell handshakes no longer race listener setup. - task-type intro audit: intros/categories corrected against the runner docs (Pro-Prime = OGT prediction → `fitness`; ThermoMPNN-D = ΔΔG prediction; PLACER = all-atom ligand modeling; GREMLIN = conservation + couplings; HyperMPNN/LASErMPNN/OpenDDE reworded). diff --git a/README.md b/README.md index 4e6c1b6a..9da06330 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ | CI status | [![Unit Test on tagging](https://github.com/YaoYinYing/REvoDesign/actions/workflows/unit_tests_tag.yml/badge.svg)](https://github.com/YaoYinYing/REvoDesign/actions/workflows/unit_tests_tag.yml) [![Docker Image](https://github.com/YaoYinYing/REvoDesign/actions/workflows/docker-image.yml/badge.svg)](https://github.com/YaoYinYing/REvoDesign/actions/workflows/docker-image.yml) [![Docs](https://github.com/YaoYinYing/REvoDesign/actions/workflows/docs.yml/badge.svg)](https://github.com/YaoYinYing/REvoDesign/actions/workflows/docs.yml)| | Code Quality Status | ![GitHub repo size](https://img.shields.io/github/repo-size/YaoYinYing/REvoDesign?logo=github&color=green) [![pylint](https://github-image-cache.yaoyy.moe/badge_dir_with_uniq_name/REvoDesign/pylint/pylint_scan.svg)](https://github.com/Silleellie/pylint-github-action) [![Maintainability](https://qlty.sh/badges/8a669407-26a2-45aa-8323-b5567c391cbf/maintainability.svg)](https://qlty.sh/gh/YaoYinYing/projects/REvoDesign) [![codecov](https://codecov.io/gh/YaoYinYing/REvoDesign/graph/badge.svg?token=2qSJ7cgk1b)](https://codecov.io/gh/YaoYinYing/REvoDesign) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/05f9fa7eb01e4bba9bf0063182513f82)](https://app.codacy.com/gh/YaoYinYing/REvoDesign/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) [![DeepSource](https://app.deepsource.com/gh/YaoYinYing/REvoDesign.svg/?label=active+issues&show_trend=true&token=OGYjIrCzCfLD9zjBpSXPCMeu)](https://app.deepsource.com/gh/YaoYinYing/REvoDesign/) [![DeepSource](https://app.deepsource.com/gh/YaoYinYing/REvoDesign.svg/?label=resolved+issues&show_trend=true&token=OGYjIrCzCfLD9zjBpSXPCMeu)](https://app.deepsource.com/gh/YaoYinYing/REvoDesign/) [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FYaoYinYing%2FREvoDesign.svg?type=shield&issueType=license)](https://app.fossa.com/projects/git%2Bgithub.com%2FYaoYinYing%2FREvoDesign?ref=badge_shield&issueType=license) [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FYaoYinYing%2FREvoDesign.svg?type=shield&issueType=security)](https://app.fossa.com/projects/git%2Bgithub.com%2FYaoYinYing%2FREvoDesign?ref=badge_shield&issueType=security)| | Server Docker Image | [![Docker Image Size](https://img.shields.io/docker/image-size/yaoyinying/revodesign-pssm-gremlin-non-root?style=social&logo=docker&label=Runner)](https://hub.docker.com/r/yaoyinying/revodesign-pssm-gremlin-non-root) [![Docker Image Size](https://img.shields.io/docker/image-size/yaoyinying/revodesign-pssm-gremlin-server-non-root?style=social&logo=docker&label=Server)](https://hub.docker.com/r/yaoyinying/revodesign-pssm-gremlin-server-non-root) | -| Server Status | **REvoCompute (New)**: [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevocompute.yaoyy.moe%2Fcompute%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Direct&link=https%3A%2F%2Frevocompute.yaoyy.moe%2Fcompute%2Fdashboard)](https://revocompute.yaoyy.moe/compute/dashboard) [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevocompute-relay.yaoyy.moe%2Fcompute%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Relay&link=https%3A%2F%2Frevocompute-relay.yaoyy.moe%2Fcompute%2Fdashboard)](https://revocompute-relay.yaoyy.moe/compute/dashboard)
*PSSM/GREMLIN (Legacy)*: [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevodesign.yaoyy.moe%2FPSSM_GREMLIN%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Main%20-%20Legacy&link=https%3A%2F%2Frevodesign.yaoyy.moe%2FPSSM_GREMLIN%2Fdashboard)](https://revodesign.yaoyy.moe/PSSM_GREMLIN/dashboard) [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevodesign-direct.yaoyy.moe%2FPSSM_GREMLIN%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Backup%20-%20Legacy&link=https%3A%2F%2Frevodesign.yaoyy.moe%2FPSSM_GREMLIN%2Fdashboard)](https://revodesign-direct.yaoyy.moe/PSSM_GREMLIN/dashboard)| +| Server Status | **REvoCompute (New)**: [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevocompute.yaoyy.moe%2Fcompute%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Direct&link=https%3A%2F%2Frevocompute.yaoyy.moe%2Fcompute%2Fdashboard)](https://revocompute.yaoyy.moe/compute/dashboard) [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevocompute-relay.yaoyy.moe%2Fcompute%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Relay&link=https%3A%2F%2Frevocompute-relay.yaoyy.moe%2Fcompute%2Fdashboard)](https://revocompute-relay.yaoyy.moe/compute/dashboard)
*PSSM/GREMLIN (Legacy)*: [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevodesign.yaoyy.moe%2FPSSM_GREMLIN%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Main%20-%20Legacy&link=https%3A%2F%2Frevodesign.yaoyy.moe%2FPSSM_GREMLIN%2Fdashboard)](https://revodesign.yaoyy.moe/PSSM_GREMLIN/dashboard) [![Website](https://img.shields.io/website?url=https%3A%2F%2Frevodesign-direct.yaoyy.moe%2FPSSM_GREMLIN%2Flogo.svg&up_message=available&down_message=unavailable&logo=gunicorn&logoColor=gold&label=Backup%20-%20Legacy&link=https%3A%2F%2Frevodesign-direct.yaoyy.moe%2FPSSM_GREMLIN%2Fdashboard)](https://revodesign-direct.yaoyy.moe/PSSM_GREMLIN/dashboard)| | Code Quality Control | [![Hook](https://img.shields.io/badge/autofix-pre--commit--hooks-000000.svg)](https://github.com/pre-commit/pre-commit-hooks) [![linting: pylint](https://img.shields.io/badge/linting-pylint-yellowgreen)](https://github.com/pylint-dev/pylint) [![imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) [![syntax upgrade: pyupgrade](https://img.shields.io/badge/pyupgrade-white?style=plastic&logo=python&logoColor=ebedf0&label=syntax&labelColor=da394b&color=white&link=https%3A%2F%2Fgithub.com%2Fasottile%2Fpyupgrade)](https://github.com/asottile/pyupgrade) [![pycln](https://img.shields.io/badge/pycln-white?style=plastic&logo=python&logoColor=35475c&label=imports&labelColor=c4fcfd&color=c6fdbc&link=https%3A%2F%2Fgithub.com%2Fhadialqattan%2Fpycln)](https://github.com/hadialqattan/pycln) [![Flake8](https://img.shields.io/badge/flake8-white?style=plastic&logo=python&logoColor=silver&label=style&link=https%3A%2F%2Fgithub.com%2FPyCQA%2Fflake8)](https://github.com/PyCQA/flake8) [![autoflake](https://img.shields.io/badge/autoflake-yellow?style=plastic&logo=python&logoColor=cyan&label=style&link=https%3A%2F%2Fgithub.com%2FPyCQA%2Fautoflake)](https://github.com/PyCQA/autoflake) [![autopep8](https://img.shields.io/badge/autopep8-orange?style=plastic&logo=python&logoColor=red&label=style&link=https%3A%2F%2Fgithub.com%2FPyCQA%2Fautopep8)](https://github.com/hhatto/autopep8) [![Code style: black](https://img.shields.io/badge/black-black.svg?logo=python&logoColor=black&label=style)](https://github.com/psf/black)| REvoDesign requires no non-standard hardware to run. diff --git a/server/revocompute/task_runtime.py b/server/revocompute/task_runtime.py index 4b751681..ca99babc 100644 --- a/server/revocompute/task_runtime.py +++ b/server/revocompute/task_runtime.py @@ -19,6 +19,7 @@ import re import shutil import subprocess +import threading import time import zipfile from datetime import datetime @@ -873,8 +874,12 @@ def _recover_orphaned_tasks() -> int: job = DockerJob(md5sum, tt, runner, [], task["result_dir"]) if job.reconnect(container_id): logging.info("Recovery: reconnected Docker %s for %s", container_id, md5sum) - state = job.poll() - _finalize_after_poll(md5sum, task, tt, state) + threading.Thread( + target=_poll_recovered_docker_job, + args=(md5sum, task, tt, job), + name=f"recover-{md5sum[:12]}", + daemon=True, + ).start() handled += 1 else: _record_failure( @@ -892,6 +897,15 @@ def _recover_orphaned_tasks() -> int: return handled +def _poll_recovered_docker_job(md5sum, task, tt, job) -> None: + """Poll and finalize a reconnected container without delaying worker readiness.""" + try: + _finalize_after_poll(md5sum, task, tt, job.poll()) + except Exception as exc: # pylint: disable=broad-except + logging.exception("Recovery polling failed for Docker task %s", md5sum) + _record_failure(md5sum, task, task.get("started_at") or time.time(), "", f"Recovery error: {exc}") + + def _finalize_after_poll(md5sum, task, tt, state): """Publish results or record failure after a recovered job completes.""" if state == JobState.FAILED: diff --git a/server/run/restart.sh b/server/run/restart.sh index 4df8cfc9..6c83471d 100755 --- a/server/run/restart.sh +++ b/server/run/restart.sh @@ -1015,14 +1015,22 @@ cmd_up() { # without this sweep can leave allocations and task records orphaned. pre_stop_sweep_slurm() { if [[ "${USE_SLURM}" != "1" ]]; then return 0; fi - # Query the scheduler on the host, then cancel and finalize through the - # worker, which owns the mounted SLURM clients and task database. - local jobs - jobs=$(squeue -h -u "${RUNNER_USERNAME:-revodesign}" -o '%i' 2>/dev/null || true) - if [[ -n "${jobs}" ]]; then - echo "Cancelling in-flight SLURM jobs before stopping the stack: ${jobs}" + # Read only this deployment's persisted allocation IDs, then cancel and + # finalize through the worker, which owns the task DB and SLURM clients. + local jobs=() + # shellcheck disable=SC2046 + mapfile -t jobs < <("${COMPOSE_CMD[@]}" $(compose_files) --env-file "${ENV_FILE}" exec -T worker python3 - <<'PY' +from revocompute.task_runtime import task_store +for task in task_store.list_tasks(): + job_id = str(task.get("slurm_job_id") or "").strip() + if task.get("status") in {"queued", "running"} and job_id.isdigit(): + print(job_id) +PY +) + if (( ${#jobs[@]} )); then + echo "Cancelling this deployment's in-flight SLURM jobs: ${jobs[*]}" # shellcheck disable=SC2046 - "${COMPOSE_CMD[@]}" $(compose_files) --env-file "${ENV_FILE}" exec -T worker scancel ${jobs} || true + "${COMPOSE_CMD[@]}" $(compose_files) --env-file "${ENV_FILE}" exec -T worker scancel "${jobs[@]}" || true fi echo "Marking in-flight tasks failed before stopping the stack..." # shellcheck disable=SC2046 diff --git a/server/tests/test_runner_script_static.py b/server/tests/test_runner_script_static.py index 114cf093..9f07af37 100644 --- a/server/tests/test_runner_script_static.py +++ b/server/tests/test_runner_script_static.py @@ -30,7 +30,10 @@ def test_slurm_pre_stop_sweep_preserves_pending_and_finalizes_started_tasks(): sweep = script.split("pre_stop_sweep_slurm() {", 1)[1].split("\n}\n\ncmd_down()", 1)[0] down = script.split("cmd_down() {", 1)[1].split("\n}", 1)[0] - assert "${RUNNER_USERNAME:-revodesign}" in sweep + assert "squeue" not in sweep + assert 'task.get("slurm_job_id")' in sweep + assert 'task.get("status") in {"queued", "running"}' in sweep + assert 'scancel "${jobs[@]}"' in sweep assert 'if task.get("status") in {"queued", "running"}:' in sweep assert '"pending"' not in sweep assert "from revocompute.task_runtime import _record_failure, task_store" in sweep diff --git a/server/tests/test_tasks.py b/server/tests/test_tasks.py index 7f48e5f9..70c290c4 100644 --- a/server/tests/test_tasks.py +++ b/server/tests/test_tasks.py @@ -534,6 +534,60 @@ def test_worker_recovery_cancels_slurm_orphans_and_preserves_unstarted_queue(mon assert failures[1][3:] == ("", "Compute task lost its worker before recording a resource handle") +def test_worker_recovery_polls_reconnected_docker_outside_startup(monkeypatch, tmp_path): + module = _load_pssm_module( + monkeypatch, + tmp_path, + extra_env={"RUNNER_UID": "1234", "RUNNER_GID": "5678"}, + ) + runtime = module.task_runtime + from revocompute import task_types + from revocompute.job.runners import docker_runner + + task = { + "md5sum": "e" * 32, + "status": "running", + "container_id": "container-1", + "task_type": "gremlin", + "result_dir": str(tmp_path / "result"), + } + started_threads = [] + poll_calls = [] + + class FakeDockerJob: + def __init__(self, *args): + del args + + def reconnect(self, container_id): + return container_id == "container-1" + + def poll(self): + poll_calls.append(True) + return runtime.JobState.COMPLETED + + class FakeThread: + def __init__(self, *, target, args, name, daemon): + self.target = target + self.args = args + self.name = name + self.daemon = daemon + + def start(self): + started_threads.append(self) + + monkeypatch.setattr(runtime.task_store, "list_tasks", lambda: [task]) + monkeypatch.setattr(task_types, "get", lambda task_type: (object(), object())) + monkeypatch.setattr(docker_runner, "DockerJob", FakeDockerJob) + monkeypatch.setattr(runtime.threading, "Thread", FakeThread) + + assert runtime._recover_orphaned_tasks() == 1 + assert poll_calls == [] + assert len(started_threads) == 1 + assert started_threads[0].target is runtime._poll_recovered_docker_job + assert started_threads[0].name == "recover-eeeeeeeeeeee" + assert started_threads[0].daemon is True + + def test_multi_file_submission_creates_isolated_workspace_snapshot(monkeypatch, tmp_path): module = _load_pssm_module( monkeypatch, From 236d6d1a78a7f9a408d7793f3f06684ac74c05dd Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:54:07 +0800 Subject: [PATCH 26/26] =?UTF-8?q?chore:=20Done=20fixing=20=E2=80=94=20CI,?= =?UTF-8?q?=20production,=20and=20living=20task=20verified?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit