From b57eb24ab90bc6d4c03e3d8caf4c384b2eeebb63 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:31:19 +0800 Subject: [PATCH 01/63] fix(server): address deepsource audit findings --- server/pssm_gremlin_server/auth.py | 24 ++++++---- server/pssm_gremlin_server/pssm_gremlin.py | 15 ++---- server/pssm_gremlin_server/routes.py | 55 +++++++++++----------- 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/server/pssm_gremlin_server/auth.py b/server/pssm_gremlin_server/auth.py index 68441817..0a1d6daf 100644 --- a/server/pssm_gremlin_server/auth.py +++ b/server/pssm_gremlin_server/auth.py @@ -23,7 +23,7 @@ from typing import Any import sqlalchemy as sa -from flask import current_app, g, jsonify, redirect, render_template, request, url_for +from flask import current_app, g, jsonify, redirect, request, url_for from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer from werkzeug.security import check_password_hash, generate_password_hash @@ -471,10 +471,13 @@ def load_current_user() -> dict[str, Any] | None: payload = validate_token(token) if payload is not None: user = db.get_user(payload["uid"]) - if user is not None and _is_account_blocked(user) is None: - if payload.get("ver", 0) == user.get("token_version", 0): - g.auth_method = "bearer" - return user + if ( + user is not None + and _is_account_blocked(user) is None + and payload.get("ver", 0) == user.get("token_version", 0) + ): + g.auth_method = "bearer" + return user # 2. Cookie — browser page navigations after login (read-only; CSRF-prone) # Guest accounts ARE permitted to use cookie-based web access. token = request.cookies.get("auth_token") @@ -482,10 +485,13 @@ def load_current_user() -> dict[str, Any] | None: payload = validate_token(token) if payload is not None: user = db.get_user(payload["uid"]) - if user is not None and _is_account_blocked(user) is None: - if payload.get("ver", 0) == user.get("token_version", 0): - g.auth_method = "cookie" - return user + if ( + user is not None + and _is_account_blocked(user) is None + and payload.get("ver", 0) == user.get("token_version", 0) + ): + g.auth_method = "cookie" + return user # 3. API key (programmatic access — restricted privileges) # Guest accounts are not permitted to use API keys. diff --git a/server/pssm_gremlin_server/pssm_gremlin.py b/server/pssm_gremlin_server/pssm_gremlin.py index e5808669..58657a52 100644 --- a/server/pssm_gremlin_server/pssm_gremlin.py +++ b/server/pssm_gremlin_server/pssm_gremlin.py @@ -2,9 +2,6 @@ # Distributed under the terms of the GNU General Public License v3.0. # SPDX-License-Identifier: GPL-3.0-only - -#! /mnt/data/envs/conda_env/envs/REvoDesign/bin/python - from __future__ import annotations import grp @@ -26,7 +23,7 @@ from celery import Celery from celery.result import AsyncResult from docker import types -from flask import Flask, current_app, g, jsonify, request +from flask import Flask, g, jsonify, request from pssm_gremlin_server.db import TaskDatabase from sqlalchemy.exc import IntegrityError from werkzeug.utils import secure_filename @@ -39,7 +36,6 @@ from pssm_gremlin_server.auth import UserDatabase # noqa: E402 from pssm_gremlin_server.auth import _env_bool # noqa: E402 -from pssm_gremlin_server.auth import _env_int # noqa: E402 from pssm_gremlin_server.auth import _env_str # noqa: E402 from pssm_gremlin_server.auth import send_admin_digest # noqa: E402 @@ -253,7 +249,7 @@ def _digest_loop() -> None: with open(_digest_lock, "w") as _lock_fh: fcntl.flock(_lock_fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) send_admin_digest() - except (BlockingIOError, OSError): + except OSError: pass # another worker has the lock — try next cycle except Exception: logging.exception("Admin digest thread failed") @@ -660,7 +656,7 @@ def _create_mount(mount_name: str, path: str, read_only=True) -> tuple[types.Mou target_path = os.path.join(_ROOT_MOUNT_DIRECTORY, mount_name) if not read_only: - logging.warning(f"{mount_name} is not read-only!") + logging.warning("%s is not read-only!", mount_name) if os.path.isdir(path): source_path = path @@ -670,7 +666,7 @@ def _create_mount(mount_name: str, path: str, read_only=True) -> tuple[types.Mou mounted_path = os.path.join(target_path, os.path.basename(path)) if not os.path.exists(source_path): os.makedirs(source_path) - logging.info(f"Mounting {source_path} -> {target_path}") + logging.info("Mounting %s -> %s", source_path, target_path) mount = types.Mount( target=str(target_path), source=str(source_path), @@ -818,8 +814,7 @@ def _pack_failed_results_archive(task: dict, error: Any) -> None: def format_times(timestamp): if timestamp: return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M:%S") - else: - return None + return None def format_walltime(seconds: Any) -> str: diff --git a/server/pssm_gremlin_server/routes.py b/server/pssm_gremlin_server/routes.py index 8bf6f2f8..b564170d 100644 --- a/server/pssm_gremlin_server/routes.py +++ b/server/pssm_gremlin_server/routes.py @@ -17,6 +17,7 @@ import os import shutil import time +from typing import Any from celery.result import AsyncResult from flask import current_app, g, jsonify, redirect, render_template, request, send_from_directory, url_for @@ -735,23 +736,21 @@ def auth_forgot_password(): return jsonify({"message": "If that email is registered, a reset link has been sent."}), 200 -@app.route("/PSSM_GREMLIN/reset_password", methods=["GET", "POST"]) -def auth_reset_password(): - """Password-reset page. +@app.route("/PSSM_GREMLIN/reset_password", methods=["GET"]) +def auth_reset_password_page(): + """Render the password-reset page for a valid reset token.""" + token = request.args.get("c", "").strip() + if not token: + return render_template("error.html", code=400, message="Missing reset token."), 400 + user_id = validate_reset_token(token) + if user_id is None: + return render_template("error.html", code=400, message="Invalid or expired reset token."), 400 + return render_template("reset-password.html", token=token), 200 - ``GET`` — renders the new-password form (requires ``?c=`` token). - ``POST`` — sets the new password. - """ - if request.method == "GET": - token = request.args.get("c", "").strip() - if not token: - return render_template("error.html", code=400, message="Missing reset token."), 400 - user_id = validate_reset_token(token) - if user_id is None: - return render_template("error.html", code=400, message="Invalid or expired reset token."), 400 - return render_template("reset-password.html", token=token), 200 - - # POST + +@app.route("/PSSM_GREMLIN/reset_password", methods=["POST"]) +def auth_reset_password(): + """Set a new password using a password-reset token.""" req = _parse_body(ResetPasswordRequest) if isinstance(req, tuple): return req @@ -1069,27 +1068,29 @@ def auth_revoke_api_key(): return jsonify({"message": "API key revoked"}), 200 -@app.route("/PSSM_GREMLIN/api/auth/admin/users", methods=["GET", "POST"]) +@app.route("/PSSM_GREMLIN/api/auth/admin/users", methods=["GET"]) @login_required def admin_users(): - """Admin-only user management. - - ``GET`` — list all users (safe fields only). - ``POST`` — create a new user (auto-verified, immediately active). - """ + """Admin-only user listing with safe fields only.""" if _blocked := require_admin(): return _blocked db = _get_user_db() + users = db.list_users() + safe = [UserResponse.model_validate(u).model_dump() for u in users] + return jsonify({"users": safe}), 200 - if request.method == "GET": - users = db.list_users() - safe = [UserResponse.model_validate(u).model_dump() for u in users] - return jsonify({"users": safe}), 200 - # POST — state-changing, require Bearer token for CSRF protection +@app.route("/PSSM_GREMLIN/api/auth/admin/users", methods=["POST"]) +@login_required +def admin_create_user(): + """Admin-only user creation for pre-approved accounts.""" + if _blocked := require_admin(): + return _blocked if _blocked := require_bearer_auth(): return _blocked + + db = _get_user_db() req = _parse_body(AdminCreateUserRequest) if isinstance(req, tuple): return req From 8d7d1f56dfd02dc2b7a2e6b7ceaf81736183b3b2 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:39:45 +0800 Subject: [PATCH 02/63] fix(deepsource): exclude server tests from audit noise --- .deepsource.toml | 8 ++++++-- pyproject.toml | 2 +- server/pssm_gremlin_server/pssm_gremlin.py | 10 +++++----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.deepsource.toml b/.deepsource.toml index 74a47e2b..95b03efc 100644 --- a/.deepsource.toml +++ b/.deepsource.toml @@ -2,7 +2,9 @@ version = 1 test_patterns = [ "tests/**", - "tests/**/**" + "tests/**/**", + "server/tests/**", + "server/tests/**/**", ] exclude_patterns = [ @@ -17,6 +19,8 @@ exclude_patterns = [ "tools/**", "tests/**", "tests/**/**", + "server/tests/**", + "server/tests/**/**", ] [[analyzers]] @@ -24,4 +28,4 @@ name = "python" [analyzers.meta] runtime_version = "3.x.x" - max_line_length = 120 \ No newline at end of file + max_line_length = 120 diff --git a/pyproject.toml b/pyproject.toml index e240cec9..2ca09bab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -226,7 +226,7 @@ include = [ ] [tool.bandit] -exclude_dirs = ["build","dist","tests","scripts"] +exclude_dirs = ["build","dist","tests","server/tests","scripts"] number = 4 recursive = true targets = "src" diff --git a/server/pssm_gremlin_server/pssm_gremlin.py b/server/pssm_gremlin_server/pssm_gremlin.py index 58657a52..89c6cd40 100644 --- a/server/pssm_gremlin_server/pssm_gremlin.py +++ b/server/pssm_gremlin_server/pssm_gremlin.py @@ -6,6 +6,7 @@ import grp import hashlib +import importlib import json import logging import os @@ -17,6 +18,7 @@ import time from dataclasses import dataclass from datetime import datetime +from pathlib import Path from typing import Any import docker @@ -286,7 +288,7 @@ def _digest_loop() -> None: def _is_binary_file(path: str) -> bool: try: - with open(path, "rb") as f: + with Path(path).open("rb") as f: chunk = f.read(4096) except OSError: return True @@ -304,7 +306,7 @@ def _is_binary_file(path: str) -> bool: def _is_fasta_content(path: str) -> bool: """Return True if *path* looks like a FASTA file (first non-blank line starts with '>').""" try: - with open(path, encoding="utf-8", errors="replace") as f: + with Path(path).open(encoding="utf-8", errors="replace") as f: for line in f: stripped = line.strip() if not stripped: @@ -778,8 +780,6 @@ def run_pssm_gremlin_in_docker(fasta_path, output_dir, docker_client=None, stage except docker.errors.DockerException: pass - return - def _pack_results_archive(task: dict) -> None: zip_filename = _task_zip_path(task) @@ -956,7 +956,7 @@ def _on_stage_change(stage: str) -> None: # Register HTTP routes (imported late to avoid circular imports — routes.py # needs ``app`` and helpers that are only available after this module loads). # --------------------------------------------------------------------------- -from pssm_gremlin_server import routes # noqa: E402, F401 +importlib.import_module("pssm_gremlin_server.routes") if __name__ == "__main__": app.run(host="0.0.0.0", port=CONFIG.port) # nosec B104: containerized server, binding to all interfaces by design From 69ad82b7d0792616f27df55d8e07ec76af0f03c0 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:47:06 +0800 Subject: [PATCH 03/63] docs(deepsource): plan active issue reduction --- docs/dev-guide/deepsource-reduction-plan.md | 231 ++++++++++++++++++++ docs/dev-guide/index.md | 2 + 2 files changed, 233 insertions(+) create mode 100644 docs/dev-guide/deepsource-reduction-plan.md diff --git a/docs/dev-guide/deepsource-reduction-plan.md b/docs/dev-guide/deepsource-reduction-plan.md new file mode 100644 index 00000000..b408b44f --- /dev/null +++ b/docs/dev-guide/deepsource-reduction-plan.md @@ -0,0 +1,231 @@ +# DeepSource Active Issue Reduction Plan + +Snapshot date: 2026-07-16 + +Target: reduce the DeepSource headline active occurrence count from about +1.3k to 300 or fewer, without masking real security findings. + +## Current Evidence + +DeepSource's issue page for the default branch reported these active occurrence +widgets in the latest snapshot: + +| Widget | Count | +|--------|-------| +| All active occurrences | 1259 | +| Recommended | 855 | +| Secrets | 674 | +| Coverage | 480 | +| Antipattern | 508 | +| Style | 168 | +| Performance | 61 | +| Bug risk | 42 | +| Docs | 0 | +| Typecheck | 0 | + +These widgets overlap and must not be added together. + +The latest default-branch analysis in the snapshot was: + +| Field | Value | +|-------|-------| +| Branch | `main` | +| Commit | `57c36bcc5e5b3d6261b5f9f8cdaeb1690f00a41e` | +| Analysis run | `9ea7977c-250b-4dbd-abd2-a5b02ae9e62e` | +| Status | `FAIL` | +| Finished | `2026-07-15T16:34:11` | + +The active PR branch for the first remediation batch was: + +| Field | Value | +|-------|-------| +| Branch | `codex/deepsource-audit-fixes` | +| Commit | `8d7d1f56dfd02dc2b7a2e6b7ceaf81736183b3b2` | +| Analysis run | `769adfbb-aaf5-48c5-a111-0ff6c6436c6d` | +| Status | `PASS` | +| Python check result | `0` issues raised, `18` resolved | +| Finished | `2026-07-16T03:40:15` | + +The default-branch Python check exported through the API contained 519 issue +nodes: + +| Category | Count | +|----------|-------| +| Security | 453 | +| Antipattern | 59 | +| Style | 3 | +| Bug risk | 2 | +| Performance | 2 | + +High-volume Python issue families on `main` were: + +| Rule | Count | Main cause | +|------|-------|------------| +| `BAN-B101` | 444 | `assert` statements in `server/tests/**` treated as production security findings | +| `BAN-B103` | 4 | File-permission test helpers in `server/tests/**` treated as production security findings | +| `PY-W2000` | 33-36 | Unused imports, mostly test and server cleanup | +| `PY-W0069` | 7 | Commented-out code blocks | +| `PTC-W0043` | 6 | Unnecessary local `del` statements | +| `PY-R1000` | 5 | High cyclomatic complexity | +| `PTC-W0048` | 2 | Mergeable `if` statements | +| `PYL-R0401` | 1 | Cyclic import | + +Evidence files from this snapshot: + +| File | Purpose | +|------|---------| +| `/tmp/deepsource_plan_issues.html` | Server-rendered issue page snapshot | +| `/tmp/deepsource_all_issues_fullscope.json` | Default-branch Python issue export | +| `/tmp/deepsource_main_concrete_all_plan.json` | Grouped default-branch issue families | +| `/tmp/deepsource_run_detail_fullscope.json` | Default-branch run detail | +| `/tmp/deepsource_pr_run_detail_plan.json` | PR branch run detail | + +## Reduction Strategy + +To reach 300 from 1259, the project must remove or legitimately resolve at +least 959 active occurrences. The fastest safe path is not to rewrite test +asserts one by one; it is to fix scope first, then handle secrets, then use +coverage and low-volume code quality work only as needed. + +### Stage 0: Merge the Python Scope Fix + +PR #193 adds `server/tests/**` to DeepSource's test/exclude configuration and +Bandit's test exclusion, then fixes the small production findings surfaced in +`server/pssm_gremlin_server`. + +Expected effect: + +| Metric | Before | After PR branch | +|--------|--------|-----------------| +| Python issues raised in run | 497 | 0 | +| Python check status | `FAIL` | `PASS` | +| Test-noise security findings | 448+ | 0 on PR branch | + +Verification gate: + +1. Merge PR #193. +2. Wait for a default-branch DeepSource reanalysis. +3. Confirm the default-branch Python check no longer raises the + `server/tests/**` assert and file-permission findings. +4. Record the new `all` active occurrence count before starting Stage 1. + +Do not spend time replacing pytest `assert` usage in `server/tests/**`; those +are test semantics, not production security defects. + +### Stage 1: Resolve the Secrets Bucket + +The issue page reports 674 active secret occurrences. This is the largest +security bucket and should be treated as the highest-risk remaining work after +the Python scope fix. + +Required workflow: + +1. Export exact secret findings from DeepSource's secret detector UI/API. +2. Classify each occurrence as real secret, generated fixture, documented + placeholder, or detector false positive. +3. For real secrets, rotate or revoke the credential before deleting it from + the repository. +4. For fixture or placeholder strings, replace them with unmistakable dummy + values or move them into ignored test fixtures where appropriate. +5. Suppress only verified false positives, with a written rationale. + +Expected effect: + +| Item | Count | +|------|-------| +| Current secrets widget | 674 | +| Stage target | 0 active real or unsuppressed secret findings | + +Verification gate: + +1. DeepSource secrets widget is 0, or every remaining item is an explicit, + documented false-positive suppression. +2. Any rotated credentials are confirmed invalid in the upstream service. +3. The new headline active count is 300 or fewer, or Stage 2 starts with the + updated count. + +Do not silence secret findings without reviewing the exact path and value +class. A false positive can be suppressed; a real secret must be rotated first. + +### Stage 2: Reduce Coverage Noise If Needed + +The snapshot reports 480 active coverage occurrences. This bucket is high +volume but lower security risk than secrets. Start it only if Stage 0 and +Stage 1 do not bring the headline count to 300 or fewer. + +Required workflow: + +1. Export coverage occurrences and group them by package/module. +2. Split production code from generated, vendored, optional, and UI-only + paths. +3. Exclude paths that are not meaningful coverage targets. +4. Add focused smoke or unit tests only for production code with realistic + behavior risk. + +Verification gate: + +1. Coverage findings are reduced enough to keep the headline count at or below + 300. +2. Any coverage exclusions are scoped to generated, fixture, optional, or + otherwise intentionally unmeasured paths. +3. New tests run under the repo's conda workflow. + +### Stage 3: Clean Up Remaining Production Findings + +After scope, secrets, and any required coverage work, handle the low-volume +production findings in small pull requests: + +| Family | Count | Approach | +|--------|-------|----------| +| `PY-R1000` | 5 | Refactor only when the function boundary is clear and tests can pin behavior | +| `PYL-R0401` | 1 | Break the import cycle with a local import or dependency inversion | +| `PY-W2000` | Remaining | Remove unused imports only in production code | +| `PY-W0069` | Remaining | Delete dead commented code when it is not useful documentation | +| `PTC-W0043` | Remaining | Remove unnecessary local `del` statements | +| `PTC-W0048` | Remaining | Merge simple adjacent conditionals | + +Verification gate: + +1. Each batch has targeted tests or a documented reason tests are not needed. +2. `git diff --check` passes. +3. DeepSource no longer reports the targeted issue family on the affected + branch. + +## Count Model + +The count model should be refreshed after every DeepSource reanalysis: + +| Step | Confirmed or expected movement | Expected active count | +|------|--------------------------------|-----------------------| +| Snapshot | Current issue page | 1259 | +| Stage 0 | Python branch check already shows `0` raised and `18` resolved; default-branch page movement waits for merge/reanalysis | lower, exact value pending | +| Stage 1 | Remove or legitimately suppress 674 secret occurrences | likely below 300 if Stage 0 page movement is reflected | +| Stage 2 | Reduce coverage only if the headline remains above target | 300 or fewer | +| Stage 3 | Clean residual production findings | Keep below 300 and reduce future regression risk | + +Because the widgets overlap, this model is intentionally conservative. The only +hard pass condition is the live DeepSource headline count after reanalysis. + +## API Refresh Checklist + +Use the DeepSource playbook for authenticated access. Save raw outputs for each +refresh and record the run ID, commit SHA, and timestamp. + +Minimum refresh artifacts: + +1. Issue page HTML with the active occurrence widgets. +2. Default-branch run detail for the latest analysis. +3. PR or follow-up branch run detail. +4. Per-check issue export for Python, secrets, and coverage. +5. Grouped issue-family summary with counts by rule and path. + +## Stop Conditions + +The reduction project is done when all of these are true: + +1. DeepSource default-branch reanalysis reports 300 or fewer active + occurrences. +2. The secrets bucket has no unresolved real credentials. +3. The Python check no longer audits `server/tests/**` as production code. +4. Any remaining active issues are documented as intentionally deferred with an + owner, rule family, and reason. diff --git a/docs/dev-guide/index.md b/docs/dev-guide/index.md index 9fbbaee4..47f6f3fe 100644 --- a/docs/dev-guide/index.md +++ b/docs/dev-guide/index.md @@ -49,3 +49,5 @@ solvers, or make changes to the core infrastructure. triaging and fixing Codacy findings in risk-prioritized batches. - **[DeepSource Fix Playbook](codex-deepsource.md)** -- AI-assisted workflow for triaging and fixing DeepSource findings with minimal, verified diffs. +- **[DeepSource Reduction Plan](deepsource-reduction-plan.md)** -- Current + issue-count snapshot and staged plan to reduce active findings below 300. From 39e911e5c32ed5383ebd3365aff328eb683ee97d Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:23:18 +0800 Subject: [PATCH 04/63] docs(deepsource): drop reduction plan --- docs/dev-guide/deepsource-reduction-plan.md | 231 -------------------- docs/dev-guide/index.md | 2 - 2 files changed, 233 deletions(-) delete mode 100644 docs/dev-guide/deepsource-reduction-plan.md diff --git a/docs/dev-guide/deepsource-reduction-plan.md b/docs/dev-guide/deepsource-reduction-plan.md deleted file mode 100644 index b408b44f..00000000 --- a/docs/dev-guide/deepsource-reduction-plan.md +++ /dev/null @@ -1,231 +0,0 @@ -# DeepSource Active Issue Reduction Plan - -Snapshot date: 2026-07-16 - -Target: reduce the DeepSource headline active occurrence count from about -1.3k to 300 or fewer, without masking real security findings. - -## Current Evidence - -DeepSource's issue page for the default branch reported these active occurrence -widgets in the latest snapshot: - -| Widget | Count | -|--------|-------| -| All active occurrences | 1259 | -| Recommended | 855 | -| Secrets | 674 | -| Coverage | 480 | -| Antipattern | 508 | -| Style | 168 | -| Performance | 61 | -| Bug risk | 42 | -| Docs | 0 | -| Typecheck | 0 | - -These widgets overlap and must not be added together. - -The latest default-branch analysis in the snapshot was: - -| Field | Value | -|-------|-------| -| Branch | `main` | -| Commit | `57c36bcc5e5b3d6261b5f9f8cdaeb1690f00a41e` | -| Analysis run | `9ea7977c-250b-4dbd-abd2-a5b02ae9e62e` | -| Status | `FAIL` | -| Finished | `2026-07-15T16:34:11` | - -The active PR branch for the first remediation batch was: - -| Field | Value | -|-------|-------| -| Branch | `codex/deepsource-audit-fixes` | -| Commit | `8d7d1f56dfd02dc2b7a2e6b7ceaf81736183b3b2` | -| Analysis run | `769adfbb-aaf5-48c5-a111-0ff6c6436c6d` | -| Status | `PASS` | -| Python check result | `0` issues raised, `18` resolved | -| Finished | `2026-07-16T03:40:15` | - -The default-branch Python check exported through the API contained 519 issue -nodes: - -| Category | Count | -|----------|-------| -| Security | 453 | -| Antipattern | 59 | -| Style | 3 | -| Bug risk | 2 | -| Performance | 2 | - -High-volume Python issue families on `main` were: - -| Rule | Count | Main cause | -|------|-------|------------| -| `BAN-B101` | 444 | `assert` statements in `server/tests/**` treated as production security findings | -| `BAN-B103` | 4 | File-permission test helpers in `server/tests/**` treated as production security findings | -| `PY-W2000` | 33-36 | Unused imports, mostly test and server cleanup | -| `PY-W0069` | 7 | Commented-out code blocks | -| `PTC-W0043` | 6 | Unnecessary local `del` statements | -| `PY-R1000` | 5 | High cyclomatic complexity | -| `PTC-W0048` | 2 | Mergeable `if` statements | -| `PYL-R0401` | 1 | Cyclic import | - -Evidence files from this snapshot: - -| File | Purpose | -|------|---------| -| `/tmp/deepsource_plan_issues.html` | Server-rendered issue page snapshot | -| `/tmp/deepsource_all_issues_fullscope.json` | Default-branch Python issue export | -| `/tmp/deepsource_main_concrete_all_plan.json` | Grouped default-branch issue families | -| `/tmp/deepsource_run_detail_fullscope.json` | Default-branch run detail | -| `/tmp/deepsource_pr_run_detail_plan.json` | PR branch run detail | - -## Reduction Strategy - -To reach 300 from 1259, the project must remove or legitimately resolve at -least 959 active occurrences. The fastest safe path is not to rewrite test -asserts one by one; it is to fix scope first, then handle secrets, then use -coverage and low-volume code quality work only as needed. - -### Stage 0: Merge the Python Scope Fix - -PR #193 adds `server/tests/**` to DeepSource's test/exclude configuration and -Bandit's test exclusion, then fixes the small production findings surfaced in -`server/pssm_gremlin_server`. - -Expected effect: - -| Metric | Before | After PR branch | -|--------|--------|-----------------| -| Python issues raised in run | 497 | 0 | -| Python check status | `FAIL` | `PASS` | -| Test-noise security findings | 448+ | 0 on PR branch | - -Verification gate: - -1. Merge PR #193. -2. Wait for a default-branch DeepSource reanalysis. -3. Confirm the default-branch Python check no longer raises the - `server/tests/**` assert and file-permission findings. -4. Record the new `all` active occurrence count before starting Stage 1. - -Do not spend time replacing pytest `assert` usage in `server/tests/**`; those -are test semantics, not production security defects. - -### Stage 1: Resolve the Secrets Bucket - -The issue page reports 674 active secret occurrences. This is the largest -security bucket and should be treated as the highest-risk remaining work after -the Python scope fix. - -Required workflow: - -1. Export exact secret findings from DeepSource's secret detector UI/API. -2. Classify each occurrence as real secret, generated fixture, documented - placeholder, or detector false positive. -3. For real secrets, rotate or revoke the credential before deleting it from - the repository. -4. For fixture or placeholder strings, replace them with unmistakable dummy - values or move them into ignored test fixtures where appropriate. -5. Suppress only verified false positives, with a written rationale. - -Expected effect: - -| Item | Count | -|------|-------| -| Current secrets widget | 674 | -| Stage target | 0 active real or unsuppressed secret findings | - -Verification gate: - -1. DeepSource secrets widget is 0, or every remaining item is an explicit, - documented false-positive suppression. -2. Any rotated credentials are confirmed invalid in the upstream service. -3. The new headline active count is 300 or fewer, or Stage 2 starts with the - updated count. - -Do not silence secret findings without reviewing the exact path and value -class. A false positive can be suppressed; a real secret must be rotated first. - -### Stage 2: Reduce Coverage Noise If Needed - -The snapshot reports 480 active coverage occurrences. This bucket is high -volume but lower security risk than secrets. Start it only if Stage 0 and -Stage 1 do not bring the headline count to 300 or fewer. - -Required workflow: - -1. Export coverage occurrences and group them by package/module. -2. Split production code from generated, vendored, optional, and UI-only - paths. -3. Exclude paths that are not meaningful coverage targets. -4. Add focused smoke or unit tests only for production code with realistic - behavior risk. - -Verification gate: - -1. Coverage findings are reduced enough to keep the headline count at or below - 300. -2. Any coverage exclusions are scoped to generated, fixture, optional, or - otherwise intentionally unmeasured paths. -3. New tests run under the repo's conda workflow. - -### Stage 3: Clean Up Remaining Production Findings - -After scope, secrets, and any required coverage work, handle the low-volume -production findings in small pull requests: - -| Family | Count | Approach | -|--------|-------|----------| -| `PY-R1000` | 5 | Refactor only when the function boundary is clear and tests can pin behavior | -| `PYL-R0401` | 1 | Break the import cycle with a local import or dependency inversion | -| `PY-W2000` | Remaining | Remove unused imports only in production code | -| `PY-W0069` | Remaining | Delete dead commented code when it is not useful documentation | -| `PTC-W0043` | Remaining | Remove unnecessary local `del` statements | -| `PTC-W0048` | Remaining | Merge simple adjacent conditionals | - -Verification gate: - -1. Each batch has targeted tests or a documented reason tests are not needed. -2. `git diff --check` passes. -3. DeepSource no longer reports the targeted issue family on the affected - branch. - -## Count Model - -The count model should be refreshed after every DeepSource reanalysis: - -| Step | Confirmed or expected movement | Expected active count | -|------|--------------------------------|-----------------------| -| Snapshot | Current issue page | 1259 | -| Stage 0 | Python branch check already shows `0` raised and `18` resolved; default-branch page movement waits for merge/reanalysis | lower, exact value pending | -| Stage 1 | Remove or legitimately suppress 674 secret occurrences | likely below 300 if Stage 0 page movement is reflected | -| Stage 2 | Reduce coverage only if the headline remains above target | 300 or fewer | -| Stage 3 | Clean residual production findings | Keep below 300 and reduce future regression risk | - -Because the widgets overlap, this model is intentionally conservative. The only -hard pass condition is the live DeepSource headline count after reanalysis. - -## API Refresh Checklist - -Use the DeepSource playbook for authenticated access. Save raw outputs for each -refresh and record the run ID, commit SHA, and timestamp. - -Minimum refresh artifacts: - -1. Issue page HTML with the active occurrence widgets. -2. Default-branch run detail for the latest analysis. -3. PR or follow-up branch run detail. -4. Per-check issue export for Python, secrets, and coverage. -5. Grouped issue-family summary with counts by rule and path. - -## Stop Conditions - -The reduction project is done when all of these are true: - -1. DeepSource default-branch reanalysis reports 300 or fewer active - occurrences. -2. The secrets bucket has no unresolved real credentials. -3. The Python check no longer audits `server/tests/**` as production code. -4. Any remaining active issues are documented as intentionally deferred with an - owner, rule family, and reason. diff --git a/docs/dev-guide/index.md b/docs/dev-guide/index.md index 47f6f3fe..9fbbaee4 100644 --- a/docs/dev-guide/index.md +++ b/docs/dev-guide/index.md @@ -49,5 +49,3 @@ solvers, or make changes to the core infrastructure. triaging and fixing Codacy findings in risk-prioritized batches. - **[DeepSource Fix Playbook](codex-deepsource.md)** -- AI-assisted workflow for triaging and fixing DeepSource findings with minimal, verified diffs. -- **[DeepSource Reduction Plan](deepsource-reduction-plan.md)** -- Current - issue-count snapshot and staged plan to reduce active findings below 300. From 58ccb65f01a7e5e7de504e6ea6ceadcbd6a44625 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:33:58 +0800 Subject: [PATCH 05/63] ci(deepsource): upload coverage reports --- .github/workflows/server-test.yml | 11 +++++++++++ .github/workflows/unit_tests_tag.yml | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml index 8d56e61c..fcf23915 100644 --- a/.github/workflows/server-test.yml +++ b/.github/workflows/server-test.yml @@ -11,6 +11,10 @@ on: paths: - "server/**" +permissions: + contents: read + id-token: write + jobs: ServerTests: runs-on: ubuntu-latest @@ -30,6 +34,8 @@ jobs: - name: Checkout Repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup Python uses: actions/setup-python@v6 @@ -47,6 +53,11 @@ jobs: python -m pytest server/tests/ -v --cov=pssm_gremlin_server --cov-report=xml --override-ini="addopts=" \ -k "not Docker and not docker and not runner_image and not server_image and not test_server_image and not test_runner_image and not test_server_rejects and not test_server_reports and not running_gremlin" + - name: Upload coverage report to DeepSource + run: | + curl -fsSL https://cli.deepsource.com/install | BINDIR=./bin sh + ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml --use-oidc + - name: Upload coverage to Codecov uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: diff --git a/.github/workflows/unit_tests_tag.yml b/.github/workflows/unit_tests_tag.yml index 0ce16ac7..3f61fc62 100644 --- a/.github/workflows/unit_tests_tag.yml +++ b/.github/workflows/unit_tests_tag.yml @@ -9,6 +9,10 @@ on: release: types: [created] +permissions: + contents: read + id-token: write + env: conda_env: REvoDesign @@ -103,6 +107,8 @@ jobs: - name: Checkout Repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: setup Qt headless Display on Ubuntu if: startsWith(matrix.os, 'ubuntu') @@ -185,6 +191,12 @@ jobs: run: | coverage xml + - name: Upload coverage report to DeepSource + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' && matrix.qt-version == '6' + run: | + curl -fsSL https://cli.deepsource.com/install | BINDIR=./bin sh + ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml --use-oidc + - name: Upload coverage reports to Codecov uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: From 883ba78553b7076ae1cadfddfba35b3beb943e4a Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:39:54 +0800 Subject: [PATCH 06/63] Revert "ci(deepsource): upload coverage reports" [ci skip] This reverts commit 58ccb65f01a7e5e7de504e6ea6ceadcbd6a44625. --- .github/workflows/server-test.yml | 11 ----------- .github/workflows/unit_tests_tag.yml | 12 ------------ 2 files changed, 23 deletions(-) diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml index fcf23915..8d56e61c 100644 --- a/.github/workflows/server-test.yml +++ b/.github/workflows/server-test.yml @@ -11,10 +11,6 @@ on: paths: - "server/**" -permissions: - contents: read - id-token: write - jobs: ServerTests: runs-on: ubuntu-latest @@ -34,8 +30,6 @@ jobs: - name: Checkout Repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup Python uses: actions/setup-python@v6 @@ -53,11 +47,6 @@ jobs: python -m pytest server/tests/ -v --cov=pssm_gremlin_server --cov-report=xml --override-ini="addopts=" \ -k "not Docker and not docker and not runner_image and not server_image and not test_server_image and not test_runner_image and not test_server_rejects and not test_server_reports and not running_gremlin" - - name: Upload coverage report to DeepSource - run: | - curl -fsSL https://cli.deepsource.com/install | BINDIR=./bin sh - ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml --use-oidc - - name: Upload coverage to Codecov uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: diff --git a/.github/workflows/unit_tests_tag.yml b/.github/workflows/unit_tests_tag.yml index 3f61fc62..0ce16ac7 100644 --- a/.github/workflows/unit_tests_tag.yml +++ b/.github/workflows/unit_tests_tag.yml @@ -9,10 +9,6 @@ on: release: types: [created] -permissions: - contents: read - id-token: write - env: conda_env: REvoDesign @@ -107,8 +103,6 @@ jobs: - name: Checkout Repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: setup Qt headless Display on Ubuntu if: startsWith(matrix.os, 'ubuntu') @@ -191,12 +185,6 @@ jobs: run: | coverage xml - - name: Upload coverage report to DeepSource - if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' && matrix.qt-version == '6' - run: | - curl -fsSL https://cli.deepsource.com/install | BINDIR=./bin sh - ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml --use-oidc - - name: Upload coverage reports to Codecov uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: From ea08200758bb91bbe47d703602e8b3edc2676015 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:48:37 +0800 Subject: [PATCH 07/63] fix(security): use cryptographic password randomness --- CHANGELOG.md | 3 +++ src/REvoDesign/tools/utils.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07bb0a4f..789479d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,6 +130,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Renamed Simplified Chinese label from 中文 to 简体中文 in language registry (`language.json`). ### Fixed +- **Generated socket authentication keys**: `generate_strong_password()` now uses + `secrets.choice` instead of the default pseudo-random generator when creating + authentication/password strings. - **Uploaded FASTA filename collision**: two users uploading files with the same name (e.g. `seqs.fasta`) would overwrite each other in the shared upload directory. Files are now saved as `{owner_scoped_md5}.fasta` so on-disk names diff --git a/src/REvoDesign/tools/utils.py b/src/REvoDesign/tools/utils.py index bb14c318..737ddf5b 100644 --- a/src/REvoDesign/tools/utils.py +++ b/src/REvoDesign/tools/utils.py @@ -14,6 +14,7 @@ import inspect import os import random +import secrets import shutil import string import sys @@ -821,8 +822,7 @@ def generate_strong_password(length=16): # Define the characters to use for generating the password password_characters = string.ascii_letters + string.digits + "!#$%&*+-./:?@^_~" - # Generate the password using random characters from the defined set - generated_password = "".join(random.choice(password_characters) for _ in range(length)) + generated_password = "".join(secrets.choice(password_characters) for _ in range(length)) return generated_password From 73abc914ede5656218299bffa8311bcf9e49717f Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:56:47 +0800 Subject: [PATCH 08/63] fix(security): require verified download hashes --- CHANGELOG.md | 4 ++++ src/REvoDesign/editor/monaco/monaco.py | 27 ++++++++++++++++++++++- src/REvoDesign/shortcuts/tools/esm2.py | 14 +++++------- src/REvoDesign/tools/download_registry.py | 15 +++++++++++-- tests/editor/test_monaco.py | 6 ++++- tests/tools/test_file_downloader.py | 6 ++++- 6 files changed, 59 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 789479d3..17070504 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,6 +130,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Renamed Simplified Chinese label from 中文 to 简体中文 in language registry (`language.json`). ### Fixed +- **Download registry integrity checks**: file registries now reject entries + without hashes, Monaco editor downloads verify npm tarball integrity metadata, + and remote ESM weight downloads fail closed until verified checksums are + configured. - **Generated socket authentication keys**: `generate_strong_password()` now uses `secrets.choice` instead of the default pseudo-random generator when creating authentication/password strings. diff --git a/src/REvoDesign/editor/monaco/monaco.py b/src/REvoDesign/editor/monaco/monaco.py index 9b433190..faa35c2c 100644 --- a/src/REvoDesign/editor/monaco/monaco.py +++ b/src/REvoDesign/editor/monaco/monaco.py @@ -3,10 +3,12 @@ # SPDX-License-Identifier: GPL-3.0-only +import base64 import os import shutil from urllib.parse import urlencode +import requests from platformdirs import user_data_dir from REvoDesign import issues @@ -21,6 +23,29 @@ logging = ROOT_LOGGER.getChild(__name__) +def _npm_integrity_to_pooch_hash(integrity: str) -> str: + hash_name, hash_value = integrity.split("-", 1) + return f"{hash_name}:{base64.b64decode(hash_value).hex()}" + + +def _get_monaco_tarball_hash(version: str) -> str: + metadata_url = f"https://registry.npmjs.org/monaco-editor/{version}" + response = requests.get(metadata_url, timeout=30) + response.raise_for_status() + package_metadata = response.json() + + tarball_dist = package_metadata.get("dist", {}) + integrity = tarball_dist.get("integrity") + if integrity: + return _npm_integrity_to_pooch_hash(integrity) + + shasum = tarball_dist.get("shasum") + if shasum: + return f"sha1:{shasum}" + + raise RuntimeError(f"Could not find npm integrity metadata for monaco-editor {version}") + + class MonacoEditorManager: def __init__(self, app_name="YourAppName", app_author="YourCompany", version="latest"): """ @@ -100,7 +125,7 @@ def download_monaco_editor(self, version="latest"): down_registry = FileDownloadRegistry( name="monaco-editor", base_url=cdn_base_url, - registry={f"monaco-editor-{version}.tgz": None}, + registry={f"monaco-editor-{version}.tgz": _get_monaco_tarball_hash(version)}, version=version, customized_directory=self.editor_path, # https://github.com/amio/npm-mirrors/blob/master/index.js diff --git a/src/REvoDesign/shortcuts/tools/esm2.py b/src/REvoDesign/shortcuts/tools/esm2.py index 6363ad67..2381e898 100644 --- a/src/REvoDesign/shortcuts/tools/esm2.py +++ b/src/REvoDesign/shortcuts/tools/esm2.py @@ -26,7 +26,6 @@ from REvoDesign import issues from REvoDesign.basic.abc_third_party_module import ThirdPartyModuleAbstract, TorchModuleAbstract from REvoDesign.bootstrap.set_config import is_package_installed -from REvoDesign.tools.download_registry import FileDownloadRegistry from REvoDesign.tools.utils import get_cited, require_installed from ...logger import ROOT_LOGGER @@ -50,9 +49,7 @@ } ) -ESM1V_WEIGHTS = FileDownloadRegistry( - name="ESM2", base_url=ESM_MODEL_BASE_URL, registry={f"{v}.pt": None for v in ESM1V_MODEL_DICT.values()} -) +ESM1V_WEIGHT_FILES = tuple(f"{v}.pt" for v in ESM1V_MODEL_DICT.values()) def list_all_esm_variant_predict_model_names() -> list[str]: @@ -62,7 +59,7 @@ def list_all_esm_variant_predict_model_names() -> list[str]: Returns: list[str]: List of ESM-1v variant predict model names """ - return ESM1V_WEIGHTS.list_all_files + return list(ESM1V_WEIGHT_FILES) def remove_insertions(sequence: str) -> str: @@ -207,9 +204,10 @@ def _resolve_model_weight(self, model_name: str): logging.info(f"Loading model from {expected_model_path=}") return expected_model_path - # Download model from remote server - logging.info(f"Fetching model {model_name=} from {ESM1V_WEIGHTS.base_url}") - return ESM1V_WEIGHTS.setup(model_name).downloaded + raise issues.ConfigureError( + "Remote ESM model downloads are disabled until verified checksums are configured. " + "Set a checkpoint directory containing the requested model weight file instead." + ) @get_cited def predict(self): diff --git a/src/REvoDesign/tools/download_registry.py b/src/REvoDesign/tools/download_registry.py index 0dcb2faa..d99f94e2 100644 --- a/src/REvoDesign/tools/download_registry.py +++ b/src/REvoDesign/tools/download_registry.py @@ -160,14 +160,25 @@ def _complete_varify_string(a_string: str | None = None, hash_type: str = "md5") return f"{hash_type}:{a_string}" @staticmethod - def preprocess_registry(registry: dict[str, str | None]) -> dict[str, str | None]: + def preprocess_registry(registry: dict[str, str | None]) -> dict[str, str]: """ Preprocess registry to ensure all hash values conform to pooch requirements. :param registry: Original registry. :return: Processed registry. """ - return {k: FileDownloadRegistry._complete_varify_string(v) for k, v in registry.items()} + processed: dict[str, str] = {} + missing_hashes: list[str] = [] + for filename, hash_value in registry.items(): + completed_hash = FileDownloadRegistry._complete_varify_string(hash_value) + if completed_hash is None: + missing_hashes.append(filename) + continue + processed[filename] = completed_hash + if missing_hashes: + missing = ", ".join(sorted(missing_hashes)) + raise ValueError(f"Download registry entries require hashes: {missing}") + return processed @get_cited def setup(self, item: str) -> DownloadedFile: diff --git a/tests/editor/test_monaco.py b/tests/editor/test_monaco.py index 2487dccc..bc82ab2f 100644 --- a/tests/editor/test_monaco.py +++ b/tests/editor/test_monaco.py @@ -11,7 +11,7 @@ import pytest -from REvoDesign.editor.monaco.monaco import MonacoEditorManager, edit_file_with_monaco +from REvoDesign.editor.monaco.monaco import MonacoEditorManager, _npm_integrity_to_pooch_hash, edit_file_with_monaco @pytest.fixture @@ -73,6 +73,10 @@ def test_copy_html_template(test_tmp_dir): mock_copy.assert_called_once_with(manager.html_template_path, os.path.join(version_dir, "index.html")) +def test_npm_integrity_to_pooch_hash(): + assert _npm_integrity_to_pooch_hash("sha512-YWJjMTIz") == "sha512:616263313233" + + def test_edit_file_with_monaco(mock_server_control, mock_config_store, test_tmp_dir, test_worker): file_path = os.path.join(test_tmp_dir, "file.txt") diff --git a/tests/tools/test_file_downloader.py b/tests/tools/test_file_downloader.py index 5afb441a..a1c78009 100644 --- a/tests/tools/test_file_downloader.py +++ b/tests/tools/test_file_downloader.py @@ -35,7 +35,6 @@ def test_complete_varify_string(self, a_string, hash_type, expected): {"file1.txt": "md5:abc123", "file2.txt": "def456"}, {"file1.txt": "md5:abc123", "file2.txt": "md5:def456"}, ), - ({"file1.txt": None, "file2.txt": ""}, {"file1.txt": None, "file2.txt": None}), ({"file1.txt": "sha256:abc123"}, {"file1.txt": "sha256:abc123"}), ], ) @@ -44,6 +43,11 @@ def test_preprocess_registry(self, registry, expected): result = FileDownloadRegistry.preprocess_registry(registry) assert result == expected + def test_preprocess_registry_rejects_missing_hashes(self): + """Test preprocess_registry rejects entries that would disable verification.""" + with pytest.raises(ValueError, match="file1.txt, file2.txt"): + FileDownloadRegistry.preprocess_registry({"file1.txt": None, "file2.txt": ""}) + @pytest.mark.parametrize( "md5_contents, expected", [ From f0b6016b02e8e756ae831f0a3c73991101fc7b88 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:00:57 +0800 Subject: [PATCH 09/63] fix(security): avoid logging editor auth tokens --- CHANGELOG.md | 2 ++ src/REvoDesign/editor/monaco/server.py | 2 +- tests/editor/test_editor_server.py | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17070504..5dae36d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -134,6 +134,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 without hashes, Monaco editor downloads verify npm tarball integrity metadata, and remote ESM weight downloads fail closed until verified checksums are configured. +- **Monaco editor token handling**: generated editor backend authentication + tokens are no longer written to application logs. - **Generated socket authentication keys**: `generate_strong_password()` now uses `secrets.choice` instead of the default pseudo-random generator when creating authentication/password strings. diff --git a/src/REvoDesign/editor/monaco/server.py b/src/REvoDesign/editor/monaco/server.py index 4ad65c62..ebb948dc 100644 --- a/src/REvoDesign/editor/monaco/server.py +++ b/src/REvoDesign/editor/monaco/server.py @@ -107,7 +107,7 @@ def initialize_token() -> str: SECRET_TOKEN = secrets.token_urlsafe(32) config_store.set("editor.token", SECRET_TOKEN) - logging.info(f"Generated SECRET_TOKEN: {SECRET_TOKEN}") + logging.info("Generated editor authentication token.") return SECRET_TOKEN diff --git a/tests/editor/test_editor_server.py b/tests/editor/test_editor_server.py index f367439a..27310794 100644 --- a/tests/editor/test_editor_server.py +++ b/tests/editor/test_editor_server.py @@ -86,6 +86,20 @@ def initialize_server(): # Test cases +def test_initialize_token_does_not_log_secret(monkeypatch, caplog): + from REvoDesign.editor.monaco import server + + monkeypatch.setattr(server.secrets, "token_urlsafe", lambda _size: "sensitive-editor-token") + + with caplog.at_level("INFO", logger=server.logging.name): + token = server.initialize_token() + + assert token == "sensitive-editor-token" + assert ConfigStore().get("editor.token") == "sensitive-editor-token" + assert "sensitive-editor-token" not in caplog.text + assert "Generated editor authentication token." in caplog.text + + def test_favicon(test_client): """Test that the favicon endpoint returns the correct file.""" response = test_client.get("/favicon.svg") From 051cb8697ffb9d19e89f1827e0ab41637ae5be19 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:09:41 +0800 Subject: [PATCH 10/63] fix(deepsource): clean up utility findings --- src/REvoDesign/tools/utils.py | 278 +++++++++++++++++----------------- 1 file changed, 137 insertions(+), 141 deletions(-) diff --git a/src/REvoDesign/tools/utils.py b/src/REvoDesign/tools/utils.py index 737ddf5b..4e324cb1 100644 --- a/src/REvoDesign/tools/utils.py +++ b/src/REvoDesign/tools/utils.py @@ -13,7 +13,6 @@ import importlib import inspect import os -import random import secrets import shutil import string @@ -161,17 +160,16 @@ def resolve_lambda_expression(expression: str, as_partial: bool | None = None) - def resolve_typed_arg(arg: str) -> Any: if arg.lower() == "true": return True - elif arg.lower() == "false": + if arg.lower() == "false": return False # integer - elif arg.isdigit(): + if arg.isdigit(): return int(arg) # float - elif arg.replace(".", "", 1).isdigit(): + if arg.replace(".", "", 1).isdigit(): return float(arg) # string fallback - else: - return arg + return arg def resolve_default_value(typing: type) -> Any: @@ -183,6 +181,7 @@ def resolve_default_value(typing: type) -> Any: return 0.0 if typing == str: return "" + return None def resolve_dotted_config_item(config_string: str) -> Any: @@ -191,7 +190,8 @@ def resolve_dotted_config_item(config_string: str) -> Any: The input string can be in one of the following formats: 1. `"CFG::"` - Specifies both the configuration section and item. - 2. `"CFG:"` - Specifies only the configuration item, with no section(default for the main config section). + 2. `"CFG:"` - Specifies only the configuration item, with + no section(default for the main config section). Args: config_string (str): The dotted configuration string to resolve. @@ -317,6 +317,59 @@ def __init__(self, *args, **kwargs): MethodKind = Literal["InstanceMethod", "ClassMethod", "StaticMethod", "Function"] +def _resolve_function_owner(method: Callable) -> tuple[type | None, str, str]: + qualname = getattr(method, "__qualname__", "") + if "." not in qualname: + return None, "", qualname + + owner_path, func_name = qualname.rsplit(".", 1) + module = inspect.getmodule(method) + if module is None: + return None, owner_path, func_name + + obj: Any = module + for part in owner_path.split("."): + obj = getattr(obj, part, None) + if obj is None: + return None, owner_path, func_name + + return (obj if isinstance(obj, type) else None), owner_path, func_name + + +def _inspect_bound_method(method: Callable) -> MethodKind | None: + self_obj = getattr(method, "__self__", None) + if self_obj is None: + return None + if isinstance(self_obj, type): + logging.debug(f"{method!r} is a classmethod bound to {self_obj!r}") + return "ClassMethod" + logging.debug(f"{method!r} is an instancemethod bound to {self_obj!r}") + return "InstanceMethod" + + +def _inspect_function(method: Callable) -> MethodKind | None: + if not inspect.isfunction(method): + return None + + owner_cls, owner_path, func_name = _resolve_function_owner(method) + if not owner_path: + logging.debug(f"{method!r} is a plain top-level function") + return "Function" + + if owner_cls is None: + logging.debug(f"{method!r} is a dotted qualname but class is not resolvable") + return "StaticMethod" + + attr = owner_cls.__dict__.get(func_name) + if isinstance(attr, staticmethod): + logging.debug(f"{method!r} is a staticmethod on {owner_cls!r}") + return "StaticMethod" + if inspect.isfunction(attr): + logging.debug(f"{method!r} is a function on {owner_cls!r}") + return "Function" + return None + + def inspect_method_types(method: Callable) -> MethodKind: """ Inspect the type of a callable: instance method, class method, @@ -336,76 +389,15 @@ def inspect_method_types(method: Callable) -> MethodKind: if not callable(method): raise issues.UnexpectedWorkflowError(f"Cannot inspect method type for non-callable {method!r}") - # --- 1. Bound methods: have a non-None __self__ ------------------------- - # obj.method -> bound instance method ( __self__ is instance ) - # cls.method -> bound class method ( __self__ is class ) - if hasattr(method, "__self__") and getattr(method, "__self__", None) is not None: - self_obj = method.__self__ # type: ignore[attr-defined] - # Bound to a class -> classmethod - if isinstance(self_obj, type): - logging.debug(f"{method!r} is a classmethod bound to {self_obj!r}") - return "ClassMethod" - logging.debug(f"{method!r} is an instancemethod bound to {self_obj!r}") - # Bound to an instance -> instance method - return "InstanceMethod" - - # --- 2. Unbound functions (no binding) ----------------------------------- - # At this point, typical Python methods accessed from the class are plain - # function objects (unbound). We must check how they are stored on the class. - if inspect.isfunction(method): - qualname = getattr(method, "__qualname__", "") - logging.debug(f"Checking unbound function {method}: {getattr(method, '__module__', '')!r}") - - # Try to resolve the owning class from qualname: "MyClass.method" - if "." in qualname: - owner_path, func_name = qualname.rsplit(".", 1) - - owner_cls = None - module = inspect.getmodule(method) - if module is not None: - obj = module - # Walk the dotted path "Outer.Inner" etc. - for part in owner_path.split("."): - if not hasattr(obj, part): - obj = None - break - obj = getattr(obj, part) - if isinstance(obj, type): - owner_cls = obj - - if owner_cls is not None: - # Look at the raw attribute on the class dict - attr = owner_cls.__dict__.get(func_name) - # Static method is stored as a staticmethod descriptor - if isinstance(attr, staticmethod): - logging.debug(f"{method!r} is a staticmethod on {owner_cls!r}") - return "StaticMethod" - # A normal "def" on the class body: unbound instance method - if inspect.isfunction(attr): - logging.debug(f"{method!r} is a function on {owner_cls!r}") - return "Function" - - # Fallback: dotted qualname but class not resolvable - # (e.g. local classes with '' in qualname). - # In your semantics we still treat this as a static-style method. - logging.debug(f"{method!r} is a dotted qualname but class is not resolvable") - return "StaticMethod" - - # No dot in qualname: a plain top-level function - logging.debug(f"{method!r} is a plain top-level function") - return "Function" + detected = _inspect_bound_method(method) or _inspect_function(method) + if detected is not None: + return detected - # --- 3. Rare cases: method-like objects not covered above --------------- if inspect.ismethod(method): - # This branch is mostly for odd cases / builtins. - self_obj = method.__self__ - if isinstance(self_obj, type): - logging.debug(f"{method!r} is a method-like classmethod object") - return "ClassMethod" - logging.debug(f"{method!r} is a method-like instancemethod object") - return "InstanceMethod" - - # --- 4. Give up ---------------------------------------------------------- + detected = _inspect_bound_method(method) + if detected is not None: + return detected + logging.error(f"Giving up for inpection on method type! {method!r}") raise TypeError(f"Cannot inspect method type for {method!r}") @@ -484,10 +476,11 @@ def _cite_for_cls(cls_or_obj: type[CitableModuleAbstract] | CitableModuleAbstrac cls_or_obj: A class or instance object of type CitableModuleAbstract """ try: - if not (hasattr(cls_or_obj, "cite") and callable(getattr(cls_or_obj, "cite"))): + cite_method = getattr(cls_or_obj, "cite", None) + if not callable(cite_method): name = getattr(cls_or_obj, "__name__", type(cls_or_obj).__name__) raise TypeError(f"{name} is not citable or missing cite()") - cls_or_obj.cite() + cite_method() except Exception as e: logging.warning(f"Ignore cite() error: {e}") @@ -597,7 +590,6 @@ def data_generator(): """ current_batch = [] for data_point in inputs_data_generator: - # print(f"Send data {data_point}") current_batch.append(data_point) if len(current_batch) == batch_size: yield current_batch @@ -608,79 +600,83 @@ def data_generator(): yield current_batch +def _safe_member_target(base_dir: str, member_name: str) -> str: + normalized = member_name.replace("\\", "/") + if normalized.startswith("/") or normalized.startswith("../") or "/../" in f"/{normalized}": + raise ValueError(f"Unsafe archive member path: {member_name}") + target_path = os.path.realpath(os.path.join(base_dir, member_name)) + base_path = os.path.realpath(base_dir) + try: + if os.path.commonpath([base_path, target_path]) != base_path: + raise ValueError(f"Unsafe archive member path: {member_name}") + except ValueError as exc: + raise ValueError(f"Unsafe archive member path: {member_name}") from exc + return target_path + + +def _extract_zip_safely(zip_ref: zipfile.ZipFile, base_dir: str) -> None: + for member in zip_ref.infolist(): + target_path = _safe_member_target(base_dir, member.filename) + if member.is_dir(): + os.makedirs(target_path, exist_ok=True) + continue + os.makedirs(os.path.dirname(target_path), exist_ok=True) + with zip_ref.open(member, "r") as source, open(target_path, "wb") as dest: + shutil.copyfileobj(source, dest) + + +def _extract_tar_safely(tar_ref: tarfile.TarFile, base_dir: str) -> None: + for member in tar_ref.getmembers(): + target_path = _safe_member_target(base_dir, member.name) + if member.isdir(): + os.makedirs(target_path, exist_ok=True) + continue + if not member.isfile(): + logging.warning("Skipping non-regular archive member: %s", member.name) + continue + stream = tar_ref.extractfile(member) + if stream is None: + raise ValueError(f"Failed to read archive member: {member.name}") + os.makedirs(os.path.dirname(target_path), exist_ok=True) + with contextlib.closing(stream) as source, open(target_path, "wb") as dest: + shutil.copyfileobj(source, dest) + + +def _extract_archive_contents(archive_file: str, extract_to: str) -> None: + if archive_file.endswith(".zip"): + with zipfile.ZipFile(archive_file, "r") as zip_ref: + _extract_zip_safely(zip_ref, extract_to) + return + + tar_modes = { + ".tar": "r:", + ".tar.gz": "r:*", + ".tgz": "r:*", + ".tar.bz2": "r:bz2", + ".tbz": "r:bz2", + ".tar.xz": "r:xz", + } + for suffix, mode in tar_modes.items(): + if archive_file.endswith(suffix): + with tarfile.open(archive_file, mode) as tar_ref: + _extract_tar_safely(tar_ref, extract_to) + return + + raise ValueError(f"Unsupported archive format: {archive_file}") + + def extract_archive(archive_file: str, extract_to: str): """ - Extracts the contents of an archive file (zip, tar.gz, tar.bz2, tar.xz, or rar) to a specified directory. + Extracts an archive file (zip, tar, tar.gz, tar.bz2, or tar.xz) to a directory. Args: archive_file (str): Path to the archive file. extract_to (str): Directory where the contents will be extracted. """ - - def _safe_member_target(base_dir: str, member_name: str) -> str: - normalized = member_name.replace("\\", "/") - if normalized.startswith("/") or normalized.startswith("../") or "/../" in f"/{normalized}": - raise ValueError(f"Unsafe archive member path: {member_name}") - target_path = os.path.realpath(os.path.join(base_dir, member_name)) - base_path = os.path.realpath(base_dir) - try: - if os.path.commonpath([base_path, target_path]) != base_path: - raise ValueError(f"Unsafe archive member path: {member_name}") - except ValueError as exc: - raise ValueError(f"Unsafe archive member path: {member_name}") from exc - return target_path - - def _extract_zip_safely(zip_ref: zipfile.ZipFile, base_dir: str) -> None: - for member in zip_ref.infolist(): - target_path = _safe_member_target(base_dir, member.filename) - if member.is_dir(): - os.makedirs(target_path, exist_ok=True) - continue - os.makedirs(os.path.dirname(target_path), exist_ok=True) - with zip_ref.open(member, "r") as source, open(target_path, "wb") as dest: - shutil.copyfileobj(source, dest) - - def _extract_tar_safely(tar_ref: tarfile.TarFile, base_dir: str) -> None: - for member in tar_ref.getmembers(): - target_path = _safe_member_target(base_dir, member.name) - if member.isdir(): - os.makedirs(target_path, exist_ok=True) - continue - if not member.isfile(): - logging.warning("Skipping non-regular archive member: %s", member.name) - continue - stream = tar_ref.extractfile(member) - if stream is None: - raise ValueError(f"Failed to read archive member: {member.name}") - os.makedirs(os.path.dirname(target_path), exist_ok=True) - with contextlib.closing(stream) as source, open(target_path, "wb") as dest: - shutil.copyfileobj(source, dest) - try: with timing(f"extracting {archive_file} to {extract_to}"): os.makedirs(extract_to, exist_ok=True) - if archive_file.endswith(".zip"): - with zipfile.ZipFile(archive_file, "r") as zip_ref: - _extract_zip_safely(zip_ref, extract_to) - - elif archive_file.endswith(".tar"): - with tarfile.open(archive_file, "r:") as tar_ref: - _extract_tar_safely(tar_ref, extract_to) - - elif archive_file.endswith((".tar.gz", ".tgz")): - with tarfile.open(archive_file, "r:*") as tar_ref: - _extract_tar_safely(tar_ref, extract_to) - - elif archive_file.endswith((".tar.bz2", ".tbz")): - with tarfile.open(archive_file, "r:bz2") as tar_ref: - _extract_tar_safely(tar_ref, extract_to) - - elif archive_file.endswith(".tar.xz"): - with tarfile.open(archive_file, "r:xz") as tar_ref: - _extract_tar_safely(tar_ref, extract_to) - - else: - raise ValueError(f"Unsupported archive format: {archive_file}") + _extract_archive_contents(archive_file, extract_to) except Exception as e: logging.error(f"Error extracting {archive_file}: {str(e)}") raise ValueError(f"Failed to extract {archive_file}: {e}") from e From 7790f417d24acb2daa7a0b2f7cc368dbc13f030c Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:14:49 +0800 Subject: [PATCH 11/63] fix(security): verify ESM model downloads --- CHANGELOG.md | 3 +-- src/REvoDesign/shortcuts/tools/esm2.py | 27 +++++++++++++++---- .../customized_widgets/test_shortcuts_util.py | 9 ++++++- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dae36d3..a722eb10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -132,8 +132,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **Download registry integrity checks**: file registries now reject entries without hashes, Monaco editor downloads verify npm tarball integrity metadata, - and remote ESM weight downloads fail closed until verified checksums are - configured. + and remote ESM weight downloads verify configured MD5 hashes. - **Monaco editor token handling**: generated editor backend authentication tokens are no longer written to application logs. - **Generated socket authentication keys**: `generate_strong_password()` now uses diff --git a/src/REvoDesign/shortcuts/tools/esm2.py b/src/REvoDesign/shortcuts/tools/esm2.py index 2381e898..da5139f9 100644 --- a/src/REvoDesign/shortcuts/tools/esm2.py +++ b/src/REvoDesign/shortcuts/tools/esm2.py @@ -26,6 +26,7 @@ from REvoDesign import issues from REvoDesign.basic.abc_third_party_module import ThirdPartyModuleAbstract, TorchModuleAbstract from REvoDesign.bootstrap.set_config import is_package_installed +from REvoDesign.tools.download_registry import FileDownloadRegistry from REvoDesign.tools.utils import get_cited, require_installed from ...logger import ROOT_LOGGER @@ -37,6 +38,19 @@ ESM_MODEL_BASE_URL = "https://dl.fbaipublicfiles.com/fair-esm/models" +# run from yinying's download archive. +# md5sum calculated by using: +# `parallel -k md5sum {} ::: $(ls esm1v_t33* esm_msa1b_t12_100M_UR50S.pt esm2_t36_3B_UR50D.pt)` +ESM1V_MODEL_MD5 = """ +1f04c2d2636b02b544ecb5fbbef8fefd esm1v_t33_650M_UR90S_1.pt +e6c58fc8107d5e6629f9ca9b910319ec esm1v_t33_650M_UR90S_2.pt +6a1a631a87cf798f72f58c0a37aeebdd esm1v_t33_650M_UR90S_3.pt +fcba894a07ec87dfea4ca366e46c78b7 esm1v_t33_650M_UR90S_4.pt +b1649837610e741f8b766b01f18890a3 esm1v_t33_650M_UR90S_5.pt +d37a0d0dbe7431e48a72072b9180b16b esm2_t36_3B_UR50D.pt +f66445cc186f77dbbbe3184399523413 esm_msa1b_t12_100M_UR50S.pt +""" + ESM1V_MODEL_DICT: immutabledict[str, str] = immutabledict( { "esm-1v_1": "esm1v_t33_650M_UR90S_1", @@ -50,6 +64,11 @@ ) ESM1V_WEIGHT_FILES = tuple(f"{v}.pt" for v in ESM1V_MODEL_DICT.values()) +ESM1V_WEIGHTS = FileDownloadRegistry( + name="ESM2", + base_url=ESM_MODEL_BASE_URL, + registry=FileDownloadRegistry.prepare_registry_from_md5(ESM1V_MODEL_MD5), +) def list_all_esm_variant_predict_model_names() -> list[str]: @@ -59,7 +78,7 @@ def list_all_esm_variant_predict_model_names() -> list[str]: Returns: list[str]: List of ESM-1v variant predict model names """ - return list(ESM1V_WEIGHT_FILES) + return ESM1V_WEIGHTS.list_all_files def remove_insertions(sequence: str) -> str: @@ -204,10 +223,8 @@ def _resolve_model_weight(self, model_name: str): logging.info(f"Loading model from {expected_model_path=}") return expected_model_path - raise issues.ConfigureError( - "Remote ESM model downloads are disabled until verified checksums are configured. " - "Set a checkpoint directory containing the requested model weight file instead." - ) + logging.info(f"Fetching model {model_name=} from {ESM1V_WEIGHTS.base_url}") + return ESM1V_WEIGHTS.setup(model_name).downloaded @get_cited def predict(self): diff --git a/tests/tools/customized_widgets/test_shortcuts_util.py b/tests/tools/customized_widgets/test_shortcuts_util.py index 34ffa96d..c8480465 100644 --- a/tests/tools/customized_widgets/test_shortcuts_util.py +++ b/tests/tools/customized_widgets/test_shortcuts_util.py @@ -9,7 +9,7 @@ from REvoDesign import issues from REvoDesign.basic.extensions import resolve_extension from REvoDesign.common.file_extensions import ExtColl -from REvoDesign.shortcuts.tools.esm2 import list_all_esm_variant_predict_model_names +from REvoDesign.shortcuts.tools.esm2 import ESM1V_WEIGHTS, list_all_esm_variant_predict_model_names from REvoDesign.shortcuts.utils import REGISTRY_DIR, _build_asked_value, resolve_choice_from, resolve_default_value from REvoDesign.tools.utils import resolve_dotted_function @@ -78,6 +78,13 @@ def test_resolve_choice_from(input_str, expected_type, expected_val): assert result == expected_val +def test_esm_model_registry_uses_hashes(): + for model_file in list_all_esm_variant_predict_model_names(): + registry_hash = ESM1V_WEIGHTS.registry[model_file] + assert registry_hash.startswith("md5:") + assert len(registry_hash) == len("md5:") + 32 + + @pytest.mark.parametrize( "input_str, error_type", [ From 802bdbea2a82a099094b5965ba0ce4fa2f67c618 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:16:31 +0800 Subject: [PATCH 12/63] fix(dev): keep UI validation active under optimization --- dev/tools/validate_ui_i18n.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/dev/tools/validate_ui_i18n.py b/dev/tools/validate_ui_i18n.py index 06207c42..af79549a 100644 --- a/dev/tools/validate_ui_i18n.py +++ b/dev/tools/validate_ui_i18n.py @@ -18,6 +18,11 @@ LINGUIST_PROJECT = REPO_ROOT / "src/REvoDesign/UI/liguist.pro" +def require(condition: bool, message: str) -> None: + if not condition: + raise RuntimeError(message) + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.parse_args(argv) @@ -36,26 +41,26 @@ def main(argv: list[str] | None = None) -> int: window, ui = load_runtime_ui(UI_PATH) try: # --- RuntimeUiProxy i18n contract --- - assert hasattr(ui, "trans"), "RuntimeUiProxy must expose a `trans` attribute for legacy i18n code" + require(hasattr(ui, "trans"), "RuntimeUiProxy must expose a `trans` attribute for legacy i18n code") trans = ui.trans - assert hasattr(trans, "load"), "translator must expose `load`" - assert hasattr(trans, "isEmpty"), "translator must expose `isEmpty`" - assert hasattr(trans, "translate"), "translator must expose `translate`" + require(hasattr(trans, "load"), "translator must expose `load`") + require(hasattr(trans, "isEmpty"), "translator must expose `isEmpty`") + require(hasattr(trans, "translate"), "translator must expose `translate`") # retranslateUi must work ui.retranslateUi(window) # `trans` must survive retranslateUi's internal refresh_bindings call - assert hasattr(ui, "trans"), "RuntimeUiProxy.trans must survive retranslateUi → refresh_bindings" + require(hasattr(ui, "trans"), "RuntimeUiProxy.trans must survive retranslateUi -> refresh_bindings") # --- Dynamic language action registry --- - assert hasattr(ui, "menuLanguage"), "menuLanguage menu must be available for dynamic actions" + require(hasattr(ui, "menuLanguage"), "menuLanguage menu must be available for dynamic actions") new_action = QtWidgets.QAction() new_action.setObjectName("actionTestLang") setattr(ui, "actionTestLang", new_action) ui.menuLanguage.addAction(new_action) - assert hasattr(ui, "actionTestLang"), "RuntimeUiProxy must accept dynamic action attributes" - assert getattr(ui, "actionTestLang") is new_action + require(hasattr(ui, "actionTestLang"), "RuntimeUiProxy must accept dynamic action attributes") + require(getattr(ui, "actionTestLang") is new_action, "RuntimeUiProxy must retain dynamic action attributes") # --- LanguageSwitch instantiation / exercise (smoke) --- # LanguageSwitch depends on a ConfigBus singleton that has `ui` already @@ -68,9 +73,9 @@ def main(argv: list[str] | None = None) -> int: config_bus.ui = ui lang_switch = LanguageSwitch(window) - assert hasattr(lang_switch, "trans"), "LanguageSwitch must own a `trans` reference" - assert lang_switch.trans is not None, "LanguageSwitch.trans must not be None" - assert hasattr(lang_switch.trans, "load"), "LanguageSwitch.trans must be a translator-like object" + require(hasattr(lang_switch, "trans"), "LanguageSwitch must own a `trans` reference") + require(lang_switch.trans is not None, "LanguageSwitch.trans must not be None") + require(hasattr(lang_switch.trans, "load"), "LanguageSwitch.trans must be a translator-like object") # English / source-language switch must not raise english = lang_switch.language_items[0] From 1ea1350b1e6e7f3f6af03573bce0e86072fbd337 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:19:42 +0800 Subject: [PATCH 13/63] fix(dev): parse UI XML with defusedxml --- dev/tools/generate_ui_typing.py | 3 ++- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/tools/generate_ui_typing.py b/dev/tools/generate_ui_typing.py index 7a42e4cb..064137d4 100644 --- a/dev/tools/generate_ui_typing.py +++ b/dev/tools/generate_ui_typing.py @@ -8,9 +8,10 @@ import argparse import sys -import xml.etree.ElementTree as ET from pathlib import Path +import defusedxml.ElementTree as ET + REPO_ROOT = Path(__file__).resolve().parents[2] UI_PATH = REPO_ROOT / "src/REvoDesign/UI/REvoDesign.ui" TYPES_PATH = REPO_ROOT / "src/REvoDesign/UI/types.py" diff --git a/pyproject.toml b/pyproject.toml index 2ca09bab..cd41e1ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ dependencies = [ "fonttools", "pooch", "xlrd", # for Legacy Excel files (Excel 2007 and earlier) + "defusedxml", ] [project.optional-dependencies] From f653f9093120095271851cd671d7291fa6bd88b0 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:22:58 +0800 Subject: [PATCH 14/63] fix(deepsource): log silent fallback errors --- src/REvoDesign/application/cluster_tab.py | 7 +++++-- src/REvoDesign/clusters/cluster_runner.py | 4 ++-- src/REvoDesign/clusters/methods/evo.py | 3 ++- src/REvoDesign/tools/measure_utils.py | 8 ++++---- src/REvoDesign/tools/package_manager.py | 15 ++++++++------- src/REvoDesign/tools/rfdiffusion_tools.py | 7 +++++-- 6 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/REvoDesign/application/cluster_tab.py b/src/REvoDesign/application/cluster_tab.py index 301d4662..3ce4900a 100644 --- a/src/REvoDesign/application/cluster_tab.py +++ b/src/REvoDesign/application/cluster_tab.py @@ -6,9 +6,12 @@ from __future__ import annotations +from REvoDesign.logger import ROOT_LOGGER from REvoDesign.Qt import QtCore, QtWidgets from REvoDesign.tools.customized_widgets import decide +logging = ROOT_LOGGER.getChild(__name__) + FALLBACK_CLUSTER_METHODS: tuple[str, ...] = ( "AgglomerativeCluster", "EvoCluster", @@ -77,8 +80,8 @@ def _configured_method_name(self) -> str: if self.bus is not None: try: return str(self.bus.get_value("ui.cluster.method.use", default_value="AgglomerativeCluster")) - except Exception: - pass + except Exception as exc: + logging.debug("Could not read configured cluster method; using current UI value: %s", exc) current = self.ui.comboBox_cluster_method.currentText() return current or "AgglomerativeCluster" diff --git a/src/REvoDesign/clusters/cluster_runner.py b/src/REvoDesign/clusters/cluster_runner.py index 50cac28a..38fe4240 100644 --- a/src/REvoDesign/clusters/cluster_runner.py +++ b/src/REvoDesign/clusters/cluster_runner.py @@ -76,8 +76,8 @@ def _representative_policy(clustering) -> str: if hasattr(clustering, "get_method_spec"): try: return clustering.get_method_spec().representative_policy - except Exception: - pass + except Exception as exc: + logging.debug("Could not read representative policy from clustering backend: %s", exc) return "Representative selection policy is not provided by this clustering backend." @staticmethod diff --git a/src/REvoDesign/clusters/methods/evo.py b/src/REvoDesign/clusters/methods/evo.py index 56192e0b..51797f64 100644 --- a/src/REvoDesign/clusters/methods/evo.py +++ b/src/REvoDesign/clusters/methods/evo.py @@ -186,7 +186,8 @@ def _build_pssm_distance_matrix(self, mutations_per_record: list[list]) -> np.nd continue try: values.append(float(df.loc[mutation.mut_res, col])) - except Exception: + except Exception as exc: + logging.debug("Skipping invalid PSSM score for %s at %s: %s", mutation.mut_res, col, exc) continue if values: scalar_scores[idx] = float(np.mean(values)) diff --git a/src/REvoDesign/tools/measure_utils.py b/src/REvoDesign/tools/measure_utils.py index c1875c52..7f6e0b20 100644 --- a/src/REvoDesign/tools/measure_utils.py +++ b/src/REvoDesign/tools/measure_utils.py @@ -319,8 +319,8 @@ def _atom_coord(a) -> tuple[float, float, float] | None: if hasattr(a, "x") and hasattr(a, "y") and hasattr(a, "z"): try: return (float(getattr(a, "x")), float(getattr(a, "y")), float(getattr(a, "z"))) - except Exception: - pass + except Exception as exc: + logging.debug("Could not parse atom coordinates from x/y/z attributes: %s", exc) return None @@ -329,8 +329,8 @@ def _atom_unique_id(a) -> int | None: if hasattr(a, "unique_id"): try: return int(getattr(a, "unique_id")) - except Exception: - pass + except Exception as exc: + logging.debug("Could not parse atom unique_id: %s", exc) return None diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index bacf7d15..3e16e4cf 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -2186,15 +2186,16 @@ def _terminate_process(process: subprocess.Popen[str]) -> None: if process.poll() is None: try: process.terminate() - except Exception: - pass + except Exception as exc: + logging.debug("Could not terminate process %s: %s", process, exc) try: process.wait(timeout=5) - except Exception: + except Exception as exc: + logging.debug("Process %s did not exit after terminate; killing it: %s", process, exc) try: process.kill() - except Exception: - pass + except Exception as kill_exc: + logging.debug("Could not kill process %s: %s", process, kill_exc) class AbortButtonOverlay(QtCore.QObject): @@ -2342,8 +2343,8 @@ def _handle_abort_request(self) -> None: self._aborted = True try: cmd.interrupt() - except Exception: - pass + except Exception as exc: + logging.debug("Could not interrupt PyMOL command loop during worker abort: %s", exc) self.worker_thread.interrupt() if self.worker_thread.isRunning(): self.worker_thread.quit() diff --git a/src/REvoDesign/tools/rfdiffusion_tools.py b/src/REvoDesign/tools/rfdiffusion_tools.py index cc562b09..b9241092 100644 --- a/src/REvoDesign/tools/rfdiffusion_tools.py +++ b/src/REvoDesign/tools/rfdiffusion_tools.py @@ -8,12 +8,15 @@ from REvoDesign.basic.abc_third_party_module import ThirdPartyModuleAbstract from REvoDesign.bootstrap.set_config import is_package_installed +from REvoDesign.logger import ROOT_LOGGER from REvoDesign.tools.utils import require_installed +logging = ROOT_LOGGER.getChild(__name__) + try: matplotlib.use("QtAgg") -except Exception: - pass # Fall back to default backend (e.g. in headless / CI environments) +except Exception as exc: + logging.debug("Falling back to default matplotlib backend: %s", exc) @require_installed From 8d56a16e52e9ec8306b897072d5e6bc25407030f Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:24:32 +0800 Subject: [PATCH 15/63] fix(deepsource): use secure random for Vina object names --- src/REvoDesign/shortcuts/tools/vina_tools.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/REvoDesign/shortcuts/tools/vina_tools.py b/src/REvoDesign/shortcuts/tools/vina_tools.py index b1466b4a..c6a5d5f5 100644 --- a/src/REvoDesign/shortcuts/tools/vina_tools.py +++ b/src/REvoDesign/shortcuts/tools/vina_tools.py @@ -9,8 +9,8 @@ """ +import secrets from dataclasses import dataclass -from random import randint from typing import Literal, overload import numpy as np @@ -27,6 +27,15 @@ from ...tools.cgo_utils import LineVertex, Point, PolyLines, Sphere logging = ROOT_LOGGER.getChild(__name__) + + +def _unique_pymol_name(prefix: str) -> str: + name = f"{prefix}_{secrets.randbelow(10001)}" + while name in cmd.get_names(): + name = f"{prefix}_{secrets.randbelow(10001)}" + return name + + ############################################################################## # GetBox Plugin.py -- Draws a box surrounding a selection and gets box information # This script is used to get box information for LeDock, Autodock Vina and AutoDock Vina. @@ -356,9 +365,7 @@ def from_selection( """ if not box_name: - boxName = "box_" + str(randint(0, 10000)) - while boxName in cmd.get_names(): - boxName = "box_" + str(randint(0, 10000)) + boxName = _unique_pymol_name("box") else: boxName = box_name @@ -691,10 +698,7 @@ def get_pca_box(selection="(sele)", new_box_name: str | None = None, extending=5 """ if not new_box_name: # Generate a unique box name if none is provided - boxName = "pca_box_" + str(randint(0, 10000)) - while boxName in cmd.get_names(): - # Ensure the generated box name is unique - boxName = "pca_box_" + str(randint(0, 10000)) + boxName = _unique_pymol_name("pca_box") else: # Use the provided box name boxName = new_box_name From 6ebdf2e9ebb2a74e3fca4d4f835e064528b9833a Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:25:58 +0800 Subject: [PATCH 16/63] fix(server): use secure jitter for digest scheduling --- server/pssm_gremlin_server/pssm_gremlin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/pssm_gremlin_server/pssm_gremlin.py b/server/pssm_gremlin_server/pssm_gremlin.py index 89c6cd40..05551a06 100644 --- a/server/pssm_gremlin_server/pssm_gremlin.py +++ b/server/pssm_gremlin_server/pssm_gremlin.py @@ -231,15 +231,14 @@ def from_env(cls) -> GremlinConfig: # notified so each user appears only once. _admin_digest_minutes = _env_int("ADMIN_NEW_USER_INFORM", 0) if _admin_digest_minutes > 0 and _env_str("ADMIN_NOTIFY_EMAIL", ""): + import secrets import threading _digest_lock = os.path.join(_env_str("SERVER_DIR", os.getcwd()), ".admin_digest.lock") def _digest_loop() -> None: - import random - while True: - _jitter = random.uniform(-5, 5) # spread worker wake-ups + _jitter = secrets.randbelow(10001) / 1000 - 5 # spread worker wake-ups time.sleep(_admin_digest_minutes * 60 + _jitter) try: # ponytail: file lock so only one worker sends the digest From 20b7792ab5e593c5e8c2c40310d6eeebd7efa2f8 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:36:22 +0800 Subject: [PATCH 17/63] fix(dev): parse UI typing XML with Qt stream reader --- dev/tools/generate_ui_typing.py | 47 ++++++++++++++++++++++----------- pyproject.toml | 1 - 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/dev/tools/generate_ui_typing.py b/dev/tools/generate_ui_typing.py index 064137d4..b5b3799a 100644 --- a/dev/tools/generate_ui_typing.py +++ b/dev/tools/generate_ui_typing.py @@ -10,7 +10,7 @@ import sys from pathlib import Path -import defusedxml.ElementTree as ET +from REvoDesign.Qt import QtCore REPO_ROOT = Path(__file__).resolve().parents[2] UI_PATH = REPO_ROOT / "src/REvoDesign/UI/REvoDesign.ui" @@ -77,14 +77,34 @@ def __init__(self, name: str, type_name: str, warning: str | None = None) -> Non self.warning = warning -def _extract_custom_widget_map(root: ET.Element) -> dict[str, str]: +def _read_ui_elements(ui_path: Path) -> tuple[dict[str, str], list[tuple[str, str, str]]]: custom_types: dict[str, str] = {} - for custom_widget in root.findall("./customwidgets/customwidget"): - class_name = custom_widget.findtext("class") - if not class_name: - continue - custom_types[class_name] = FALLBACK_WIDGET_TYPE - return custom_types + elements: list[tuple[str, str, str]] = [] + reader = QtCore.QXmlStreamReader(ui_path.read_text(encoding="utf-8")) + in_custom_widget = False + + while not reader.atEnd(): + token = reader.readNext() + if token == QtCore.QXmlStreamReader.StartElement: + tag = reader.name() + if tag == "customwidget": + in_custom_widget = True + continue + if in_custom_widget and tag == "class": + class_name = reader.readElementText().strip() + if class_name: + custom_types[class_name] = FALLBACK_WIDGET_TYPE + continue + if tag in {"widget", "layout", "action", "buttongroup"}: + attributes = reader.attributes() + elements.append((tag, attributes.value("name"), attributes.value("class"))) + elif token == QtCore.QXmlStreamReader.EndElement and reader.name() == "customwidget": + in_custom_widget = False + + if reader.hasError(): + raise ValueError(f"Failed to parse {ui_path}: {reader.errorString()}") + + return custom_types, elements def _resolve_type_name(tag: str, class_name: str, custom_types: dict[str, str]) -> tuple[str, str | None]: @@ -104,18 +124,13 @@ def _resolve_type_name(tag: str, class_name: str, custom_types: dict[str, str]) def collect_ui_bindings(ui_path: Path = UI_PATH) -> list[UiBinding]: - root = ET.parse(ui_path).getroot() - custom_types = _extract_custom_widget_map(root) + custom_types, elements = _read_ui_elements(ui_path) bindings: dict[str, UiBinding] = {} - for element in root.iter(): - if element.tag not in {"widget", "layout", "action", "buttongroup"}: - continue - name = element.attrib.get("name", "") - class_name = element.attrib.get("class", "") + for tag, name, class_name in elements: if not name or not name.isidentifier(): continue - type_name, warning = _resolve_type_name(element.tag, class_name, custom_types) + type_name, warning = _resolve_type_name(tag, class_name, custom_types) bindings.setdefault(name, UiBinding(name=name, type_name=type_name, warning=warning)) return [bindings[name] for name in sorted(bindings)] diff --git a/pyproject.toml b/pyproject.toml index cd41e1ef..2ca09bab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,6 @@ dependencies = [ "fonttools", "pooch", "xlrd", # for Legacy Excel files (Excel 2007 and earlier) - "defusedxml", ] [project.optional-dependencies] From 45c8a22b9b9b600d1e2b3c13c289bae40a8ece4f Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:36:36 +0800 Subject: [PATCH 18/63] fix(security): clean remaining production audit findings --- server/pssm_gremlin_server/pssm_gremlin.py | 3 ++- server/scripts/GREMLIN_TFv1.py | 3 ++- src/REvoDesign/clients/QtSocketConnector.py | 3 ++- src/REvoDesign/clusters/cluster_sequence.py | 3 ++- src/REvoDesign/common/multi_mutant_designer.py | 6 +++--- src/REvoDesign/phylogenetics/revo_designer.py | 3 ++- src/REvoDesign/shortcuts/function_utils.py | 18 ++++++++++++++---- src/REvoDesign/shortcuts/tools/evolution.py | 3 ++- .../shortcuts/tools/rfdiffusion_tasks.py | 3 ++- src/REvoDesign/tools/package_manager.py | 5 +++-- src/REvoDesign/tools/safe_pickle.py | 3 ++- 11 files changed, 36 insertions(+), 17 deletions(-) diff --git a/server/pssm_gremlin_server/pssm_gremlin.py b/server/pssm_gremlin_server/pssm_gremlin.py index 05551a06..d35c7dfd 100644 --- a/server/pssm_gremlin_server/pssm_gremlin.py +++ b/server/pssm_gremlin_server/pssm_gremlin.py @@ -958,4 +958,5 @@ def _on_stage_change(stage: str) -> None: importlib.import_module("pssm_gremlin_server.routes") if __name__ == "__main__": - app.run(host="0.0.0.0", port=CONFIG.port) # nosec B104: containerized server, binding to all interfaces by design + # Containerized server binds to all interfaces by design. + app.run(host="0.0.0.0", port=CONFIG.port) # nosec B104 diff --git a/server/scripts/GREMLIN_TFv1.py b/server/scripts/GREMLIN_TFv1.py index 1b220eec..9d53b750 100644 --- a/server/scripts/GREMLIN_TFv1.py +++ b/server/scripts/GREMLIN_TFv1.py @@ -12,7 +12,8 @@ import os import pathlib -import pickle # nosec B403: MRF model serialization, trusted data +# MRF model serialization uses trusted data. +import pickle # nosec B403 import sys import matplotlib diff --git a/src/REvoDesign/clients/QtSocketConnector.py b/src/REvoDesign/clients/QtSocketConnector.py index da07b03e..0259fc04 100644 --- a/src/REvoDesign/clients/QtSocketConnector.py +++ b/src/REvoDesign/clients/QtSocketConnector.py @@ -15,7 +15,8 @@ import base64 import copy -import pickle # nosec B403: inter-process Qt socket serialization, trusted local context +# Inter-process Qt socket serialization uses a trusted local context. +import pickle # nosec B403 import socket import time import traceback diff --git a/src/REvoDesign/clusters/cluster_sequence.py b/src/REvoDesign/clusters/cluster_sequence.py index bfd242f9..5859c48e 100644 --- a/src/REvoDesign/clusters/cluster_sequence.py +++ b/src/REvoDesign/clusters/cluster_sequence.py @@ -246,7 +246,8 @@ def _calculate_pairwise_scores(self, progressbar) -> np.ndarray: handle = open(self.fastafile) self.records = list(SeqIO.parse(handle, "fasta")) if self.shuffle_variant: - rng = random.Random(self.random_seed) + # Deterministic clustering order is controlled by the UI random-seed setting. + rng = random.Random(self.random_seed) # nosec B311 self.records = rng.sample(self.records, len(self.records)) nm_seqs = len(self.records) diff --git a/src/REvoDesign/common/multi_mutant_designer.py b/src/REvoDesign/common/multi_mutant_designer.py index 0f376d0b..a74a56e5 100644 --- a/src/REvoDesign/common/multi_mutant_designer.py +++ b/src/REvoDesign/common/multi_mutant_designer.py @@ -10,7 +10,7 @@ import copy import os -import random +import secrets import warnings from itertools import pairwise @@ -461,8 +461,8 @@ def _select_random_mutant(self) -> tuple[str, tuple[str, Mutant]]: This method randomly selects a mutant for the design process. """ - branch = random.choice(self.design_pool_tree_copy.all_mutant_branch_ids) - mut = random.choice(list(self.design_pool_tree_copy.get_a_branch(branch).items())) + branch = secrets.choice(self.design_pool_tree_copy.all_mutant_branch_ids) + mut = secrets.choice(list(self.design_pool_tree_copy.get_a_branch(branch).items())) return branch, mut def _is_compatible_mutant( diff --git a/src/REvoDesign/phylogenetics/revo_designer.py b/src/REvoDesign/phylogenetics/revo_designer.py index 39371b03..5221ee8f 100644 --- a/src/REvoDesign/phylogenetics/revo_designer.py +++ b/src/REvoDesign/phylogenetics/revo_designer.py @@ -64,7 +64,8 @@ def __init__(self, input_profile): self.randomized_sample_num = 10 self.mutate_runner: MutateRunnerAbstract = None # type: ignore - self.pwd = "." # nosec B105: present working directory, not a password + # Present working directory, not a password. + self.pwd = "." # nosec B105 self.sequence = "" self.design_case = "default" diff --git a/src/REvoDesign/shortcuts/function_utils.py b/src/REvoDesign/shortcuts/function_utils.py index a5c4aaf4..9ffca794 100644 --- a/src/REvoDesign/shortcuts/function_utils.py +++ b/src/REvoDesign/shortcuts/function_utils.py @@ -8,7 +8,9 @@ """ import os -import subprocess +import shutil +# Used for launching a user-requested PyMOL preview with an argv list. +import subprocess # nosec B404 from typing import Literal from pymol import cmd @@ -19,6 +21,13 @@ logging = ROOT_LOGGER.getChild(__name__) +def _resolve_pymol_executable() -> str: + pymol_executable = shutil.which("pymol") + if pymol_executable is None: + raise RuntimeError("Unable to launch PyMOL preview: 'pymol' executable was not found on PATH.") + return pymol_executable + + def visualize_conformer_sdf(sdf_file_path: str, show_conformer: Literal["New Window", "Current Window"]): """ Visualize a ligand conformer file (SDF) in a new PyMOL window. @@ -54,10 +63,11 @@ def visualize_conformer_sdf(sdf_file_path: str, show_conformer: Literal["New Win pmlh.write("bg_color white\n") # Explicitly call a new PyMOL instance in the background - pymol_command = ["pymol", "-xi", pml_file_path] - subprocess.Popen(pymol_command) + pymol_command = [_resolve_pymol_executable(), "-xi", pml_file_path] + # Launches a user-requested PyMOL preview with an argv list and a resolved executable. + subprocess.Popen(pymol_command, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # nosec B603 - print(f"PyMOL launched in the background with {sdf_file_path}.") + logging.info("PyMOL launched in the background with %s.", sdf_file_path) def smiles_conformer_batch(smi: dict[str, str], num_conformer: int, save_dir: str, n_jobs: int = 1): diff --git a/src/REvoDesign/shortcuts/tools/evolution.py b/src/REvoDesign/shortcuts/tools/evolution.py index eb61754e..10ee3c2f 100644 --- a/src/REvoDesign/shortcuts/tools/evolution.py +++ b/src/REvoDesign/shortcuts/tools/evolution.py @@ -8,7 +8,8 @@ """ import gc -import pickle # nosec B403: MRF model serialization, trusted data +# MRF model serialization uses trusted data. +import pickle # nosec B403 from REvoDesign.basic.abc_third_party_module import ThirdPartyModuleAbstract, TorchModuleAbstract from REvoDesign.bootstrap.set_config import is_package_installed diff --git a/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py b/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py index b1e8a26e..48262baa 100644 --- a/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py +++ b/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py @@ -5,7 +5,8 @@ import glob import os -import pickle # nosec B403: traceback serialization, trusted context +# Traceback serialization uses a trusted context. +import pickle # nosec B403 import random import re import shutil diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index 3e16e4cf..f1170972 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -21,7 +21,8 @@ import re import shutil import socket -import subprocess +# Used for package-management commands with argv lists and registered process lifecycle handling. +import subprocess # nosec B404 import sys import threading import time @@ -573,7 +574,7 @@ def stream_reader(pipe: io.IOBase, collector: list[str], label: str): collector.append(line) pipe.close() - process = subprocess.Popen( + process = subprocess.Popen( # nosec B603 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/src/REvoDesign/tools/safe_pickle.py b/src/REvoDesign/tools/safe_pickle.py index ee189f98..b615cc52 100644 --- a/src/REvoDesign/tools/safe_pickle.py +++ b/src/REvoDesign/tools/safe_pickle.py @@ -4,7 +4,8 @@ import importlib import io -import pickle # nosec B403 - this module provides a restricted unpickler wrapper +# This module provides a restricted unpickler wrapper. +import pickle # nosec B403 from collections import OrderedDict, defaultdict, deque from collections.abc import Iterable from typing import BinaryIO From 5b61c1f78e1f74c8febab6c6a0d38b53d96051fb Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:58:54 +0800 Subject: [PATCH 19/63] fix(deepsource): clear higher-risk flake8 findings --- dev/tools/generate_ui_typing.py | 2 ++ dev/tools/validate_ui_i18n.py | 6 +++--- server/pssm_gremlin_server/pssm_gremlin.py | 2 +- server/scripts/GREMLIN_TFv1.py | 2 +- server/scripts/fasta_lower_char_rm.py | 2 +- server/scripts/make_paired_MSA_simple.py | 4 ++-- src/REvoDesign/Qt/ui_runtime_loader.py | 2 +- src/REvoDesign/application/font/font_manager.py | 2 +- src/REvoDesign/application/i18n/language_settings.py | 4 ++-- src/REvoDesign/citations/citation_manager.py | 2 +- src/REvoDesign/common/multi_mutant_designer.py | 4 ++-- src/REvoDesign/driver/ui_driver.py | 4 ++-- src/REvoDesign/editor/monaco/server.py | 5 +++-- src/REvoDesign/magician/designers/colabdesign.py | 7 ++----- src/REvoDesign/phylogenetics/revo_designer.py | 2 +- src/REvoDesign/shortcuts/tools/represents.py | 2 +- src/REvoDesign/shortcuts/tools/vina_tools.py | 6 +++--- src/REvoDesign/sidechain/sidechain_solver.py | 2 +- src/REvoDesign/tools/customized_widgets.py | 6 +++--- src/REvoDesign/tools/measure_utils.py | 6 +++--- src/REvoDesign/tools/package_manager.py | 2 +- src/REvoDesign/tools/pymol_utils.py | 5 ++++- src/REvoDesign/tools/rfdiffusion_tools.py | 2 +- 23 files changed, 42 insertions(+), 39 deletions(-) diff --git a/dev/tools/generate_ui_typing.py b/dev/tools/generate_ui_typing.py index b5b3799a..aa4267c4 100644 --- a/dev/tools/generate_ui_typing.py +++ b/dev/tools/generate_ui_typing.py @@ -71,6 +71,8 @@ class UiBinding: + __slots__ = ("name", "type_name", "warning") + def __init__(self, name: str, type_name: str, warning: str | None = None) -> None: self.name = name self.type_name = type_name diff --git a/dev/tools/validate_ui_i18n.py b/dev/tools/validate_ui_i18n.py index af79549a..82926429 100644 --- a/dev/tools/validate_ui_i18n.py +++ b/dev/tools/validate_ui_i18n.py @@ -10,7 +10,7 @@ import sys from pathlib import Path -from REvoDesign.Qt import QtCore, QtWidgets +from REvoDesign.Qt import QtWidgets from REvoDesign.Qt.ui_runtime_loader import load_runtime_ui REPO_ROOT = Path(__file__).resolve().parents[2] @@ -57,10 +57,10 @@ def main(argv: list[str] | None = None) -> int: require(hasattr(ui, "menuLanguage"), "menuLanguage menu must be available for dynamic actions") new_action = QtWidgets.QAction() new_action.setObjectName("actionTestLang") - setattr(ui, "actionTestLang", new_action) + ui.actionTestLang = new_action ui.menuLanguage.addAction(new_action) require(hasattr(ui, "actionTestLang"), "RuntimeUiProxy must accept dynamic action attributes") - require(getattr(ui, "actionTestLang") is new_action, "RuntimeUiProxy must retain dynamic action attributes") + require(ui.actionTestLang is new_action, "RuntimeUiProxy must retain dynamic action attributes") # --- LanguageSwitch instantiation / exercise (smoke) --- # LanguageSwitch depends on a ConfigBus singleton that has `ui` already diff --git a/server/pssm_gremlin_server/pssm_gremlin.py b/server/pssm_gremlin_server/pssm_gremlin.py index d35c7dfd..5ccd2c87 100644 --- a/server/pssm_gremlin_server/pssm_gremlin.py +++ b/server/pssm_gremlin_server/pssm_gremlin.py @@ -541,7 +541,7 @@ def _build_running_trace(task: dict[str, Any]) -> str: try: _ROOT_MOUNT_DIRECTORY = f"/home/{os.getlogin()}" -except BaseException: +except OSError: _ROOT_MOUNT_DIRECTORY = os.path.abspath(tempfile.gettempdir()) os.makedirs(_ROOT_MOUNT_DIRECTORY, exist_ok=True) diff --git a/server/scripts/GREMLIN_TFv1.py b/server/scripts/GREMLIN_TFv1.py index 9d53b750..3238ef55 100644 --- a/server/scripts/GREMLIN_TFv1.py +++ b/server/scripts/GREMLIN_TFv1.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-only -#!/usr/bin/env python +# Original script marker: /usr/bin/env python # # GREMLIN_TF # diff --git a/server/scripts/fasta_lower_char_rm.py b/server/scripts/fasta_lower_char_rm.py index c60bb848..2750ab0c 100644 --- a/server/scripts/fasta_lower_char_rm.py +++ b/server/scripts/fasta_lower_char_rm.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-only -#! python +# Original script marker: python # created by Yao Yin Ying # for hhblits a3m alignment file treatment diff --git a/server/scripts/make_paired_MSA_simple.py b/server/scripts/make_paired_MSA_simple.py index 6292a5a6..b5bbd72d 100644 --- a/server/scripts/make_paired_MSA_simple.py +++ b/server/scripts/make_paired_MSA_simple.py @@ -51,7 +51,7 @@ def read_a3m(fn): try: idx = line.index("OX") is_ignore = False - except BaseException: + except ValueError: is_ignore = True continue TaxID = line[idx:].split()[0].split("=")[-1] @@ -76,7 +76,7 @@ def read_a3m(fn): continue # Get the best sequence only score_s = list() - for seqID, seq in tmp[TaxID]: + for _seqID, seq in tmp[TaxID]: seq_in_num = seq2number(seq) score = calc_seqID(query_in_num, seq_in_num) score_s.append(score) diff --git a/src/REvoDesign/Qt/ui_runtime_loader.py b/src/REvoDesign/Qt/ui_runtime_loader.py index 367f6a54..537381ba 100644 --- a/src/REvoDesign/Qt/ui_runtime_loader.py +++ b/src/REvoDesign/Qt/ui_runtime_loader.py @@ -29,7 +29,7 @@ def _load_pyqt_uic_module() -> Any: def _language_change_event_type(): event_type_owner = getattr(QtCore.QEvent, "Type", QtCore.QEvent) - return getattr(event_type_owner, "LanguageChange") + return event_type_owner.LanguageChange def is_language_change_event(event: QtCore.QEvent) -> bool: diff --git a/src/REvoDesign/application/font/font_manager.py b/src/REvoDesign/application/font/font_manager.py index aca89ed9..b9b7e393 100644 --- a/src/REvoDesign/application/font/font_manager.py +++ b/src/REvoDesign/application/font/font_manager.py @@ -155,7 +155,7 @@ def set_font(font: QtGui.QFont | str | None = None): # Iterate through and set font for all open windows if hasattr(ui, "open_windows"): - open_windows: list[QtWidgets.QWidget] = getattr(ui, "open_windows") + open_windows: list[QtWidgets.QWidget] = ui.open_windows for window in open_windows: window.setFont(font) diff --git a/src/REvoDesign/application/i18n/language_settings.py b/src/REvoDesign/application/i18n/language_settings.py index 0c40d624..654e7ccf 100644 --- a/src/REvoDesign/application/i18n/language_settings.py +++ b/src/REvoDesign/application/i18n/language_settings.py @@ -224,7 +224,7 @@ def _ensure_translator(self) -> QtCore.QTranslator: if app is not None: for child in app.children(): if isinstance(child, QtCore.QTranslator): - setattr(self.bus.ui, "trans", child) + self.bus.ui.trans = child self._translator_installed = True return child @@ -238,7 +238,7 @@ def _ensure_translator(self) -> QtCore.QTranslator: return existing translator = QtCore.QTranslator(self.window) - setattr(self.bus.ui, "trans", translator) + self.bus.ui.trans = translator return translator def _bind_to_action(self, language: LanguageItem): diff --git a/src/REvoDesign/citations/citation_manager.py b/src/REvoDesign/citations/citation_manager.py index c6c6f62a..1682f3d3 100644 --- a/src/REvoDesign/citations/citation_manager.py +++ b/src/REvoDesign/citations/citation_manager.py @@ -200,7 +200,7 @@ def get_citable_class(cls, func: Callable) -> type["CitableModuleAbstract"]: class AnonymousCitableModule(cls): __name__ = f"{func.__name__}_CitableModule" - __bibtex__ = getattr(func, "__bibtex__") + __bibtex__ = func.__bibtex__ AnonymousCitableModule.__doc__ = f"""Citable module for function { func.__name__}, auto generated by CitableModuleAbstract.""" # type: ignore[assignment] diff --git a/src/REvoDesign/common/multi_mutant_designer.py b/src/REvoDesign/common/multi_mutant_designer.py index a74a56e5..efb7b8c4 100644 --- a/src/REvoDesign/common/multi_mutant_designer.py +++ b/src/REvoDesign/common/multi_mutant_designer.py @@ -133,7 +133,7 @@ def refresh_design_color(self): mut_obj.mutant_score for mut_obj in self.all_design_multi_design_mutant_object if mut_obj.mutant_score ] - for (i_obj, obj), (j_des, des) in zip( + for (i_obj, obj), (_j_des, des) in zip( enumerate(self.design_case_variant_objects), enumerate(self.all_design_multi_design_mutant_object), ): @@ -226,7 +226,7 @@ def _auto_pick_tryout(self, tryout=30): This method tries to automatically pick mutants for the design process. """ - for i in range(tryout): + for _i in range(tryout): try: branch, (mutant_id, mutant_obj) = self._select_random_mutant() except IndexError: diff --git a/src/REvoDesign/driver/ui_driver.py b/src/REvoDesign/driver/ui_driver.py index 60ac1bb7..ed5b238d 100644 --- a/src/REvoDesign/driver/ui_driver.py +++ b/src/REvoDesign/driver/ui_driver.py @@ -63,7 +63,7 @@ def reset_instance(cls): for k, s in attr_dict.items(): if hasattr(s, "controller"): - controller = getattr(s, "controller") + controller = s.controller try: if issubclass(s.controller.__class__, SingletonAbstract): print(f"Resetting {k}: {controller.__class__.__name__}", end=" ") @@ -391,7 +391,7 @@ def singleton_init(self, ui=None): def initialize_widget_with_group(self): # Initializes UI widgets with their corresponding configuration settings. - for i, gr in enumerate(GroupRegistryCollection): + for gr in GroupRegistryCollection: group_values = [] widget = self.get_widget_from_id(widget_id=gr.cfg_item) if isinstance(widget, str): diff --git a/src/REvoDesign/editor/monaco/server.py b/src/REvoDesign/editor/monaco/server.py index ebb948dc..2a747283 100644 --- a/src/REvoDesign/editor/monaco/server.py +++ b/src/REvoDesign/editor/monaco/server.py @@ -12,6 +12,7 @@ from contextlib import asynccontextmanager from html import escape from pathlib import Path +from typing import Annotated import uvicorn from fastapi import FastAPI, HTTPException, Query, Request @@ -225,7 +226,7 @@ async def serve_editor(file_path: str, token: str = None, request: Request = Non @app.get("/load_file", response_class=JSONResponse) -async def load_file(file_path: str, token: str = Query(None), request: Request = None): +async def load_file(file_path: str, token: Annotated[str | None, Query()] = None, request: Request = None): verify_token(token, request) # Validate file existence @@ -253,7 +254,7 @@ class SaveFileRequest(BaseModel): @app.post("/save_file", response_class=JSONResponse) -async def save_file(data: SaveFileRequest, token: str = Query(None), request: Request = None): +async def save_file(data: SaveFileRequest, token: Annotated[str | None, Query()] = None, request: Request = None): verify_token(token, request) # Validate the file path diff --git a/src/REvoDesign/magician/designers/colabdesign.py b/src/REvoDesign/magician/designers/colabdesign.py index 89cae03e..74ad08ee 100644 --- a/src/REvoDesign/magician/designers/colabdesign.py +++ b/src/REvoDesign/magician/designers/colabdesign.py @@ -59,11 +59,8 @@ def initialize(self, *args, **kwargs): self.mpnn_model = mk_mpnn_model() if not os.path.exists(self.pdb_filename): raise FileNotFoundError(f"Input pdb file does not exist: {self.pdb_filename}") - self.mpnn_model.prep_inputs( - pdb_filename=self.pdb_filename, - *args, - **kwargs, - ) + prep_kwargs = {**kwargs, "pdb_filename": self.pdb_filename} + self.mpnn_model.prep_inputs(*args, **prep_kwargs) self.initialized = True self.cite() diff --git a/src/REvoDesign/phylogenetics/revo_designer.py b/src/REvoDesign/phylogenetics/revo_designer.py index 5221ee8f..fdfd6c18 100644 --- a/src/REvoDesign/phylogenetics/revo_designer.py +++ b/src/REvoDesign/phylogenetics/revo_designer.py @@ -183,7 +183,7 @@ def plot_custom_indices_segments( "mutations": {}, } mutations = [] - for idx, resid in enumerate(custom_indices): + for resid in custom_indices: # fetch wt aa from untruncated sequence wt_aa = sequence[resid - 1] profile_scores = df_trunc.loc[:, resid] diff --git a/src/REvoDesign/shortcuts/tools/represents.py b/src/REvoDesign/shortcuts/tools/represents.py index 867013f9..171459b3 100644 --- a/src/REvoDesign/shortcuts/tools/represents.py +++ b/src/REvoDesign/shortcuts/tools/represents.py @@ -855,5 +855,5 @@ def _load_to_one_chain(chain_id: str, state: int = 0): _load_to_one_chain(chain_id) -setattr(_load_b_factors, "__bibtex__", load_b_factors_citation) +_load_b_factors.__bibtex__ = load_b_factors_citation load_b_factors = get_cited(_load_b_factors) diff --git a/src/REvoDesign/shortcuts/tools/vina_tools.py b/src/REvoDesign/shortcuts/tools/vina_tools.py index c6a5d5f5..135e3efc 100644 --- a/src/REvoDesign/shortcuts/tools/vina_tools.py +++ b/src/REvoDesign/shortcuts/tools/vina_tools.py @@ -172,7 +172,7 @@ def rebuild(self): self.always_left_corner = bool(self.always_left_corner) self._data = [] - for (idxa, axis), (idxc, colorname) in zip(enumerate("xyz"), enumerate("rgb")): + for axis, colorname in zip("xyz", "rgb"): p2_kwargs = {i: 0.0 for i in "xyz" if i != axis} self._data.extend( Cylinder( @@ -661,7 +661,7 @@ def plot_pca_box(orig_vertices, new_box_name: str = "pca_box"): goc = GOC([]) # Create a PolyLines object for each face - for i, face_indices in enumerate(faces): + for face_indices in faces: face_vertices = [orig_vertices[idx] for idx in face_indices] goc.objects.append( PolyLines(width=2.0, color="cyan", points=LineVertex.from_points(face_vertices), line_type="LINE_LOOP") @@ -676,7 +676,7 @@ def plot_pca_box(orig_vertices, new_box_name: str = "pca_box"): ).load_as(f"pca_axes_{_idx}") # Plot each vertex as a sphere - for i, v in enumerate(orig_vertices): + for v in orig_vertices: goc.objects.append(Sphere(Point(*v), radius=1)) # Rebuild the GOC object and load the new box diff --git a/src/REvoDesign/sidechain/sidechain_solver.py b/src/REvoDesign/sidechain/sidechain_solver.py index 9ae655ab..5ed7e221 100644 --- a/src/REvoDesign/sidechain/sidechain_solver.py +++ b/src/REvoDesign/sidechain/sidechain_solver.py @@ -61,7 +61,7 @@ class SidechainSolverConfig: def reconfigured(self, new_config: "SidechainSolverConfig") -> bool: reconfigured = False if new_config != self: - for (k1, v1), (k2, v2) in zip(asdict(self).items(), asdict(new_config).items()): + for (k1, v1), (_k2, v2) in zip(asdict(self).items(), asdict(new_config).items()): if v1 == v2: continue print_diff(k1, {"Before": v1, "After": v2}) diff --git a/src/REvoDesign/tools/customized_widgets.py b/src/REvoDesign/tools/customized_widgets.py index 24c64213..08692fb4 100644 --- a/src/REvoDesign/tools/customized_widgets.py +++ b/src/REvoDesign/tools/customized_widgets.py @@ -156,7 +156,7 @@ def check_repeat(self): the_windows = [ w for w in bus.ui.open_windows - if hasattr(w, "objectName") and getattr(w, "objectName")() == self.objectName() + if hasattr(w, "objectName") and w.objectName() == self.objectName() ] if any(the_windows): # Raise the existing window to the front @@ -180,7 +180,7 @@ def attach(self): # Ensure the open_windows list exists and add this widget to it if not hasattr(bus.ui, "open_windows"): logging.debug(f"Creating open_windows list under ui for {self.objectName()}") - setattr(bus.ui, "open_windows", []) + bus.ui.open_windows = [] bus.ui.open_windows.append(self) logging.debug(f"Window {self.objectName()} attached.") @@ -2667,7 +2667,7 @@ def _remove_row(self, row_layout): Args: row_layout (QHBoxLayout): The row layout to be removed. """ - for i, (layout, key_edit, val_edit) in enumerate(self.row_widgets): + for i, (layout, _key_edit, _val_edit) in enumerate(self.row_widgets): if layout == row_layout: # Remove all widgets in the row for j in reversed(range(layout.count())): diff --git a/src/REvoDesign/tools/measure_utils.py b/src/REvoDesign/tools/measure_utils.py index 7f6e0b20..4f16dca8 100644 --- a/src/REvoDesign/tools/measure_utils.py +++ b/src/REvoDesign/tools/measure_utils.py @@ -313,12 +313,12 @@ def _get_model_atoms(cmd_module, obj: str, state: int = -1): def _atom_coord(a) -> tuple[float, float, float] | None: """Extract ``(x, y, z)`` from a PyMOL atom object.""" if hasattr(a, "coord"): - c = getattr(a, "coord") + c = a.coord if isinstance(c, (list, tuple)) and len(c) >= 3: return (float(c[0]), float(c[1]), float(c[2])) if hasattr(a, "x") and hasattr(a, "y") and hasattr(a, "z"): try: - return (float(getattr(a, "x")), float(getattr(a, "y")), float(getattr(a, "z"))) + return (float(a.x), float(a.y), float(a.z)) except Exception as exc: logging.debug("Could not parse atom coordinates from x/y/z attributes: %s", exc) return None @@ -328,7 +328,7 @@ def _atom_unique_id(a) -> int | None: """Extract ``unique_id`` from a PyMOL atom (only uses the explicit attribute).""" if hasattr(a, "unique_id"): try: - return int(getattr(a, "unique_id")) + return int(a.unique_id) except Exception as exc: logging.debug("Could not parse atom unique_id: %s", exc) return None diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index f1170972..2f5778d0 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -2650,7 +2650,7 @@ def execute_on_main_thread(func: Callable[..., GuiResult], *args, **kwargs) -> G def callback(): try: result["value"] = func(*args, **kwargs) - except BaseException as exc: # pragma: no cover - defensive against unexpected UI failures + except BaseException as exc: # noqa: B036 # pragma: no cover - re-raised across the GUI-thread handoff error["error"] = exc finally: semaphore.release() diff --git a/src/REvoDesign/tools/pymol_utils.py b/src/REvoDesign/tools/pymol_utils.py index 8c5bfc8c..ffbefb18 100644 --- a/src/REvoDesign/tools/pymol_utils.py +++ b/src/REvoDesign/tools/pymol_utils.py @@ -397,7 +397,7 @@ def make_temperal_input_pdb( resn="", selection="", save_as_format="pdb", - wd=os.getcwd(), + wd=None, reload=True, ): """ @@ -420,6 +420,9 @@ def make_temperal_input_pdb( Returns: - str: Path to the generated temporary input PDB file """ + if wd is None: + wd = os.getcwd() + os.makedirs(wd, exist_ok=True) input_file = os.path.join( wd, diff --git a/src/REvoDesign/tools/rfdiffusion_tools.py b/src/REvoDesign/tools/rfdiffusion_tools.py index b9241092..b1a43fc1 100644 --- a/src/REvoDesign/tools/rfdiffusion_tools.py +++ b/src/REvoDesign/tools/rfdiffusion_tools.py @@ -250,7 +250,7 @@ def plot_potential_field(self, grid_size=200, margin=10, save_to: str = "default default_size = 90 # Overlay ligand atoms - for idx, (coord, element) in enumerate(zip(self.ligand_coords, self.ligand_elements)): + for coord, element in zip(self.ligand_coords, self.ligand_elements): color = color_map.get(element, default_color) size = size_map.get(element, default_size) plt.scatter(coord[0], coord[1], c=color, s=size, edgecolors="black", linewidth=1.5, zorder=3) From 4135caddbc9eb7976fd6241a5b6b9dd7718f3c4a Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:08:29 +0800 Subject: [PATCH 20/63] fix(deepsource): add warning stacklevels --- src/REvoDesign/REvoDesign.py | 39 ++++++++++++------- src/REvoDesign/citations/citation_manager.py | 4 +- src/REvoDesign/clients/QtSocketConnector.py | 38 +++++++++++------- .../common/multi_mutant_designer.py | 9 +++-- src/REvoDesign/common/mutant_visualise.py | 3 +- src/REvoDesign/editor/monaco/server.py | 2 +- .../magician/designers/colabdesign.py | 4 +- src/REvoDesign/phylogenetics/evo_mutator.py | 12 +++--- src/REvoDesign/phylogenetics/gremlin_tools.py | 9 +++-- src/REvoDesign/shortcuts/tools/designs.py | 2 +- src/REvoDesign/shortcuts/tools/represents.py | 3 +- .../shortcuts/tools/rfdiffusion_tasks.py | 9 +++-- src/REvoDesign/tools/customized_widgets.py | 6 +-- src/REvoDesign/tools/mutant_tools.py | 15 ++++--- src/REvoDesign/tools/package_manager.py | 3 +- src/REvoDesign/tools/pymol_utils.py | 10 +++-- src/REvoDesign/tools/rosetta_utils.py | 9 +++-- src/REvoDesign/tools/system_tools.py | 3 +- 18 files changed, 110 insertions(+), 70 deletions(-) diff --git a/src/REvoDesign/REvoDesign.py b/src/REvoDesign/REvoDesign.py index 324ae5a8..172bd2f2 100644 --- a/src/REvoDesign/REvoDesign.py +++ b/src/REvoDesign/REvoDesign.py @@ -104,7 +104,8 @@ def __init__( warnings.warn( issues.DisabledFunctionWarning( "Teamwork is disabled because the active PyMOL Qt backend does not provide QtWebSockets." - ) + ), + stacklevel=2, ) self.teamwork_enabled = False @@ -210,7 +211,8 @@ def reinitialize(self, delete: bool = False): warnings.warn( issues.ConflictWarning( "Reinitialized with default configuration. " "Restart REvoDesign to take effort." - ) + ), + stacklevel=2, ) def __del__(self): @@ -661,7 +663,9 @@ def reload_molecule_info(self): cmd.reinitialize() cmd.load(self.temperal_session) else: - warnings.warn(issues.EmptySessionWarning("Current session is empty! Please load one PDB/PSE/PZE!")) + warnings.warn( + issues.EmptySessionWarning("Current session is empty! Please load one PDB/PSE/PZE!"), stacklevel=2 + ) new_session_file = self.file_dialog.browse_filename( mode="r", exts=( @@ -671,10 +675,10 @@ def reload_molecule_info(self): ), ) if not new_session_file: - warnings.warn(issues.NoInputWarning("Abored recognizing sessions from input.")) + warnings.warn(issues.NoInputWarning("Abored recognizing sessions from input."), stacklevel=2) return if not os.path.exists(new_session_file): - warnings.warn(issues.NoInputWarning(f"File does not exist: {new_session_file}.")) + warnings.warn(issues.NoInputWarning(f"File does not exist: {new_session_file}."), stacklevel=2) return cmd.reinitialize() @@ -701,13 +705,13 @@ def save_as_a_session(self, cfg_to_pse: str): self.logging.info(f"Output file is set as {output_pse_fn}") self.bus.set_widget_value(cfg_to_pse, output_pse_fn) else: - warnings.warn(issues.NoInputWarning(f"Invalid output path: {output_pse_fn}.")) + warnings.warn(issues.NoInputWarning(f"Invalid output path: {output_pse_fn}."), stacklevel=2) def update_chain_id(self): """Update chain ids for picked molecule.""" molecule = self.bus.get_widget_value("ui.header_panel.input.molecule", str) if not molecule: - warnings.warn(issues.NoInputWarning("No available designable molecule!")) + warnings.warn(issues.NoInputWarning("No available designable molecule!"), stacklevel=2) return chain_ids = find_all_protein_chain_ids_in_protein(molecule) self.designable_sequences = RosettaPyProteinSequence.from_dict( @@ -735,18 +739,23 @@ def find_session_path(self) -> str | None: session_path: str = cmd.get("session_file") if not session_path: - warnings.warn(issues.EmptySessionWarning("Session not found, please use a new session path to save.")) + warnings.warn( + issues.EmptySessionWarning("Session not found, please use a new session path to save."), stacklevel=2 + ) return self.file_dialog.browse_filename(mode="w", exts=(file_extensions.Session,)) if not os.path.exists(session_path): - warnings.warn(issues.NoInputWarning("Invalid session file path, please use a new session path to save.")) + warnings.warn( + issues.NoInputWarning("Invalid session file path, please use a new session path to save."), stacklevel=2 + ) return self.file_dialog.browse_filename(mode="w", exts=(file_extensions.Session,)) if os.path.basename(session_path).startswith("tmp") and session_path.endswith(".pse"): warnings.warn( issues.InvalidSessionWarning( f"Found temperal session path: {session_path}, " "please use a new session path to save." - ) + ), + stacklevel=2, ) return self.file_dialog.browse_filename(mode="w", exts=(file_extensions.Session,)) @@ -761,7 +770,7 @@ def reload_determine_tab_setup(self): """Setup pocket determination""" molecule = self.bus.get_widget_value("ui.header_panel.input.molecule", str) if not molecule: - warnings.warn(issues.NoResultsWarning("No design molecule found.")) + warnings.warn(issues.NoResultsWarning("No design molecule found."), stacklevel=2) return small_molecules = [""] @@ -769,7 +778,7 @@ def reload_determine_tab_setup(self): small_molecules.extend(more_hetatm) self.logging.info(f"Small molecules found: {more_hetatm}") else: - warnings.warn(issues.NoResultsWarning("No small molecule found.")) + warnings.warn(issues.NoResultsWarning("No small molecule found."), stacklevel=2) self.bus.set_widget_value("ui.prepare.input.pocket.substrate", small_molecules) self.bus.set_widget_value("ui.prepare.input.pocket.cofactor", small_molecules) @@ -945,7 +954,7 @@ def update_mutant_table_columns(self): comboBox_group_name = self.bus.get_widget_from_cfg_item("ui.visualize.input.group_name") if not os.path.exists(mut_table_fp): - warnings.warn(issues.NoInputWarning(f"Mutant Table path is not valid: {mut_table_fp}")) + warnings.warn(issues.NoInputWarning(f"Mutant Table path is not valid: {mut_table_fp}"), stacklevel=2) # reset cols to combo boxes to empty for comboBox in [comboBox_best_leaf, comboBox_totalscore]: set_widget_value(comboBox, [""]) @@ -956,7 +965,9 @@ def update_mutant_table_columns(self): mut_table_cols = get_mutant_table_columns(mutfile=mut_table_fp) if not mut_table_cols: - warnings.warn(issues.BadDataWarning(f"Mutant Table column names is not valid: {mut_table_cols}")) + warnings.warn( + issues.BadDataWarning(f"Mutant Table column names is not valid: {mut_table_cols}"), stacklevel=2 + ) return # set cols to combo boxes diff --git a/src/REvoDesign/citations/citation_manager.py b/src/REvoDesign/citations/citation_manager.py index 1682f3d3..68f06e92 100644 --- a/src/REvoDesign/citations/citation_manager.py +++ b/src/REvoDesign/citations/citation_manager.py @@ -69,7 +69,7 @@ def update(self, new_citations: dict): new_citations (dict): A dictionary containing new citations. """ if not (new_citations and isinstance(new_citations, dict)): - warnings.warn(issues.NoInputWarning(f"{new_citations=} is not a valid dictionary.")) + warnings.warn(issues.NoInputWarning(f"{new_citations=} is not a valid dictionary."), stacklevel=2) return self.called_citations.update(new_citations) @@ -91,7 +91,7 @@ def output(self, cwd: str = "."): library = bibtexparser.parse_string("\n".join(self.collected_citations)) if library.failed_blocks: - warnings.warn(issues.REvoDesignWarning(f"Could not parse {library.failed_blocks=}")) + warnings.warn(issues.REvoDesignWarning(f"Could not parse {library.failed_blocks=}"), stacklevel=2) citation_output = os.path.join( cwd, diff --git a/src/REvoDesign/clients/QtSocketConnector.py b/src/REvoDesign/clients/QtSocketConnector.py index 0259fc04..5e03d32e 100644 --- a/src/REvoDesign/clients/QtSocketConnector.py +++ b/src/REvoDesign/clients/QtSocketConnector.py @@ -15,6 +15,7 @@ import base64 import copy + # Inter-process Qt socket serialization uses a trusted local context. import pickle # nosec B403 import socket @@ -82,7 +83,7 @@ def does_user_exist(self, uuid: str) -> bool: def add_user(self, uuid: str, user: Client): if uuid in self.clients: - warnings.warn(issues.SocketUserAlreadyExists("user already joined.")) + warnings.warn(issues.SocketUserAlreadyExists("user already joined."), stacklevel=2) return self.clients[uuid] = user @@ -203,13 +204,15 @@ def unpack(package: bytes) -> dict[str, str]: def digest_dict(self, unpacked_msg_dict: dict[str, str]) -> tuple[supported_datatypes | None, Any]: datatype: self.supported_datatypes = unpacked_msg_dict.get("datatype") if not datatype: - warnings.warn(issues.BadDataWarning(f"bad data {datatype=} has been discarded")) + warnings.warn(issues.BadDataWarning(f"bad data {datatype=} has been discarded"), stacklevel=2) return None, None obj_encoded = unpacked_msg_dict.get("obj") obj = self.decode(obj_encoded) expected_obj_type = self.supported_datatypes_mapping.get(datatype) if not isinstance(obj, expected_obj_type): - warnings.warn(issues.BadDataWarning(f"Bad data type mismatch: {datatype=}: {expected_obj_type=}")) + warnings.warn( + issues.BadDataWarning(f"Bad data type mismatch: {datatype=}: {expected_obj_type=}"), stacklevel=2 + ) return None, None return datatype, obj @@ -228,7 +231,7 @@ def broadcast( all_clients = [meetingroom.client] elif isinstance(meetingroom, MeetingRoom): if not meetingroom or meetingroom.empty: - warnings.warn(issues.SocketMeetingRoomIsEmpty("Empty meeting room.")) + warnings.warn(issues.SocketMeetingRoomIsEmpty("Empty meeting room."), stacklevel=2) return all_clients = meetingroom.current_clients elif isinstance(meetingroom, list): @@ -277,7 +280,7 @@ def received(self, packed_msg: bytes) -> tuple[None, None] | tuple[str, Any] | t unpacked_msg = self.unpack(packed_msg) (datatype, obj) = self.digest_dict(unpacked_msg_dict=unpacked_msg) if not datatype: - warnings.warn(issues.BadDataWarning("Bad data received with unexpected none datatype.")) + warnings.warn(issues.BadDataWarning("Bad data received with unexpected none datatype."), stacklevel=2) return None, None if datatype != "MessageStack": return datatype, obj @@ -477,11 +480,15 @@ def processBinaryMessage(self, client, message: QtCore.QByteArray): try: msg_type, msg_content = self.bc_worker.received(packed_msg=message) except issues.FobbidenDataTypeError: - warnings.warn(issues.BadDataWarning(f"Bad data from {username}@{node}({user_id}) is discarded.")) + warnings.warn( + issues.BadDataWarning(f"Bad data from {username}@{node}({user_id}) is discarded."), stacklevel=2 + ) return if not self.bc_worker.typing_is_valid(msg_type=msg_type, msg_content=msg_content): - warnings.warn(issues.BadDataWarning(f"Bad data from socket discarded: {msg_type=},{msg_content=}")) + warnings.warn( + issues.BadDataWarning(f"Bad data from socket discarded: {msg_type=},{msg_content=}"), stacklevel=2 + ) return # printout readable msgs @@ -632,7 +639,7 @@ async def broadcast_object(self, obj, data_type): None """ if not self.is_running: - warnings.warn(issues.SocketWarning("Host is not running")) + warnings.warn(issues.SocketWarning("Host is not running"), stacklevel=2) return self.bc_worker.broadcast(meetingroom=self.meetingroom, obj=obj, obj_type=data_type) @@ -916,7 +923,8 @@ def messageDispatcher(self, msg_type: str, msg_content: Any): warnings.warn( issues.FobbidenDataTypeError( f"The content of a MessageStack is expected to be an Iterable instead of a {type(msg_content)}" - ) + ), + stacklevel=2, ) return @@ -926,12 +934,14 @@ def messageDispatcher(self, msg_type: str, msg_content: Any): issues.FobbidenDataTypeError( f"The content of a MessageStack item {msg_data_item=} is expected to be " f"a tuple with 2 item instead of a {type(msg_content)}" - ) + ), + stacklevel=2, ) continue if (len_msg_content := len(msg_data_item)) != 2: warnings.warn( - issues.FobbidenDataTypeError(f"The MessageStack item has invalid length {len_msg_content}") + issues.FobbidenDataTypeError(f"The MessageStack item has invalid length {len_msg_content}"), + stacklevel=2, ) continue @@ -951,11 +961,13 @@ def handleBinaryMessage(self, message: QtCore.QByteArray): try: msg_type, msg_content = self.bc_worker.received(packed_msg=msg_bytes) except issues.FobbidenDataTypeError: - warnings.warn(issues.BadDataWarning("Bad data from Host is discarded.")) + warnings.warn(issues.BadDataWarning("Bad data from Host is discarded."), stacklevel=2) return if not self.bc_worker.typing_is_valid(msg_type=msg_type, msg_content=msg_content): - warnings.warn(issues.BadDataWarning(f"Bad data from socket discarded: {msg_type=},{msg_content=}")) + warnings.warn( + issues.BadDataWarning(f"Bad data from socket discarded: {msg_type=},{msg_content=}"), stacklevel=2 + ) return if msg_type in self.bc_worker.readble_type: diff --git a/src/REvoDesign/common/multi_mutant_designer.py b/src/REvoDesign/common/multi_mutant_designer.py index efb7b8c4..40283783 100644 --- a/src/REvoDesign/common/multi_mutant_designer.py +++ b/src/REvoDesign/common/multi_mutant_designer.py @@ -182,7 +182,9 @@ def evaluate_design(self) -> Mutant: tmp_mutant_obj.wt_protein_sequence = self.designable_sequences if not self.magician.gimmick: - warnings.warn(issues.ConflictWarning("Abord design evaluation because no external scorer is defined.")) + warnings.warn( + issues.ConflictWarning("Abord design evaluation because no external scorer is defined."), stacklevel=2 + ) else: tmp_mutant_obj.mutant_score = self.magician.gimmick.scorer(mutant=tmp_mutant_obj) @@ -268,7 +270,8 @@ def pick_next_mutant(self): warnings.warn( issues.NoInputWarning( "Temperal mutant tree for multi-design is empty after designing! This design is ended." - ) + ), + stacklevel=2, ) self.start_new_design() return @@ -278,7 +281,7 @@ def pick_next_mutant(self): self._auto_pick_tryout() if num_mut_before_picking == self.in_design_multi_design_case.mutant_num: # failed picking - warnings.warn(issues.NoResultsWarning("Failed auto picking. Please take anther try.")) + warnings.warn(issues.NoResultsWarning("Failed auto picking. Please take anther try."), stacklevel=2) return # last mutant diff --git a/src/REvoDesign/common/mutant_visualise.py b/src/REvoDesign/common/mutant_visualise.py index 99e83418..077f430e 100644 --- a/src/REvoDesign/common/mutant_visualise.py +++ b/src/REvoDesign/common/mutant_visualise.py @@ -418,6 +418,7 @@ def merge_sessions_via_commandline(self): f"Temperal merged result is failed to create. Try again with a clean PyMOL session. \n" f"STDOUT:\n{merge_results.stdout}\n" f"STDERR:\n{merge_results.stderr}" - ) + ), + stacklevel=2, ) return diff --git a/src/REvoDesign/editor/monaco/server.py b/src/REvoDesign/editor/monaco/server.py index 2a747283..820c1b67 100644 --- a/src/REvoDesign/editor/monaco/server.py +++ b/src/REvoDesign/editor/monaco/server.py @@ -150,7 +150,7 @@ async def lifespan(app: FastAPI): config_store = ConfigStore() html_dir = config_store.get("editor.backend.html_dir") if not html_dir: - warnings.warn(issues.MissingExternalTool("Monaco Editor is not ready.")) + warnings.warn(issues.MissingExternalTool("Monaco Editor is not ready."), stacklevel=2) # ensure monaco editor is ready ensure_monaco() # re-fetch the updated value from config store diff --git a/src/REvoDesign/magician/designers/colabdesign.py b/src/REvoDesign/magician/designers/colabdesign.py index 74ad08ee..c5580801 100644 --- a/src/REvoDesign/magician/designers/colabdesign.py +++ b/src/REvoDesign/magician/designers/colabdesign.py @@ -95,14 +95,14 @@ def scorer(self, mutant: Mutant | RosettaPyProteinSequence, **kwargs) -> float: if isinstance(mutant, Mutant): if len(mutant.wt_protein_sequence.all_chain_ids) > 1: - warnings.warn(issues.REvoDesignWarning("MPNN only supports single chain sequences")) + warnings.warn(issues.REvoDesignWarning("MPNN only supports single chain sequences"), stacklevel=2) sequence: str = mutant.get_mutant_sequence_single_chain( chain_id=mutant.wt_protein_sequence.all_chain_ids[0], ignore_missing=True, ).sequence else: if len(mutant.all_chain_ids) > 1: - warnings.warn(issues.REvoDesignWarning("MPNN only supports single chain sequences")) + warnings.warn(issues.REvoDesignWarning("MPNN only supports single chain sequences"), stacklevel=2) sequence: str = mutant.get_sequence_by_chain(chain_id=mutant.all_chain_ids[0]).replace("X", "") # scorer must return a float score value given a mutant sequence. return self.mpnn_model.score(seq=sequence)["score"] diff --git a/src/REvoDesign/phylogenetics/evo_mutator.py b/src/REvoDesign/phylogenetics/evo_mutator.py index 36152fce..29ec685e 100644 --- a/src/REvoDesign/phylogenetics/evo_mutator.py +++ b/src/REvoDesign/phylogenetics/evo_mutator.py @@ -193,7 +193,7 @@ def run_mutant_loading_from_profile(self): ) if not os.path.isdir(os.path.dirname(self.design.output_pse)): - warnings.warn(issues.NoResultsWarning("No output PyMOL session is created.")) + warnings.warn(issues.NoResultsWarning("No output PyMOL session is created."), stacklevel=2) return cmd.reinitialize() @@ -409,7 +409,7 @@ def _get_dist( atom2 = self._get_ca_atom(chain_2, j_1) return atom1 - atom2 except ValueError as e: - warnings.warn(f"Error calculating distance: {e}", category=UserWarning) + warnings.warn(f"Error calculating distance: {e}", category=UserWarning, stacklevel=2) return -1 def bind_chains(self, coevolved_pairs: tuple[CoevolvedPair]) -> tuple[CoevolvedPair]: @@ -616,7 +616,7 @@ def run_gremlin_tool(self): ) if not coevolved_pairs: - warnings.warn(issues.NoResultsWarning("No Available co-evolutionary signal in global")) + warnings.warn(issues.NoResultsWarning("No Available co-evolutionary signal in global"), stacklevel=2) # early return if no data. return @@ -644,7 +644,7 @@ def run_gremlin_tool(self): del chain_binder if self.coevolved_pairs.empty: - warnings.warn(issues.NoResultsWarning("No coevolved_pairs passes filter.")) + warnings.warn(issues.NoResultsWarning("No coevolved_pairs passes filter."), stacklevel=2) return logging.info("Visualizing as bonds ...") @@ -658,7 +658,9 @@ def run_gremlin_tool(self): self.bus.button("previous").clicked.disconnect() self.bus.button("next").clicked.disconnect() except Exception as e: - warnings.warn(issues.AlreadyDisconnectedWarning(f"button is already disconnected. do nothing. {e=}")) + warnings.warn( + issues.AlreadyDisconnectedWarning(f"button is already disconnected. do nothing. {e=}"), stacklevel=2 + ) self.bus.button("previous").clicked.connect(partial(self.load_co_evolving_pairs, False)) diff --git a/src/REvoDesign/phylogenetics/gremlin_tools.py b/src/REvoDesign/phylogenetics/gremlin_tools.py index bd1480dd..c5ea7cc6 100644 --- a/src/REvoDesign/phylogenetics/gremlin_tools.py +++ b/src/REvoDesign/phylogenetics/gremlin_tools.py @@ -83,7 +83,7 @@ def is_out_of_range(self, chain_pair: str) -> bool: @property def min_dist(self): if self.empty: - warnings.warn(issues.NoInputWarning(f"Pair {repr(self)} is empty! ")) + warnings.warn(issues.NoInputWarning(f"Pair {repr(self)} is empty! "), stacklevel=2) return -1 return min(d for d in self.homochains_dist.values() if d > 0) @@ -332,7 +332,8 @@ def plot_w(self, i: int, j: int, i_aa: str, j_aa: str, idx: int = 0) -> Coevolve if not matching_indices: # No matching pairs found, handle this case warnings.warn( - issues.NoResultsWarning(f"No matching co-evolutionary pairs found for positions {i} and {j}.") + issues.NoResultsWarning(f"No matching co-evolutionary pairs found for positions {i} and {j}."), + stacklevel=2, ) return None @@ -381,7 +382,7 @@ def plot_w(self, i: int, j: int, i_aa: str, j_aa: str, idx: int = 0) -> Coevolve traceback.print_exc() logging.error(f"Error occured while processing '{wt_i_aa=}' or '{wt_j_aa=}' from {self.alphabet=}") # early return to skip ploting - warnings.warn(issues.BadDataWarning(f"Bad pair: {str(a_pair)}")) + warnings.warn(issues.BadDataWarning(f"Bad pair: {str(a_pair)}"), stacklevel=2) return None plt.text( @@ -451,7 +452,7 @@ def plot_w_o2a(self, resi) -> tuple[CoevolvedPair]: top_N_pairs = coevolving_pairs[: self.topN] if not top_N_pairs: - warnings.warn(issues.NoResultsWarning("No coevolving pairs found!")) + warnings.warn(issues.NoResultsWarning("No coevolving pairs found!"), stacklevel=2) return {} logging.info(f"top {self.topN} items selected: {str(top_N_pairs)}") diff --git a/src/REvoDesign/shortcuts/tools/designs.py b/src/REvoDesign/shortcuts/tools/designs.py index 6e46893c..42d88ced 100644 --- a/src/REvoDesign/shortcuts/tools/designs.py +++ b/src/REvoDesign/shortcuts/tools/designs.py @@ -31,6 +31,6 @@ def shortcut_pssm2csv(pssm: str) -> None: expected_csv = f"{pssm}.csv" if not os.path.exists(expected_csv): - warnings.warn(issues.NoResultsWarning(f"Expected {expected_csv=}")) + warnings.warn(issues.NoResultsWarning(f"Expected {expected_csv=}"), stacklevel=2) logging.info(expected_csv) diff --git a/src/REvoDesign/shortcuts/tools/represents.py b/src/REvoDesign/shortcuts/tools/represents.py index 171459b3..7d681bec 100644 --- a/src/REvoDesign/shortcuts/tools/represents.py +++ b/src/REvoDesign/shortcuts/tools/represents.py @@ -612,7 +612,8 @@ def assign_to_res_pymol(self, pos_one_idx: int, state: int = 0) -> float: warnings.warn( issues.BadDataWarning( f"Failed to set B-factor for position {pos_one_idx} (zero-indexed {pos_one_idx - 1})" - ) + ), + stacklevel=2, ) return bfact diff --git a/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py b/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py index 48262baa..55d3ea48 100644 --- a/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py +++ b/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py @@ -5,6 +5,7 @@ import glob import os + # Traceback serialization uses a trusted context. import pickle # nosec B403 import random @@ -77,7 +78,8 @@ def fetch_cuda_version_before_install(self): warnings.warn( issues.PlatformNotSupportedWarning( f"CUDA version {cuda_version} is not supported by DGL. Please install CUDA version >= 11.8 if you need to use DGL with CUDA support." - ) + ), + stacklevel=2, ) def install(self): @@ -166,7 +168,8 @@ def pick_model(self, model_name: str | None = None) -> None: warnings.warn( issues.FallingBackWarning( f"Model {model_name} not found. Falling back to pick one according to the input." - ) + ), + stacklevel=2, ) # automatically pick the model based on the input @@ -209,7 +212,7 @@ def ensure_dgl(): if (dgl_solver := DglSolver()).installed: return - warnings.warn(issues.MissingExternalTool("DGL is not installed. Now try to install it.")) + warnings.warn(issues.MissingExternalTool("DGL is not installed. Now try to install it."), stacklevel=2) dgl_solver.install() if not dgl_solver.installed: diff --git a/src/REvoDesign/tools/customized_widgets.py b/src/REvoDesign/tools/customized_widgets.py index 08692fb4..bd89a3d0 100644 --- a/src/REvoDesign/tools/customized_widgets.py +++ b/src/REvoDesign/tools/customized_widgets.py @@ -115,7 +115,7 @@ def __init__(self, object_name: str | None = None, allow_repeat: bool = False, p try: self.check_repeat() except RuntimeError as e: - warnings.warn(issues.REvoDesignWidgetWarning(e)) + warnings.warn(issues.REvoDesignWidgetWarning(e), stacklevel=2) self.destroy() raise RuntimeError(f"a window named {self.objectName()} is already open.") from e @@ -154,9 +154,7 @@ def check_repeat(self): # Find windows with the same name the_windows = [ - w - for w in bus.ui.open_windows - if hasattr(w, "objectName") and w.objectName() == self.objectName() + w for w in bus.ui.open_windows if hasattr(w, "objectName") and w.objectName() == self.objectName() ] if any(the_windows): # Raise the existing window to the front diff --git a/src/REvoDesign/tools/mutant_tools.py b/src/REvoDesign/tools/mutant_tools.py index 305f568d..e17bc1e5 100644 --- a/src/REvoDesign/tools/mutant_tools.py +++ b/src/REvoDesign/tools/mutant_tools.py @@ -135,7 +135,7 @@ def extract_mutants_from_mutant_id( _mut_res = _mut.group(2) else: - warnings.warn(issues.BadDataWarning(f"Error while processing mutant id {mut}. ")) + warnings.warn(issues.BadDataWarning(f"Error while processing mutant id {mut}. "), stacklevel=2) continue mutations.append( @@ -222,7 +222,7 @@ def extract_mutant_from_sequences( return if "X" in wt_sequence and not fix_missing: - warnings.warn(issues.ResidueMissingWarning('WT has missing residue masked as "X"!')) + warnings.warn(issues.ResidueMissingWarning('WT has missing residue masked as "X"!'), stacklevel=2) if fix_missing: _mutant_sequence = "" @@ -391,7 +391,8 @@ def extract_mutant_from_pymol_object(pymol_object: str, sequences: RosettaPyProt warnings.warn( issues.BadDataWarning( f"{at.resn} at {at.resi} (chain {chain.chain_id}) is out of range of sequence length." - ) + ), + stacklevel=2, ) continue @@ -552,7 +553,7 @@ def quick_mutagenesis(mutant_tree: MutantTree) -> None: nproc = bus.get_value("ui.header_panel.nproc") if mutant_tree.empty: - warnings.warn(issues.NoResultsWarning("Mutant tree is empty!")) + warnings.warn(issues.NoResultsWarning("Mutant tree is empty!"), stacklevel=2) return score_list = mutant_tree.all_mutant_scores @@ -614,7 +615,7 @@ def save_mutant_choices(output_mut_txt_fn: str, mutant_tree: MutantTree): raise issues.NoInputError("No Mutant tree is given!") if mutant_tree.empty: - warnings.warn(issues.NoResultsWarning("mutant tree is empty. save nothing.")) + warnings.warn(issues.NoResultsWarning("mutant tree is empty. save nothing."), stacklevel=2) return mutants_to_save = mutant_tree.all_mutant_ids @@ -629,7 +630,9 @@ def save_mutant_choices(output_mut_txt_fn: str, mutant_tree: MutantTree): return if os.path.exists(output_mut_txt_fn): - warnings.warn(issues.OverridesWarning(f"Mutant table exists and will be overriden! {output_mut_txt_fn}")) + warnings.warn( + issues.OverridesWarning(f"Mutant table exists and will be overriden! {output_mut_txt_fn}"), stacklevel=2 + ) write_input_mutant_table( output_mut_txt_fn, [mt.raw_mutant_id for mt in mutant_tree.all_mutant_objects], diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index 2f5778d0..485be474 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -21,6 +21,7 @@ import re import shutil import socket + # Used for package-management commands with argv lists and registered process lifecycle handling. import subprocess # nosec B404 import sys @@ -2720,7 +2721,7 @@ def _show_notification_dialog( # error_type is a Warning => also bool if issubclass(error_type, Warning): - warnings.warn(error_type(message)) + warnings.warn(error_type(message), stacklevel=2) return # Otherwise, raise => NoReturn diff --git a/src/REvoDesign/tools/pymol_utils.py b/src/REvoDesign/tools/pymol_utils.py index ffbefb18..4684c38e 100644 --- a/src/REvoDesign/tools/pymol_utils.py +++ b/src/REvoDesign/tools/pymol_utils.py @@ -74,7 +74,7 @@ def find_small_molecules_in_protein(sele): Returns None if the selection is not provided. """ if not sele: - warnings.warn(issues.NoInputWarning("Selection for small molecules is not provided.")) + warnings.warn(issues.NoInputWarning("Selection for small molecules is not provided."), stacklevel=2) return None # Return None if the selection is not provided # Retrieve the atoms that belong to small molecules within the selection and extract unique small molecule names @@ -88,13 +88,14 @@ def find_small_molecules_in_protein(sele): "Could not find unique small molecules with standalone chain id. \n" 'A possible fix is calling `alter r. RES, chain=""` to fix the problem \n' "then re-load this session." - ) + ), + stacklevel=2, ) # Return a list of unique small molecule names found within the selection return unique_small_molecules - warnings.warn(issues.FallingBackWarning("Falling back to all `hetatm`")) + warnings.warn(issues.FallingBackWarning("Falling back to all `hetatm`"), stacklevel=2) small_molecules = [at.resn for at in cmd.get_model("hetatm and (not polymer.protein)").atom] unique_small_molecules = list(set(small_molecules)) @@ -385,7 +386,8 @@ def is_a_REvoDesign_session(): "session's feature, you should always create seperate" "sessions according to your dataset and merge them " "manually in PyMOL window." - ) + ), + stacklevel=2, ) return check diff --git a/src/REvoDesign/tools/rosetta_utils.py b/src/REvoDesign/tools/rosetta_utils.py index 1d1cc501..e41a81d2 100644 --- a/src/REvoDesign/tools/rosetta_utils.py +++ b/src/REvoDesign/tools/rosetta_utils.py @@ -131,11 +131,11 @@ def extra_res_to_opts(ligands_params: list[str] | str) -> list[str]: ligands = [] for _, l in enumerate(ligands_params): if not (isinstance(l, str) and l.endswith(".params")): - warnings.warn(issues.BadDataWarning(f"Invalid Parameter input for ligand - {l}")) + warnings.warn(issues.BadDataWarning(f"Invalid Parameter input for ligand - {l}"), stacklevel=2) continue if not os.path.isfile(l): - warnings.warn(issues.BadDataWarning(f"Ignore nofound ligand - {l}")) + warnings.warn(issues.BadDataWarning(f"Ignore nofound ligand - {l}"), stacklevel=2) continue ligands.extend(["-extra_res_fa" if l.endswith("fa.params") else "-extra_res_cen", os.path.abspath(l)]) @@ -191,7 +191,7 @@ def is_wsl_available(): return wsl_bin is not None except RuntimeError: # If an error occurs, it indicates that WSL may not be available - warnings.warn(issues.PlatformNotSupportedWarning("WSL is not available on this machine.")) + warnings.warn(issues.PlatformNotSupportedWarning("WSL is not available on this machine."), stacklevel=2) # Return False, indicating that WSL is not available return False @@ -218,7 +218,8 @@ def is_docker_available() -> bool: warnings.warn( issues.PlatformNotSupportedWarning( f"Docker is not available(uninstalled or unlaunched) on this machine: {e}" - ) + ), + stacklevel=2, ) return False diff --git a/src/REvoDesign/tools/system_tools.py b/src/REvoDesign/tools/system_tools.py index a924a150..41c98996 100644 --- a/src/REvoDesign/tools/system_tools.py +++ b/src/REvoDesign/tools/system_tools.py @@ -49,7 +49,8 @@ def check_mac_rosetta2(): issues.AppleSiliconRosetta2Warning( "Oops! You are in Rosetta-translated PyMOL bundle from official channel. " "This might limit the performance of joblib, causing MutantVisualizer slower." - ) + ), + stacklevel=2, ) From 7bcbb14f039a69903f0c837ddfa73b7108c2c5cd Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:13:58 +0800 Subject: [PATCH 21/63] fix(deepsource): expand overload stubs --- src/REvoDesign/common/mutant_tree.py | 9 +++-- src/REvoDesign/driver/ui_driver.py | 12 ++++-- src/REvoDesign/shortcuts/tools/vina_tools.py | 6 ++- src/REvoDesign/tools/customized_widgets.py | 39 +++++++++++++------- src/REvoDesign/tools/package_manager.py | 12 ++++-- 5 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/REvoDesign/common/mutant_tree.py b/src/REvoDesign/common/mutant_tree.py index eb956813..54358903 100644 --- a/src/REvoDesign/common/mutant_tree.py +++ b/src/REvoDesign/common/mutant_tree.py @@ -24,11 +24,14 @@ class MutantDict(TypedDict): class MutateRunner(Protocol): - def run_mutate_parallel(self, mutants: list[Mutant], nproc: int = 2) -> list[str]: ... + def run_mutate_parallel(self, mutants: list[Mutant], nproc: int = 2) -> list[str]: + ... - def mutated_pdb_mapping(self, mutant_tree: "MutantTree", pdb_fps: list[str]) -> "MutantTree": ... + def mutated_pdb_mapping(self, mutant_tree: "MutantTree", pdb_fps: list[str]) -> "MutantTree": + ... - def cite(self) -> None: ... + def cite(self) -> None: + ... class MutantTree: diff --git a/src/REvoDesign/driver/ui_driver.py b/src/REvoDesign/driver/ui_driver.py index ed5b238d..71f6d45f 100644 --- a/src/REvoDesign/driver/ui_driver.py +++ b/src/REvoDesign/driver/ui_driver.py @@ -511,7 +511,8 @@ def get_value( reject_none: bool, default_value: None = ..., cfg: Config | str = "main", - ) -> ValueFromConfigT: ... + ) -> ValueFromConfigT: + ... @overload def get_value( @@ -521,7 +522,8 @@ def get_value( reject_none: bool, default_value: bool = ..., cfg: Config | str = "main", - ) -> bool: ... + ) -> bool: + ... @overload def get_value( @@ -531,10 +533,12 @@ def get_value( reject_none: bool = True, default_value: ValueFromConfigT | None = ..., cfg: Config | str = "main", - ) -> ValueFromConfigT: ... + ) -> ValueFromConfigT: + ... @overload - def get_value(self, cfg_item: str, converter: None) -> Any: ... + def get_value(self, cfg_item: str, converter: None) -> Any: + ... def get_value( self, diff --git a/src/REvoDesign/shortcuts/tools/vina_tools.py b/src/REvoDesign/shortcuts/tools/vina_tools.py index 135e3efc..d6061177 100644 --- a/src/REvoDesign/shortcuts/tools/vina_tools.py +++ b/src/REvoDesign/shortcuts/tools/vina_tools.py @@ -383,7 +383,8 @@ def from_selection( @overload -def showbox(box: str, minX: float, maxX: float, minY: float, maxY: float, minZ: float, maxZ: float) -> CgoBox: ... +def showbox(box: str, minX: float, maxX: float, minY: float, maxY: float, minZ: float, maxZ: float) -> CgoBox: + ... @overload @@ -395,7 +396,8 @@ def showbox( maxY: float | str | None = None, minZ: float | str | None = None, maxZ: float | str | None = None, -) -> CgoBox: ... +) -> CgoBox: + ... def showbox( diff --git a/src/REvoDesign/tools/customized_widgets.py b/src/REvoDesign/tools/customized_widgets.py index bd89a3d0..7a529e9a 100644 --- a/src/REvoDesign/tools/customized_widgets.py +++ b/src/REvoDesign/tools/customized_widgets.py @@ -1361,37 +1361,45 @@ def getOpenFileNameWithExt(*args, **kwargs): @overload -def set_widget_value(widget: QtWidgets.QFontComboBox, value: QtGui.QFont | str): ... +def set_widget_value(widget: QtWidgets.QFontComboBox, value: QtGui.QFont | str): + ... @overload -def set_widget_value(widget: QtWidgets.QStackedWidget, value: list): ... +def set_widget_value(widget: QtWidgets.QStackedWidget, value: list): + ... @overload -def set_widget_value(widget: QtWidgets.QProgressBar, value: int | list[int] | tuple[int, int]): ... +def set_widget_value(widget: QtWidgets.QProgressBar, value: int | list[int] | tuple[int, int]): + ... @overload def set_widget_value( widget: QtWidgets.QDoubleSpinBox | QtWidgets.QSpinBox, value: int | float | list[str] | tuple[str, str] -): ... +): + ... @overload -def set_widget_value(widget: MultiCheckableComboBox, value: list | tuple | str | int | float): ... +def set_widget_value(widget: MultiCheckableComboBox, value: list | tuple | str | int | float): + ... @overload -def set_widget_value(widget: QtWidgets.QComboBox, value: list | tuple | dict | str | int | float | bool): ... +def set_widget_value(widget: QtWidgets.QComboBox, value: list | tuple | dict | str | int | float | bool): + ... @overload -def set_widget_value(widget: QtWidgets.QGridLayout, value: str): ... +def set_widget_value(widget: QtWidgets.QGridLayout, value: str): + ... @overload -def set_widget_value(widget: QtWidgets.QLineEdit | QtWidgets.QLCDNumber | QtWidgets.QCheckBox, value: Any): ... +def set_widget_value(widget: QtWidgets.QLineEdit | QtWidgets.QLCDNumber | QtWidgets.QCheckBox, value: Any): + ... def set_widget_value(widget, value): @@ -1492,25 +1500,30 @@ def set_value_error(widget: QtWidgets.QWidget, value: Any): @overload -def get_widget_value(widget: QtWidgets.QCheckBox) -> bool: ... # type: ignore +def get_widget_value(widget: QtWidgets.QCheckBox) -> bool: # type: ignore + ... @overload def get_widget_value( widget: QtWidgets.QComboBox | QtWidgets.QLineEdit | QtWidgets.QFontComboBox, -) -> str: ... # type: ignore +) -> str: # type: ignore + ... @overload -def get_widget_value(widget: QtWidgets.QDoubleSpinBox | QtWidgets.QLCDNumber) -> float: ... # type: ignore +def get_widget_value(widget: QtWidgets.QDoubleSpinBox | QtWidgets.QLCDNumber) -> float: # type: ignore + ... @overload -def get_widget_value(widget: QtWidgets.QSpinBox | QtWidgets.QProgressBar) -> int: ... # type: ignore +def get_widget_value(widget: QtWidgets.QSpinBox | QtWidgets.QProgressBar) -> int: # type: ignore + ... @overload -def get_widget_value(widget: MultiCheckableComboBox) -> list[str]: ... # type: ignore +def get_widget_value(widget: MultiCheckableComboBox) -> list[str]: # type: ignore + ... def get_widget_value(widget: QtWidgets.QWidget) -> Any: diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index 485be474..fdb10306 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -2667,14 +2667,16 @@ def callback(): @overload -def notify_box(message: str = "", error_type: None | type[Warning] = None, details: str | None = None) -> None: ... +def notify_box(message: str = "", error_type: None | type[Warning] = None, details: str | None = None) -> None: + ... # Overload #2: Exception => NoReturn @overload -def notify_box(message: str, error_type: type[Exception], details: str | None = None) -> NoReturn: ... +def notify_box(message: str, error_type: type[Exception], details: str | None = None) -> NoReturn: + ... def notify_box( @@ -2746,7 +2748,8 @@ def run_worker_thread_in_pool( trigger_buttons: QtWidgets.QPushButton | Iterable[QtWidgets.QPushButton] | None = None, notify_slot: Callable[[str], None] | None = None, **kwargs, -) -> R: ... +) -> R: + ... @overload @@ -2756,7 +2759,8 @@ def run_worker_thread_in_pool( trigger_buttons: QtWidgets.QPushButton | Iterable[QtWidgets.QPushButton] | None = None, notify_slot: Callable[[str], None] | None = None, **kwargs, -) -> R | None: ... +) -> R | None: + ... def run_worker_thread_in_pool( From 2049401f92c9f1b7702f9cec14751624a6d9cdcc Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:39:20 +0800 Subject: [PATCH 22/63] fix(deepsource): wrap long source lines --- src/REvoDesign/application/cluster_tab.py | 24 ++++++--- .../designers/openkinetics/_scorers.py | 33 ++++++++---- src/REvoDesign/tools/cgo_utils.py | 52 +++++++++++++------ src/REvoDesign/tools/package_manager.py | 39 ++++++++------ 4 files changed, 98 insertions(+), 50 deletions(-) diff --git a/src/REvoDesign/application/cluster_tab.py b/src/REvoDesign/application/cluster_tab.py index 3ce4900a..c9b52d45 100644 --- a/src/REvoDesign/application/cluster_tab.py +++ b/src/REvoDesign/application/cluster_tab.py @@ -145,19 +145,24 @@ def _install_tooltips(self) -> None: "AgglomerativeCluster uses the sequence distance matrix with average linkage and a precomputed metric." ), self.ui.label_cluster_agglomerative_representative_value: ( - "Representatives are selected as the nearest centroid among clustered variants. This is not a medoid policy." + "Representatives are selected as the nearest centroid among clustered variants. " + "This is not a medoid policy." ), self.ui.page_cluster_evo: ( - "Optional Evo inputs with positive weights are used when available; missing optional inputs are skipped and remaining weights are renormalized." + "Optional Evo inputs with positive weights are used when available; " + "missing optional inputs are skipped and remaining weights are renormalized." ), self.ui.lineEdit_cluster_evo_pssm_profile: ( - "Optional PSSM input for EvoCluster. If the corresponding weight is positive, the file path must exist." + "Optional PSSM input for EvoCluster. If the corresponding weight is positive, " + "the file path must exist." ), self.ui.lineEdit_cluster_evo_esm1v_table: ( - "Optional ESM-1v input for EvoCluster. If the corresponding weight is positive, the file path must exist." + "Optional ESM-1v input for EvoCluster. If the corresponding weight is positive, " + "the file path must exist." ), self.ui.lineEdit_cluster_evo_structure_pdb: ( - "Optional structure input for EvoCluster spatial distance. A temporary input PDB may be generated when needed." + "Optional structure input for EvoCluster spatial distance. " + "A temporary input PDB may be generated when needed." ), self.ui.lineEdit_cluster_evo_esm_mutation_col: "Column name used to read ESM mutation identifiers.", self.ui.page_cluster_kmeans: ( @@ -170,13 +175,16 @@ def _install_tooltips(self) -> None: "LegacyCluster is a deprecated compatibility mode that uses Ward linkage on the score matrix." ), self.ui.label_cluster_legacy_warning: ( - "Use this only to reproduce historical behavior. Prefer AgglomerativeCluster or EvoCluster for new analyses." + "Use this only to reproduce historical behavior. " + "Prefer AgglomerativeCluster or EvoCluster for new analyses." ), self.ui.checkBox_cluster_mutate_and_relax: ( - "Run Rosetta Mutate/Relax scoring after clustering without rewriting representative FASTA files by default." + "Run Rosetta Mutate/Relax scoring after clustering without rewriting representative " + "FASTA files by default." ), self.ui.checkBox_cluster_rosetta_override_representatives: ( - "Rosetta scoring normally exports score tables only. Representative FASTA files are rewritten only when this explicit override option is enabled." + "Rosetta scoring normally exports score tables only. Representative FASTA files " + "are rewritten only when this explicit override option is enabled." ), } diff --git a/src/REvoDesign/magician/designers/openkinetics/_scorers.py b/src/REvoDesign/magician/designers/openkinetics/_scorers.py index 6d31e327..5199a09b 100644 --- a/src/REvoDesign/magician/designers/openkinetics/_scorers.py +++ b/src/REvoDesign/magician/designers/openkinetics/_scorers.py @@ -61,7 +61,9 @@ _PREDICTOR_BIBTEX = { "CataPro": r"""@article{Wang2025CataPro, title = {Robust enzyme discovery and engineering with deep learning using CataPro}, - author = {Wang, Zechen and Xie, Dongqi and Wu, Dong and Luo, Xiaozhou and Wang, Sheng and Li, Yangyang and Yang, Yanmei and Li, Weifeng and Zheng, Liangzhen}, + author = {Wang, Zechen and Xie, Dongqi and Wu, Dong and Luo, Xiaozhou and + Wang, Sheng and Li, Yangyang and Yang, Yanmei and Li, Weifeng and Zheng, + Liangzhen}, journal = {Nature Communications}, volume = {16}, number = {1}, @@ -81,7 +83,8 @@ }""", "DLKcat": r"""@article{Li2022DLKcat, title = {Deep learning-based kcat prediction enables improved enzyme-constrained model reconstruction}, - author = {Li, Feiran and Yuan, Le and Lu, Hongzhong and Li, Gang and Chen, Yu and Engqvist, Martin K. M. and Kerkhoven, Eduard J. and Nielsen, Jens}, + author = {Li, Feiran and Yuan, Le and Lu, Hongzhong and Li, Gang and Chen, + Yu and Engqvist, Martin K. M. and Kerkhoven, Eduard J. and Nielsen, Jens}, journal = {Nature Catalysis}, volume = {5}, number = {8}, @@ -102,7 +105,8 @@ url = {https://doi.org/10.1016/j.checat.2024.101094} }""", "KinForm": r"""@article{Alwer2026KinForm, - title = {KinForm: kinetics-informed feature optimised representation models for enzyme kcat and KM prediction}, + title = {KinForm: kinetics-informed feature optimised representation models + for enzyme kcat and KM prediction}, author = {Alwer, Saleh and Fleming, Ronan M. T.}, journal = {npj Systems Biology and Applications}, volume = {12}, @@ -112,7 +116,8 @@ url = {https://doi.org/10.1038/s41540-026-00692-5} }""", "MMISA-KM": r"""@inproceedings{Song2025MMISAKM, - title = {MMISA-KM: a deep-learning method using multi-modal information and self-attention mechanisms for the prediction of Michaelis constants}, + title = {MMISA-KM: a deep-learning method using multi-modal information and + self-attention mechanisms for the prediction of Michaelis constants}, author = {Song, Aijie and Wang, Kai}, booktitle = {2025 IEEE 14th Data Driven Control and Learning Systems (DDCLS)}, pages = {2023--2028}, @@ -121,15 +126,22 @@ url = {https://doi.org/10.1109/DDCLS66240.2025.11064981} }""", "OmniESI": r"""@misc{Nie2025OmniESI, - title = {OmniESI: A unified framework for enzyme-substrate interaction prediction with progressive conditional deep learning}, - author = {Nie, Zhiwei and Zhang, Hongyu and Jiang, Hao and Liu, Yutian and Huang, Xiansong and Xu, Fan and Fu, Jie and Ren, Zhixiang and Tian, Yonghong and Zhang, Wen-Bin and Chen, Jie}, + title = {OmniESI: A unified framework for enzyme-substrate interaction + prediction with progressive conditional deep learning}, + author = {Nie, Zhiwei and Zhang, Hongyu and Jiang, Hao and Liu, Yutian and + Huang, Xiansong and Xu, Fan and Fu, Jie and Ren, Zhixiang and Tian, Yonghong + and Zhang, Wen-Bin and Chen, Jie}, year = {2025}, doi = {10.48550/arXiv.2506.17963}, url = {https://doi.org/10.48550/arXiv.2506.17963} }""", "RealKcat": r"""@article{Sajeevan2025RealKcat, title = {Robust Prediction of Enzyme Variant Kinetics with RealKcat}, - author = {Sajeevan, Karuna Anna and Osinuga, Abraham and B, Arunraj and Ferdous, Sakib and Shahreen, Nabia and Noor, Mohammed and Koneru, Shashank and Santa-Correa, Laura Mariana and Salehi, Rahil and Chowdhury, Niaz Bahar and Aryee, Randy and Calderon-Lopez, Brisa and Mali, Ankur and Saha, Rajib and Chowdhury, Ratul}, + author = {Sajeevan, Karuna Anna and Osinuga, Abraham and B, Arunraj and + Ferdous, Sakib and Shahreen, Nabia and Noor, Mohammed and Koneru, Shashank + and Santa-Correa, Laura Mariana and Salehi, Rahil and Chowdhury, Niaz Bahar + and Aryee, Randy and Calderon-Lopez, Brisa and Mali, Ankur and Saha, Rajib + and Chowdhury, Ratul}, year = {2025}, doi = {10.1101/2025.02.10.637555}, url = {https://doi.org/10.1101/2025.02.10.637555} @@ -145,8 +157,11 @@ url = {https://doi.org/10.1038/s41467-023-44113-1} }""", "IECata": r"""@article{Wang2025IECata, - title = {IECata: interpretable bilinear attention network and evidential deep learning improve the catalytic efficiency prediction of enzymes}, - author = {Wang, Jingjing and Zhao, Yanpeng and Yang, Zhijiang and Yao, Ge and Han, Penggang and Liu, Jiajia and Chen, Chang and Zan, Peng and Wan, Xiukun and Bo, Xiaochen and Jiang, Hui}, + title = {IECata: interpretable bilinear attention network and evidential deep + learning improve the catalytic efficiency prediction of enzymes}, + author = {Wang, Jingjing and Zhao, Yanpeng and Yang, Zhijiang and Yao, Ge and + Han, Penggang and Liu, Jiajia and Chen, Chang and Zan, Peng and Wan, Xiukun + and Bo, Xiaochen and Jiang, Hui}, journal = {Briefings in Bioinformatics}, volume = {26}, number = {3}, diff --git a/src/REvoDesign/tools/cgo_utils.py b/src/REvoDesign/tools/cgo_utils.py index 723392f8..4011dced 100644 --- a/src/REvoDesign/tools/cgo_utils.py +++ b/src/REvoDesign/tools/cgo_utils.py @@ -6,12 +6,16 @@ """ Utilities for pymol.cgo, partially modified from `pymol.cgobuilder` -This module is aimed at building programable application protocol for CGO creating via pymol.cgo module +This module is aimed at building programable application protocol for CGO +creating via pymol.cgo module # docs of cgo: -Lowercase names below should be replaced with floating-point numbers. Generally, the TRIANGLE primitive should only be used only as a last restore since it is much less effective to render than using a series of vertices with a BEGIN/END group. +Lowercase names below should be replaced with floating-point numbers. +Generally, the TRIANGLE primitive should only be used only as a last restore +since it is much less effective to render than using a series of vertices with a +BEGIN/END group. BEGIN, { POINTS | LINES | LINE_LOOP | LINE_STRIP | TRIANGLES | TRIANGLE_STRIP | TRIANGLE_FAN }, @@ -204,14 +208,16 @@ def as_vertex(self): def as_normal(self): """ Generate a CGO normal command for the point - This method inserts the point's coordinates into a CGO normal command, used for specifying normals in PyMOL. + This method inserts the point's coordinates into a CGO normal command, + used for specifying normals in PyMOL. """ return np.insert(cgo.NORMAL, 1, self.array) def move(self, x: float | None = None, y: float | None = None, z: float | None = None) -> "Point": """ Move the point - This method allows the point to be moved along the x, y, and z axes. If a coordinate is not provided, the original value is used. + This method allows the point to be moved along the x, y, and z axes. If + a coordinate is not provided, the original value is used. Parameters: - x: Optional[float] = None, the new x-coordinate, if not provided, the original x-coordinate is used @@ -227,7 +233,8 @@ def move(self, x: float | None = None, y: float | None = None, z: float | None = def as_arrays(points: Iterable["Point"]): """ Convert a collection of points to a numpy array - This static method converts a collection of Point objects into a single numpy array, facilitating batch processing. + This static method converts a collection of Point objects into a single + numpy array, facilitating batch processing. Parameters: - points: Iterable['Point'], a collection of Point objects @@ -241,7 +248,8 @@ def as_arrays(points: Iterable["Point"]): def as_vertexes(points: Iterable["Point"]): """ Convert a collection of points to CGO vertex commands - This static method converts a collection of Point objects into CGO vertex commands, used for batch rendering in PyMOL. + This static method converts a collection of Point objects into CGO vertex + commands, used for batch rendering in PyMOL. Parameters: - points: Iterable['Point'], a collection of Point objects @@ -779,7 +787,8 @@ class LineVertex(GraphicObject): """ Represents a line vertex, inheriting from GraphicObject. - This class is used to define a line drawing element, which can be a point or a Bezier curve, and can include line width and color attributes. + This class is used to define a line drawing element, which can be a point or + a Bezier curve, and can include line width and color attributes. Attributes: - point: A Point or Bezier instance, representing the starting point or control point of the line. @@ -795,7 +804,8 @@ def rebuild(self): """ Rebuilds the line vertex data. - This method initializes the internal data list, and rebuilds the line vertex data based on the width, color, and point type. + This method initializes the internal data list, and rebuilds the line + vertex data based on the width, color, and point type. """ # Initialize the data list self._data = [] @@ -843,7 +853,8 @@ def rebuild(self): """ Rebuilds the sphere's data representation using CGO (Chimera Graphics Object) format. - This method constructs the sphere's data by combining the color information and the sphere's geometric properties. + This method constructs the sphere's data by combining the color + information and the sphere's geometric properties. """ self._data = [ *Color(self.color).as_cgo, # Convert color to CGO format and unpack it into the data list @@ -1012,7 +1023,9 @@ class Cone(GraphicObject): radius_base: The radius of the cone at the base. color_tip: The color of the cone's tip. Default is 'w' for white. color_base: The color of the cone's base. Default is 'g' for green. - caps: A tuple indicating whether to add caps to the tip and/or base. 1 for True, 0 for False. Default is (1, 0), meaning the tip has a cap and the base does not. + caps: A tuple indicating whether to add caps to the tip and/or base. + 1 for True, 0 for False. Default is (1, 0), meaning the tip has a + cap and the base does not. """ tip: Point @@ -1251,7 +1264,8 @@ def rebuild(self): """ Rebuilds the square object's drawing data. - This method initializes a graphic object drawing data (_data) by defining the vertices and colors of the triangles that make up the square. + This method initializes a graphic object drawing data (_data) by defining + the vertices and colors of the triangles that make up the square. It uses the Color utility class to handle color conversion and the Point class for vertex coordinates. """ # Start drawing, defining the drawing mode as triangles @@ -1282,13 +1296,16 @@ class PolyLines(GraphicObject): """ Represents a collection of polylines, inheriting from GraphicObject. - This class is used to define a series of lines connected in a specific way, with common attributes such as width and color. + This class is used to define a series of lines connected in a specific way, + with common attributes such as width and color. Attributes: width (float): The line width. color (str): The line color, represented as a string. points (Iterable[LineVertex]): A collection of line vertices. - line_type (Literal['LINE_STRIP', 'LINE_LOOP', 'TRIANGLE_STRIP', 'TRIANGLE_FAN']): The drawing mode of the line, defaulting to 'LINE_STRIP'. + line_type (Literal['LINE_STRIP', 'LINE_LOOP', 'TRIANGLE_STRIP', + 'TRIANGLE_FAN']): The drawing mode of the line, defaulting to + 'LINE_STRIP'. """ # global @@ -1302,7 +1319,8 @@ def rebuild(self): """ Rebuilds the line data. - This method initializes the line drawing data, including setting the line width, color, and type, and updates the data for each vertex. + This method initializes the line drawing data, including setting the line + width, color, and type, and updates the data for each vertex. """ # Initialize the line drawing data, including line width and color self._data = [ @@ -1322,7 +1340,8 @@ def rebuild(self): @dataclass class Arrow(GraphicObject): """ - Represents an arrow object for visualization in PyMOL, with properties for start and end points, line width, and color. + Represents an arrow object for visualization in PyMOL, with properties for + start and end points, line width, and color. """ start: Point # the start point of the arrow @@ -2135,7 +2154,8 @@ def rebuild(self): """ Rebuilds the data for all graphic objects in the collection. - This method empties the existing graphic object data, then iterates through each graphic object in the collection. + This method empties the existing graphic object data, then iterates + through each graphic object in the collection. If the force_to_rebuild flag is set to True, it calls the rebuild method on each graphic object. Finally, it adds the data of each graphic object to the collection's _data list. """ diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index fdb10306..bbf5768b 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -1034,14 +1034,17 @@ class MenuItem: """ A data class representing a menu item. - This class is used to define the properties of a menu item, including its name, associated function, and optional arguments. - The use of the @dataclass decorator automatically generates special methods such as __init__(), __repr__(), and __eq__(). + This class is used to define the properties of a menu item, including its + name, associated function, and optional arguments. + The use of the @dataclass decorator automatically generates special methods + such as __init__(), __repr__(), and __eq__(). The frozen parameter ensures that instances of the class are immutable, enhancing thread safety and consistency. Attributes: name (str): The name of the menu item, used for display and identification. func (Callable): The function associated with the menu item, which is executed when the item is selected. - kwargs (Optional[Mapping]): Optional arguments passed to the associated function when it is executed. Defaults to None. + kwargs (Optional[Mapping]): Optional arguments passed to the associated + function when it is executed. Defaults to None. """ name: str @@ -1102,8 +1105,10 @@ def upgrade_check(original_file: str, new_file: str, title: str): """ Check and apply an upgrade if necessary. - This function compares the original file with a new fetched file, generates a diff file if there are differences, - and prompts the user to confirm whether to apply the upgrade. If the user confirms, the new file replaces the original file. + This function compares the original file with a new fetched file, + generates a diff file if there are differences, and prompts the user to + confirm whether to apply the upgrade. If the user confirms, the new file + replaces the original file. Parameters: - original_file (str): The path to the original file. @@ -1139,9 +1144,12 @@ def upgrade_check(original_file: str, new_file: str, title: str): ':::::Upgrade Summary:::::

' "" "" - f'' - f'' - f'' + '' + f'' + '' + f'' + '' + f'' "
Event-Affected Lines
Added :{num_added_lines}
Changed:{num_chged_lines}
Deleted:{num_deled_lines}
Added :{num_added_lines}
Changed:{num_chged_lines}
Deleted:{num_deled_lines}
" "You must check out these changes carefully.

" f"See all changes in this diff file of {title}.", @@ -1359,7 +1367,8 @@ def collect_diagnostic_data(collect_dummy: bool = False, drop_sensitives=True): if not drop_sensitives: confirmed = decide( title="Agree to collect SENSITIVE data?", - description="[!!!CAUSION!!!]Do you REALLY want to collect diagnostic information INCLUDING ALL SENSITIVE data?\n" + description="[!!!CAUSION!!!]Do you REALLY want to collect diagnostic " + "information INCLUDING ALL SENSITIVE data?\n" "Please DO NOT share this information with anyone else or post it to public channels.", ) if not confirmed: @@ -2667,16 +2676,14 @@ def callback(): @overload -def notify_box(message: str = "", error_type: None | type[Warning] = None, details: str | None = None) -> None: - ... +def notify_box(message: str = "", error_type: None | type[Warning] = None, details: str | None = None) -> None: ... # Overload #2: Exception => NoReturn @overload -def notify_box(message: str, error_type: type[Exception], details: str | None = None) -> NoReturn: - ... +def notify_box(message: str, error_type: type[Exception], details: str | None = None) -> NoReturn: ... def notify_box( @@ -2748,8 +2755,7 @@ def run_worker_thread_in_pool( trigger_buttons: QtWidgets.QPushButton | Iterable[QtWidgets.QPushButton] | None = None, notify_slot: Callable[[str], None] | None = None, **kwargs, -) -> R: - ... +) -> R: ... @overload @@ -2759,8 +2765,7 @@ def run_worker_thread_in_pool( trigger_buttons: QtWidgets.QPushButton | Iterable[QtWidgets.QPushButton] | None = None, notify_slot: Callable[[str], None] | None = None, **kwargs, -) -> R | None: - ... +) -> R | None: ... def run_worker_thread_in_pool( From 831340f09b78775f04d02c387de583b8cdeb8633 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:42:33 +0800 Subject: [PATCH 23/63] fix(deepsource): preserve expanded overload stubs --- src/REvoDesign/tools/package_manager.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index bbf5768b..f21af547 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -2675,15 +2675,19 @@ def callback(): # Overload #1: None or Warning => returns bool +# fmt: off @overload -def notify_box(message: str = "", error_type: None | type[Warning] = None, details: str | None = None) -> None: ... +def notify_box(message: str = "", error_type: None | type[Warning] = None, details: str | None = None) -> None: + ... # Overload #2: Exception => NoReturn @overload -def notify_box(message: str, error_type: type[Exception], details: str | None = None) -> NoReturn: ... +def notify_box(message: str, error_type: type[Exception], details: str | None = None) -> NoReturn: + ... +# fmt: on def notify_box( @@ -2748,6 +2752,7 @@ def raise_error(error_type: type[Exception], message: str) -> NoReturn: raise error_type(message) +# fmt: off @overload def run_worker_thread_in_pool( worker_function: Callable[..., R], @@ -2755,7 +2760,8 @@ def run_worker_thread_in_pool( trigger_buttons: QtWidgets.QPushButton | Iterable[QtWidgets.QPushButton] | None = None, notify_slot: Callable[[str], None] | None = None, **kwargs, -) -> R: ... +) -> R: + ... @overload @@ -2765,7 +2771,9 @@ def run_worker_thread_in_pool( trigger_buttons: QtWidgets.QPushButton | Iterable[QtWidgets.QPushButton] | None = None, notify_slot: Callable[[str], None] | None = None, **kwargs, -) -> R | None: ... +) -> R | None: + ... +# fmt: on def run_worker_thread_in_pool( From 52551c89958709d5999fbf8749ec67a0458c7b68 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:47:33 +0800 Subject: [PATCH 24/63] fix(deepsource): wrap runtime doc lines --- .../application/font/font_manager.py | 10 ++++--- src/REvoDesign/application/launching.py | 6 +++-- src/REvoDesign/basic/abc_singleton.py | 6 +++-- src/REvoDesign/basic/extensions.py | 9 ++++--- src/REvoDesign/basic/param_toggle.py | 12 ++++++--- src/REvoDesign/bootstrap/set_config.py | 14 ++++++---- src/REvoDesign/driver/ui_driver.py | 26 +++++++++++++------ src/REvoDesign/evaluate/evaluator.py | 9 ++++--- 8 files changed, 61 insertions(+), 31 deletions(-) diff --git a/src/REvoDesign/application/font/font_manager.py b/src/REvoDesign/application/font/font_manager.py index b9b7e393..dbda9377 100644 --- a/src/REvoDesign/application/font/font_manager.py +++ b/src/REvoDesign/application/font/font_manager.py @@ -98,10 +98,12 @@ def set_window_font(self, custom_font: QtGui.QFont | str | None = None): """ Set the window font based on the operating system type. - This method retrieves the current client operating system type and checks if it exists in the predefined font dictionary. - If the OS type is not found, the method returns without making any changes. If the OS type is found, it iterates through - the list of fonts associated with that OS type and sets the window font to the first available font family found in the - `self.font_families` list. + This method retrieves the current client operating system type and + checks if it exists in the predefined font dictionary. If the OS type is + not found, the method returns without making any changes. If the OS type + is found, it iterates through the list of fonts associated with that OS + type and sets the window font to the first available font family found + in the `self.font_families` list. Parameters: - None diff --git a/src/REvoDesign/application/launching.py b/src/REvoDesign/application/launching.py index 3d304502..499ba0d6 100644 --- a/src/REvoDesign/application/launching.py +++ b/src/REvoDesign/application/launching.py @@ -23,7 +23,8 @@ QLabel#labelStatus { color: #8b95a8; font-size: 13pt; } QLabel#labelDot { color: #4ecdc4; font-size: 10pt; } QLabel#labelInfoLeft, QLabel#labelInfoRight { color: #5a6578; font-size: 9pt; } - QProgressBar#progressBar { background-color: #232a3a; border: none; border-radius: 2px; min-height: 3px; max-height: 3px; } + QProgressBar#progressBar { background-color: #232a3a; border: none; border-radius: 2px; + min-height: 3px; max-height: 3px; } QProgressBar#progressBar::chunk { background-color: #4ecdc4; border-radius: 2px; } """ @@ -34,7 +35,8 @@ QLabel#labelStatus { color: #5a6578; font-size: 13pt; } QLabel#labelDot { color: #3db8b0; font-size: 10pt; } QLabel#labelInfoLeft, QLabel#labelInfoRight { color: #8b95a8; font-size: 9pt; } - QProgressBar#progressBar { background-color: #dfdcd6; border: none; border-radius: 2px; min-height: 3px; max-height: 3px; } + QProgressBar#progressBar { background-color: #dfdcd6; border: none; border-radius: 2px; + min-height: 3px; max-height: 3px; } QProgressBar#progressBar::chunk { background-color: #3db8b0; border-radius: 2px; } """ diff --git a/src/REvoDesign/basic/abc_singleton.py b/src/REvoDesign/basic/abc_singleton.py index 4066436b..c57407f1 100644 --- a/src/REvoDesign/basic/abc_singleton.py +++ b/src/REvoDesign/basic/abc_singleton.py @@ -260,8 +260,10 @@ def reset_singletons(): """ Reset singleton classes. - This function is used to gracefully reset all singleton classes. It iterates through all subclasses of SingletonAbstract, - and if the subclass has an '_instance' attribute and it is not None, it calls the reset_instance method of that class. + This function is used to gracefully reset all singleton classes. It iterates + through all subclasses of SingletonAbstract, and if the subclass has an + '_instance' attribute and it is not None, it calls the reset_instance method + of that class. Note: This function does not accept parameters and does not return any value. """ diff --git a/src/REvoDesign/basic/extensions.py b/src/REvoDesign/basic/extensions.py index 03455c7c..34a14763 100644 --- a/src/REvoDesign/basic/extensions.py +++ b/src/REvoDesign/basic/extensions.py @@ -72,7 +72,8 @@ def list_all(self) -> list[str]: """ Returns a list of all file extensions. - This property method iterates over the `self.extensions` list, extracting the 'ext' attribute from each extension object. + This property method iterates over the `self.extensions` list, + extracting the 'ext' attribute from each extension object. """ return [e.ext for e in self.extensions] @@ -189,10 +190,12 @@ def resolve_extension(extension: str) -> FileExtensionCollection: user-friendly names with a prefix `'Customized - '`. Args: - extension (str): The extension string to be resolved. It can be a predefined name or a custom list like `"pdb;csv"`. + extension (str): The extension string to be resolved. It can be a + predefined name or a custom list like `"pdb;csv"`. Returns: - Fext.ExtColl: An object representing the file extension collection, either from predefined values or custom input. + Fext.ExtColl: An object representing the file extension collection, + either from predefined values or custom input. Example: Given input `"pdb;csv"`, this will generate: diff --git a/src/REvoDesign/basic/param_toggle.py b/src/REvoDesign/basic/param_toggle.py index 082f0993..524cf729 100644 --- a/src/REvoDesign/basic/param_toggle.py +++ b/src/REvoDesign/basic/param_toggle.py @@ -22,8 +22,10 @@ class ParamChangeRegistryItem: """ A class that registers a mapping between two config items. - This class is used to define how a change in one configuration item (source_cfg_item) should be mapped and registered - to another configuration item (target_cfg_item) through a UI widget signal. It uses a dictionary to map parameter values. + This class is used to define how a change in one configuration item + (source_cfg_item) should be mapped and registered to another + configuration item (target_cfg_item) through a UI widget signal. It uses + a dictionary to map parameter values. Attributes: widget_name (str): The name of the UI widget. @@ -64,10 +66,12 @@ def widget_signal(self, ui) -> QtCore.pyqtBoundSignal: def register(self, register_func: Callable[[str, str, dict[str, tuple]], None], ui: Any): """ - Registers the mapping between two configuration items by connecting the widget signal to the registration function. + Registers the mapping between two configuration items by connecting the + widget signal to the registration function. Args: - register_func (Callable[[str, str, Dict[str, Tuple]], None]): The function to be called when the signal is emitted. + register_func (Callable[[str, str, Dict[str, Tuple]], None]): The function to be + called when the signal is emitted. ui (Any): The UI instance containing the widgets. """ event = self.widget_signal(ui=ui) diff --git a/src/REvoDesign/bootstrap/set_config.py b/src/REvoDesign/bootstrap/set_config.py index 6ef50084..8d87a2d9 100644 --- a/src/REvoDesign/bootstrap/set_config.py +++ b/src/REvoDesign/bootstrap/set_config.py @@ -117,10 +117,12 @@ def reload_config_file( config_name: str = "main", overrides: list[str] | None = None, return_hydra_config: bool = False ) -> DictConfig: """ - Reload a configuration yaml file in a Hydra manner. As we initialize hydra w/ initialize_config_dir, - which is the REVODESIGN_CONFIG_DIR at user's data directory, the config_name is supposed as a relative path of the yaml file. + Reload a configuration yaml file in a Hydra manner. As we initialize hydra + w/ initialize_config_dir, which is the REVODESIGN_CONFIG_DIR at user's data + directory, the config_name is supposed as a relative path of the yaml file. e.g. config_name="experiments/my_experiment" refers to experiments/my_experiment.yaml - the DictConfig object can be accessed like `reload_config_file(config_name="experiments/my_experiment")["experiments"]` + the DictConfig object can be accessed like + `reload_config_file(config_name="experiments/my_experiment")["experiments"]` Arguments: config_name (str): The name of the configuration file. Defaults to "main". @@ -276,13 +278,15 @@ def list_all_config_files(config_dir: str, tree: bool = False) -> list[str]: def _iter_yaml_rel_paths(base_dir: str) -> list[str]: """ - Iterates through the specified directory and its subdirectories to collect relative paths of all .yaml files + Iterates through the specified directory and its subdirectories to collect + relative paths of all .yaml files Args: base_dir (str): The base directory path to traverse Returns: - list[str]: A list containing relative paths of all found .yaml files with respect to base_dir, sorted in lexicographical order + list[str]: A list containing relative paths of all found .yaml files + with respect to base_dir, sorted in lexicographical order """ base_dir = os.path.abspath(base_dir) rel_paths: list[str] = [] diff --git a/src/REvoDesign/driver/ui_driver.py b/src/REvoDesign/driver/ui_driver.py index 71f6d45f..110252bc 100644 --- a/src/REvoDesign/driver/ui_driver.py +++ b/src/REvoDesign/driver/ui_driver.py @@ -97,7 +97,8 @@ class HeadlessProtocol(Protocol): @dataclass class Config: """ - A dataclass to represent a configuration file. It contains the name, path, and configuration data of a configuration file. + A dataclass to represent a configuration file. It contains the name, path, + and configuration data of a configuration file. Attributes: name: str -- The name of the configuration file. @@ -344,22 +345,29 @@ class ConfigBus(SingletonAbstract, CitableModuleAbstract): Methods: Non-headless Methods: - initialize_widget_with_cfg_group(): Initializes UI widgets with their corresponding configuration settings. - update_cfg_item_from_widget(widget_id: str): Updates a configuration setting based on the value of a UI widget. - register_widget_changes_to_cfg(): Registers UI widget changes to update the configuration settings. + initialize_widget_with_cfg_group(): Initializes UI widgets with their corresponding + configuration settings. + update_cfg_item_from_widget(widget_id: str): Updates a configuration setting based on + the value of a UI widget. + register_widget_changes_to_cfg(): Registers UI widget changes to update the + configuration settings. get_widget_from_id(widget_id: str): Retrieves a UI widget based on its ID. - get_widget_from_cfg_item(cfg_item: str): Retrieves a UI widget based on its corresponding configuration item. + get_widget_from_cfg_item(cfg_item: str): Retrieves a UI widget based on its + corresponding configuration item. get_widget_value(cfg_item: str): Retrieves the value of a UI widget based on its corresponding configuration item. set_widget_value(cfg_item: str, value): Sets the value of a UI widget based on its corresponding configuration item. - restore_widget_value(cfg_item: str): Restores the value of a UI widget to its default configuration setting. + restore_widget_value(cfg_item: str): Restores the value of a UI widget to its default + configuration setting. get_cfg_item(widget_id: str): Retrieves the configuration item corresponding to a UI widget ID. button(id: str): Retrieves a button widget based on its ID. - toggle_buttons(buttons: Iterable, set_enabled: bool = False): Toggles the enabled state of a list of buttons. + toggle_buttons(buttons: Iterable, set_enabled: bool = False): Toggles the enabled + state of a list of buttons. Headless Only Methods: - get_value(cfg_item: str, typing=None): Retrieves the value of a configuration item, with optional type casting. + get_value(cfg_item: str, typing=None): Retrieves the value of a configuration item, + with optional type casting. set_value(cfg_item: str, value): Sets the value of a configuration item. fp_lock(cfg_fps: Union[list, tuple, str], buttons_id_to_release: Union[list, tuple, str]): Locks or unlocks @@ -503,6 +511,7 @@ def get_cfg_item(self, widget_id: str) -> str: raise ValueError(f"{widget_id} is not a valid widget ID.") return cfg_item + # fmt: off @overload def get_value( self, @@ -539,6 +548,7 @@ def get_value( @overload def get_value(self, cfg_item: str, converter: None) -> Any: ... + # fmt: on def get_value( self, diff --git a/src/REvoDesign/evaluate/evaluator.py b/src/REvoDesign/evaluate/evaluator.py index c6f2aec3..4ca9030a 100644 --- a/src/REvoDesign/evaluate/evaluator.py +++ b/src/REvoDesign/evaluate/evaluator.py @@ -99,7 +99,8 @@ def mutant_decision(self, decision_to_accept: bool): return logging.debug( - f'{"Accepting" if decision_to_accept else "Rejecting"} mutant {self.mutant_tree_candidates.current_mutant_id}' + f'{"Accepting" if decision_to_accept else "Rejecting"} mutant ' + f"{self.mutant_tree_candidates.current_mutant_id}" ) if decision_to_accept: @@ -271,8 +272,10 @@ def find_all_best_mutants(self): # Ask whether to overide confirmed = decide( title="Override existed mutant table choices?", - description="You currently have existed mutant table choices, which shall be overriden by using `I'm lucky`. \n \ - Are you really sure? ", + description=( + "You currently have existed mutant table choices, which shall be overriden " + "by using `I'm lucky`.\nAre you really sure? " + ), ) if not confirmed: From 9f63f34d47e3027f0592669c99b08adfa25af08c Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:53:03 +0800 Subject: [PATCH 25/63] fix(deepsource): wrap utility long lines --- src/REvoDesign/citations/citation_manager.py | 7 +++++-- src/REvoDesign/clusters/cluster_sequence.py | 4 +++- .../common/multi_mutant_designer.py | 4 +++- src/REvoDesign/common/mutant_tree.py | 5 ++++- .../driver/param_toggle_register.py | 3 ++- .../designers/openkinetics/_client.py | 20 +++++++++++++------ src/REvoDesign/shortcuts/__init__.py | 10 +++++++--- src/REvoDesign/shortcuts/function_utils.py | 11 ++++++++-- .../tools/mutation_effect_predictors.py | 5 +++-- src/REvoDesign/structure/PocketSearcher.py | 8 +++++--- src/REvoDesign/tools/download_registry.py | 16 ++++++++++----- src/REvoDesign/tools/mutant_tools.py | 3 ++- 12 files changed, 68 insertions(+), 28 deletions(-) diff --git a/src/REvoDesign/citations/citation_manager.py b/src/REvoDesign/citations/citation_manager.py index 68f06e92..99855f0f 100644 --- a/src/REvoDesign/citations/citation_manager.py +++ b/src/REvoDesign/citations/citation_manager.py @@ -130,7 +130,9 @@ def notice(cls): """ Display the citation notice. - This method checks if there are citations to be displayed, and if the current module's citations have not been silenced, it logs the citation information. + This method checks if there are citations to be displayed, and if the + current module's citations have not been silenced, it logs the citation + information. """ # If the __bibtex__ dictionary is empty, log a debug message and return. if not cls.__bibtex__: @@ -171,7 +173,8 @@ def cite(cls): """ Add citation to the citation manager. - This method adds the current module's citation information to the citation manager and then displays the citation notice. + This method adds the current module's citation information to the + citation manager and then displays the citation notice. """ citations = cls.__bibtex__ # Check if the citation information is a dictionary. diff --git a/src/REvoDesign/clusters/cluster_sequence.py b/src/REvoDesign/clusters/cluster_sequence.py index 5859c48e..bf1d5053 100644 --- a/src/REvoDesign/clusters/cluster_sequence.py +++ b/src/REvoDesign/clusters/cluster_sequence.py @@ -406,7 +406,9 @@ def build_method_report(self) -> dict[str, object]: "deprecated": spec.deprecated, "representative_selection_policy": spec.representative_policy, "compatibility_outputs": { - "cluster_centers_stochastic.fasta": "Deprecated compatibility alias of nearest-centroid representatives." + "cluster_centers_stochastic.fasta": ( + "Deprecated compatibility alias of nearest-centroid representatives." + ) }, } diff --git a/src/REvoDesign/common/multi_mutant_designer.py b/src/REvoDesign/common/multi_mutant_designer.py index 40283783..f79a1714 100644 --- a/src/REvoDesign/common/multi_mutant_designer.py +++ b/src/REvoDesign/common/multi_mutant_designer.py @@ -99,7 +99,9 @@ def get_input_and_initialize(self): self.design_case = cmd.get_unused_name("multi_design") logging.info( - f"Mutant Tree for multi-design is initialized. {len(self.design_pool_tree.all_mutant_branch_ids)} groups with {len(self.design_pool_tree.all_mutant_ids)} mutants." + "Mutant Tree for multi-design is initialized. " + f"{len(self.design_pool_tree.all_mutant_branch_ids)} groups with " + f"{len(self.design_pool_tree.all_mutant_ids)} mutants." ) @staticmethod diff --git a/src/REvoDesign/common/mutant_tree.py b/src/REvoDesign/common/mutant_tree.py index 54358903..ce000ed4 100644 --- a/src/REvoDesign/common/mutant_tree.py +++ b/src/REvoDesign/common/mutant_tree.py @@ -24,6 +24,7 @@ class MutantDict(TypedDict): class MutateRunner(Protocol): + # fmt: off def run_mutate_parallel(self, mutants: list[Mutant], nproc: int = 2) -> list[str]: ... @@ -32,6 +33,7 @@ def mutated_pdb_mapping(self, mutant_tree: "MutantTree", pdb_fps: list[str]) -> def cite(self) -> None: ... + # fmt: on class MutantTree: @@ -156,7 +158,8 @@ def has(self, obj: str | Mutant) -> bool: """ Check if the given object exists in the instance. - This function accepts an object of type string or Mutant and determines whether the object is present in the instance's mutant collection. + This function accepts an object of type string or Mutant and determines + whether the object is present in the instance's mutant collection. Parameters: - obj (Union[str, Mutant]): The object to check. It can be a string representing a mutant ID or a Mutant object. diff --git a/src/REvoDesign/driver/param_toggle_register.py b/src/REvoDesign/driver/param_toggle_register.py index e83cb9cb..7986ac96 100644 --- a/src/REvoDesign/driver/param_toggle_register.py +++ b/src/REvoDesign/driver/param_toggle_register.py @@ -37,7 +37,8 @@ class ParserOrDesigner(Protocol): """ Protocol class to define the structure for a parser or designer type. - This class inherits from Protocol and specifies the attributes that any instance conforming to this protocol must have. + This class inherits from Protocol and specifies the attributes that any + instance conforming to this protocol must have. Attributes: name (str): The name of the parser or designer. diff --git a/src/REvoDesign/magician/designers/openkinetics/_client.py b/src/REvoDesign/magician/designers/openkinetics/_client.py index 43a6340d..f8952985 100644 --- a/src/REvoDesign/magician/designers/openkinetics/_client.py +++ b/src/REvoDesign/magician/designers/openkinetics/_client.py @@ -737,7 +737,8 @@ def submit( Args: self: The instance of the class containing this method - rows: A list of dictionaries, each containing 'Protein Sequence' and 'Substrate' keys with corresponding values + rows: A list of dictionaries, each containing 'Protein Sequence' and + 'Substrate' keys with corresponding values method: The prediction method to be used for the analysis prediction_type: The type of prediction to perform handle_long_sequences: Strategy for handling long sequences, defaults to "truncate" @@ -749,7 +750,8 @@ def submit( A dictionary containing the response from the OpenKinetics service, including job identifier Raises: - OpenKineticsValidationError: If no rows are provided or if the submit response doesn't contain a job identifier + OpenKineticsValidationError: If no rows are provided or if the + submit response doesn't contain a job identifier """ if not rows: raise OpenKineticsValidationError("At least one row is required for submission") @@ -786,7 +788,10 @@ def submit( logging.info("OpenKinetics service health: %s", self.check_health()) submit_response = self._request( - "POST", OPENKINETICS_ENDPOINTS["submit"], json_payload=payload, timeout=self.timeout_seconds + "POST", + OPENKINETICS_ENDPOINTS["submit"], + json_payload=payload, + timeout=self.timeout_seconds, ) # Extract and validate the job ID from the response @@ -813,11 +818,14 @@ def get_result( Args: job_id: The unique identifier of the job to retrieve results for result_format: Format of the result, either 'json' or 'csv' (default is 'json') - poll_interval_seconds: Interval in seconds between polling attempts when result is not ready (default is 3) - timeout_seconds: Maximum time to wait for the result in seconds, None means default timeout (default is None) + poll_interval_seconds: Interval in seconds between polling attempts + when result is not ready (default is 3) + timeout_seconds: Maximum time to wait for the result in seconds, + None means default timeout (default is None) Returns: - A dictionary containing the result data if format is JSON, or a string containing CSV data if format is CSV + A dictionary containing the result data if format is JSON, or a + string containing CSV data if format is CSV Raises: OpenKineticsAPIError: If the API request fails or returns an error status diff --git a/src/REvoDesign/shortcuts/__init__.py b/src/REvoDesign/shortcuts/__init__.py index 9811de88..31acbed2 100644 --- a/src/REvoDesign/shortcuts/__init__.py +++ b/src/REvoDesign/shortcuts/__init__.py @@ -25,8 +25,11 @@ # How to set up autocompletion for a extend command with multiple arguments? # The general format is: -# `cmd.auto_arg[][""]=[, '', '']` where -# is the argument index (starting from 0) that we want to set autocompletion for in the extend command +# `cmd.auto_arg[][""]=[ +# , '', '' +# ]` where +# is the argument index (starting from 0) that we want to set +# autocompletion for in the extend command # is the name of the extend command # is a lambda function that returns a Shortcut object # is a string label for the argument that will be displayed in the autocomplete output, @@ -40,7 +43,8 @@ # 0 1 # '' is a string that will be appended to the argument value -# Normally, to support autocomplete dynamically during runtime, PyMOL uses a lambda function that returns a Shortcut object +# Normally, to support autocomplete dynamically during runtime, PyMOL uses a +# lambda function that returns a Shortcut object # A shortcut object defines the possible keywords for autocompletion of arguments. # One can either bollow the existing Shortcut lambda from `cmd.auto_arg` or create a new one # Here we borrow the existing ones for 'show' and 'select' diff --git a/src/REvoDesign/shortcuts/function_utils.py b/src/REvoDesign/shortcuts/function_utils.py index 9ffca794..47c96095 100644 --- a/src/REvoDesign/shortcuts/function_utils.py +++ b/src/REvoDesign/shortcuts/function_utils.py @@ -9,6 +9,7 @@ import os import shutil + # Used for launching a user-requested PyMOL preview with an argv list. import subprocess # nosec B404 from typing import Literal @@ -65,7 +66,12 @@ def visualize_conformer_sdf(sdf_file_path: str, show_conformer: Literal["New Win # Explicitly call a new PyMOL instance in the background pymol_command = [_resolve_pymol_executable(), "-xi", pml_file_path] # Launches a user-requested PyMOL preview with an argv list and a resolved executable. - subprocess.Popen(pymol_command, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # nosec B603 + subprocess.Popen( # nosec B603 + pymol_command, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) logging.info("PyMOL launched in the background with %s.", sdf_file_path) @@ -75,7 +81,8 @@ def smiles_conformer_batch(smi: dict[str, str], num_conformer: int, save_dir: st Generates 3D conformers for a SMILES string using RDKit. Args: - smi (Dict[str, str]): Dictionary containing the name of the molecule as the key and the SMILES string as the value. + smi (Dict[str, str]): Dictionary containing the name of the molecule as + the key and the SMILES string as the value. num_conformer (int): Number of conformers to generate for each molecule. save_dir (str): Directory to save the generated conformer files. n_jobs (int, optional): Number of parallel jobs to run. Defaults to 1. diff --git a/src/REvoDesign/shortcuts/tools/mutation_effect_predictors.py b/src/REvoDesign/shortcuts/tools/mutation_effect_predictors.py index 49d4fbb1..8c5bcc91 100644 --- a/src/REvoDesign/shortcuts/tools/mutation_effect_predictors.py +++ b/src/REvoDesign/shortcuts/tools/mutation_effect_predictors.py @@ -29,8 +29,9 @@ @require_installed class ThermoMpnnPredictor(ThirdPartyModuleAbstract, TorchModuleAbstract): """ - ThermoMpnnPredictor class for predicting the thermodynamic stability effects of protein mutations. - It utilizes the ThermoMPNN model to analyze protein structure and sequence to predict stability changes due to mutations. + ThermoMpnnPredictor class for predicting the thermodynamic stability effects + of protein mutations. It utilizes the ThermoMPNN model to analyze protein + structure and sequence to predict stability changes due to mutations. Attributes: name (str): Name of the predictor, set to 'ThermoMPNN'. diff --git a/src/REvoDesign/structure/PocketSearcher.py b/src/REvoDesign/structure/PocketSearcher.py index 92e573d1..b11f24bd 100644 --- a/src/REvoDesign/structure/PocketSearcher.py +++ b/src/REvoDesign/structure/PocketSearcher.py @@ -79,7 +79,8 @@ def search_pockets(self): ) cmd.select( substrate_pocket_id, - f"({self.molecule} and c. {self.chain_id}) and (byres ({ligand_sele}) around {self.ligand_radius}) and polymer.protein", + f"({self.molecule} and c. {self.chain_id}) and " + f"(byres ({ligand_sele}) around {self.ligand_radius}) and polymer.protein", ) cmd.select( design_shell_id, @@ -94,13 +95,14 @@ def search_pockets(self): if self.cofactor and self.cofactor_radius > 0: cofact_label, cofact_sele = self.process_multiple_resn(self.cofactor) - logging.debug(f"cofactor info {self.cofactor} ({cofact_label}: {cofact_sele}): {self.cofactor_radius}") + logging.debug(f"cofactor info {self.cofactor} ({cofact_label}: {cofact_sele}): " f"{self.cofactor_radius}") cofactor_pocket_id = cmd.get_unused_name(f"pkt_cof_{cofact_label}_{self.cofactor_radius}_") logging.info(f"Setting cofactor {self.cofactor}: {cofactor_pocket_id}") cmd.select( cofactor_pocket_id, - f"({self.molecule} and c. {self.chain_id}) and (byres ({cofact_sele}) around {self.cofactor_radius}) and polymer.protein", + f"({self.molecule} and c. {self.chain_id}) and " + f"(byres ({cofact_sele}) around {self.cofactor_radius}) and polymer.protein", ) cmd.select( design_shell_id, diff --git a/src/REvoDesign/tools/download_registry.py b/src/REvoDesign/tools/download_registry.py index d99f94e2..93837583 100644 --- a/src/REvoDesign/tools/download_registry.py +++ b/src/REvoDesign/tools/download_registry.py @@ -85,14 +85,18 @@ class FileDownloadRegistry(CitableModuleAbstract): """ A file download registry manager for handling remote file resources. - This class implements automatic file downloading, verification, and cache management based on the pooch library. - It supports specifying file hash values through a registry and provides convenient methods to fetch and verify remote files. + This class implements automatic file downloading, verification, and cache + management based on the pooch library. It supports specifying file hash + values through a registry and provides convenient methods to fetch and + verify remote files. :param name: Module name, used to construct the default data directory path. :param base_url: Base URL for remote files. - :param registry: File registry where keys are filenames and values are corresponding hash values (optional). + :param registry: File registry where keys are filenames and values are + corresponding hash values (optional). :param version: Optional version number for creating versioned data directories. - :param customized_directory: Optional custom download directory path. If not provided, uses the default user data directory. + :param customized_directory: Optional custom download directory path. If not + provided, uses the default user data directory. :param alternative_base_urls: Optional list of alternative base URLs for downloading files. :param retry_count: Number of retries for downloading files. @@ -252,7 +256,9 @@ def prepare_registry_from_md5(md5_contents: str) -> dict[str, str | None]: "Pooch": """ @article{uieda2020, title = {{Pooch}: {A} friend to fetch your data files}, - author = {Leonardo Uieda and Santiago Soler and R{\'{e}}mi Rampin and Hugo van Kemenade and Matthew Turk and Daniel Shapero and Anderson Banihirwe and John Leeman}, + author = {Leonardo Uieda and Santiago Soler and R{\'{e}}mi Rampin and Hugo + van Kemenade and Matthew Turk and Daniel Shapero and Anderson Banihirwe and + John Leeman}, year = {2020}, doi = {10.21105/joss.01943}, url = {https://doi.org/10.21105/joss.01943}, diff --git a/src/REvoDesign/tools/mutant_tools.py b/src/REvoDesign/tools/mutant_tools.py index e17bc1e5..95c635f7 100644 --- a/src/REvoDesign/tools/mutant_tools.py +++ b/src/REvoDesign/tools/mutant_tools.py @@ -82,7 +82,8 @@ def extract_mutants_from_mutant_id( mutant_string (str): Underscore-seperated mutant string that contains the mutations and score (if possible). _...._ sequences (dict): Wild-type chain: sequence of design molecule - wt_before_chain (bool): Some people write WT residue before chain_id. This helps to correct recognizing of this pattern + wt_before_chain (bool): Some people write WT residue before chain_id. This + helps to correct recognizing of this pattern Returns: tuple: From c01e01bf09430d35cb7d83fae36b8efd0fbe7ccc Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:56:53 +0800 Subject: [PATCH 26/63] fix(deepsource): wrap tools doc lines --- src/REvoDesign/tools/customized_widgets.py | 5 ++++- src/REvoDesign/tools/pymol_utils.py | 11 ++++++++--- src/REvoDesign/tools/ssl_certificates.py | 9 ++++++--- src/REvoDesign/tools/system_tools.py | 3 ++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/REvoDesign/tools/customized_widgets.py b/src/REvoDesign/tools/customized_widgets.py index 7a529e9a..33dd4e5d 100644 --- a/src/REvoDesign/tools/customized_widgets.py +++ b/src/REvoDesign/tools/customized_widgets.py @@ -142,7 +142,8 @@ def show(self): def check_repeat(self): """ - Checks if a window with the same name is already open. If found, raises it to the front and raises a RuntimeError. + Checks if a window with the same name is already open. If found, raises + it to the front and raises a RuntimeError. """ from REvoDesign.driver.ui_driver import ConfigBus @@ -1360,6 +1361,7 @@ def getOpenFileNameWithExt(*args, **kwargs): return fname +# fmt: off @overload def set_widget_value(widget: QtWidgets.QFontComboBox, value: QtGui.QFont | str): ... @@ -1400,6 +1402,7 @@ def set_widget_value(widget: QtWidgets.QGridLayout, value: str): @overload def set_widget_value(widget: QtWidgets.QLineEdit | QtWidgets.QLCDNumber | QtWidgets.QCheckBox, value: Any): ... +# fmt: on def set_widget_value(widget, value): diff --git a/src/REvoDesign/tools/pymol_utils.py b/src/REvoDesign/tools/pymol_utils.py index 4684c38e..7216707e 100644 --- a/src/REvoDesign/tools/pymol_utils.py +++ b/src/REvoDesign/tools/pymol_utils.py @@ -166,7 +166,8 @@ def is_distal_residue_pair( - chain_id (str): The chain identifier. - resi_1 (int): The residue number of the first amino acid. - resi_2 (int): The residue number of the second amino acid. - - minimal_distance (float, optional): The minimum distance threshold for residues to be considered distal. Default is 20. + - minimal_distance (float, optional): The minimum distance threshold for + residues to be considered distal. Default is 20. - use_sidechain_angle (bool, optional): Whether to consider the orientation of side chains. Default is False. Returns: @@ -348,7 +349,8 @@ def refresh_all_selections(): Function: refresh_all_selections Usage: selections = refresh_all_selections() - This function refreshes and logs information about all PyMOL selections except 'sele' and those starting with '_align'. + This function refreshes and logs information about all PyMOL selections + except 'sele' and those starting with '_align'. Returns: - list: List of all non-'sele' selections (excluding those starting with '_align') @@ -404,7 +406,10 @@ def make_temperal_input_pdb( ): """ Function: make_temperal_input_pdb - Usage: input_file = make_temperal_input_pdb(molecule, chain_id=None, segment_id=None, format='pdb', wd=os.getcwd(), reload=True) + Usage: input_file = make_temperal_input_pdb( + molecule, chain_id=None, segment_id=None, format='pdb', + wd=os.getcwd(), reload=True + ) exi This function generates a temporary input PDB file from the specified molecule selection. It supports selection by chain ID, segment ID, or both. diff --git a/src/REvoDesign/tools/ssl_certificates.py b/src/REvoDesign/tools/ssl_certificates.py index f8d75f27..67c4444c 100644 --- a/src/REvoDesign/tools/ssl_certificates.py +++ b/src/REvoDesign/tools/ssl_certificates.py @@ -4,7 +4,8 @@ """ -This module contains functions and classes related to managing SSL certificates and generating unique identifiers (UUIDs). +This module contains functions and classes related to managing SSL certificates +and generating unique identifiers (UUIDs). """ import datetime @@ -69,7 +70,8 @@ def get_certificate(self): Function: get_certificate Usage: get_certificate() - This function checks for the existence of an SSL certificate and generates a new one if it doesn't exist or has expired. + This function checks for the existence of an SSL certificate and + generates a new one if it doesn't exist or has expired. Returns: @@ -100,7 +102,8 @@ def create_new_certificate(self): Function: create_new_certificate Usage: create_new_certificate() - This function creates a new SSL certificate and private key if they do not exist or if the certificate has expired. + This function creates a new SSL certificate and private key if they do + not exist or if the certificate has expired. Returns: - None diff --git a/src/REvoDesign/tools/system_tools.py b/src/REvoDesign/tools/system_tools.py index 41c98996..642c5147 100644 --- a/src/REvoDesign/tools/system_tools.py +++ b/src/REvoDesign/tools/system_tools.py @@ -24,7 +24,8 @@ def check_mac_rosetta2(): This function first checks if the current operating system is macOS. If not, it returns immediately. It then determines if the machine is an ARM-based Mac and whether it is recognized as an x86_64 architecture. - If both conditions are met, a warning is issued indicating that the environment is using Rosetta 2, which may impact performance. + If both conditions are met, a warning is issued indicating that the + environment is using Rosetta 2, which may impact performance. Returns: None From cb52d7a2f3c9b6cf5371746e0f09e2e4eaf37e13 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:02:24 +0800 Subject: [PATCH 27/63] fix(deepsource): wrap shortcut command strings --- src/REvoDesign/shortcuts/tools/esm2.py | 14 ++++-- .../shortcuts/tools/openmm_utils.py | 12 ++++- src/REvoDesign/shortcuts/tools/vina_tools.py | 44 ++++++++++++++----- src/REvoDesign/shortcuts/utils.py | 3 +- 4 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/REvoDesign/shortcuts/tools/esm2.py b/src/REvoDesign/shortcuts/tools/esm2.py index da5139f9..39983a53 100644 --- a/src/REvoDesign/shortcuts/tools/esm2.py +++ b/src/REvoDesign/shortcuts/tools/esm2.py @@ -351,7 +351,9 @@ def compute_pppl(self, row, sequence, model, alphabet, offset_idx): __bibtex__ = { "ESM": """@article{rives2019biological, - author={Rives, Alexander and Meier, Joshua and Sercu, Tom and Goyal, Siddharth and Lin, Zeming and Liu, Jason and Guo, Demi and Ott, Myle and Zitnick, C. Lawrence and Ma, Jerry and Fergus, Rob}, + author={Rives, Alexander and Meier, Joshua and Sercu, Tom and Goyal, + Siddharth and Lin, Zeming and Liu, Jason and Guo, Demi and Ott, Myle and + Zitnick, C. Lawrence and Ma, Jerry and Fergus, Rob}, title={Biological Structure and Function Emerge from Scaling Unsupervised Learning to 250 Million Protein Sequences}, year={2019}, doi={10.1101/622803}, @@ -360,14 +362,16 @@ def compute_pppl(self, row, sequence, model, alphabet, offset_idx): }""", "ESM-1v": """@article{meier2021language, author = {Meier, Joshua and Rao, Roshan and Verkuil, Robert and Liu, Jason and Sercu, Tom and Rives, Alexander}, - title = {Language models enable zero-shot prediction of the effects of mutations on protein function}, + title = {Language models enable zero-shot prediction of the effects of + mutations on protein function}, year={2021}, doi={10.1101/2021.07.09.450648}, url={https://www.biorxiv.org/content/10.1101/2021.07.09.450648v1}, journal={bioRxiv} }""", "MSA Transformer": """@article{rao2021msa, - author = {Rao, Roshan and Liu, Jason and Verkuil, Robert and Meier, Joshua and Canny, John F. and Abbeel, Pieter and Sercu, Tom and Rives, Alexander}, + author = {Rao, Roshan and Liu, Jason and Verkuil, Robert and Meier, Joshua + and Canny, John F. and Abbeel, Pieter and Sercu, Tom and Rives, Alexander}, title={MSA Transformer}, year={2021}, doi={10.1101/2021.02.12.430858}, @@ -376,7 +380,9 @@ def compute_pppl(self, row, sequence, model, alphabet, offset_idx): }""", "ESM-2": """@article{lin2022language, title={Language models of protein sequences at the scale of evolution enable accurate structure prediction}, - author={Lin, Zeming and Akin, Halil and Rao, Roshan and Hie, Brian and Zhu, Zhongkai and Lu, Wenting and Smetanin, Nikita and dos Santos Costa, Allan and Fazel-Zarandi, Maryam and Sercu, Tom and Candido, Sal and others}, + author={Lin, Zeming and Akin, Halil and Rao, Roshan and Hie, Brian and Zhu, + Zhongkai and Lu, Wenting and Smetanin, Nikita and dos Santos Costa, Allan and + Fazel-Zarandi, Maryam and Sercu, Tom and Candido, Sal and others}, journal={bioRxiv}, year={2022}, publisher={Cold Spring Harbor Laboratory} diff --git a/src/REvoDesign/shortcuts/tools/openmm_utils.py b/src/REvoDesign/shortcuts/tools/openmm_utils.py index f76cd6ce..e7db4b1a 100644 --- a/src/REvoDesign/shortcuts/tools/openmm_utils.py +++ b/src/REvoDesign/shortcuts/tools/openmm_utils.py @@ -53,7 +53,9 @@ def start_server(self): """ Start the OpenMMSetup server. - This method extends the start_server method of the parent class, configures and starts the Uvicorn server in a separate thread, and attempts to open the server URL in the default browser. + This method extends the start_server method of the parent class, + configures and starts the Uvicorn server in a separate thread, and + attempts to open the server URL in the default browser. """ """ Behavior of the server start action. @@ -104,7 +106,13 @@ def start_server(self): __bibtex__ = { "OpenMM": """@article{doi:10.1021/acs.jpcb.3c06662, -author = {Eastman, Peter and Galvelis, Raimondas and Peláez, Raúl P. and Abreu, Charlles R. A. and Farr, Stephen E. and Gallicchio, Emilio and Gorenko, Anton and Henry, Michael M. and Hu, Frank and Huang, Jing and Krämer, Andreas and Michel, Julien and Mitchell, Joshua A. and Pande, Vijay S. and Rodrigues, João PGLM and Rodriguez-Guerra, Jaime and Simmonett, Andrew C. and Singh, Sukrit and Swails, Jason and Turner, Philip and Wang, Yuanqing and Zhang, Ivy and Chodera, John D. and De Fabritiis, Gianni and Markland, Thomas E.}, +author = {Eastman, Peter and Galvelis, Raimondas and Peláez, Raúl P. and +Abreu, Charlles R. A. and Farr, Stephen E. and Gallicchio, Emilio and Gorenko, +Anton and Henry, Michael M. and Hu, Frank and Huang, Jing and Krämer, Andreas +and Michel, Julien and Mitchell, Joshua A. and Pande, Vijay S. and Rodrigues, +João PGLM and Rodriguez-Guerra, Jaime and Simmonett, Andrew C. and Singh, +Sukrit and Swails, Jason and Turner, Philip and Wang, Yuanqing and Zhang, Ivy +and Chodera, John D. and De Fabritiis, Gianni and Markland, Thomas E.}, title = {OpenMM 8: Molecular Dynamics Simulation with Machine Learning Potentials}, journal = {The Journal of Physical Chemistry B}, volume = {128}, diff --git a/src/REvoDesign/shortcuts/tools/vina_tools.py b/src/REvoDesign/shortcuts/tools/vina_tools.py index d6061177..a56363df 100644 --- a/src/REvoDesign/shortcuts/tools/vina_tools.py +++ b/src/REvoDesign/shortcuts/tools/vina_tools.py @@ -257,7 +257,8 @@ class CgoBox(GraphicObject): def rebuild(self): """ - Post-initialization processing. Deletes existing objects with the same name and ensures all coordinates are floats. + Post-initialization processing. Deletes existing objects with the same + name and ensures all coordinates are floats. """ self.size_xyz = tuple(abs(x) for x in self.p1.delta_xyz(self.p2)) @@ -279,8 +280,10 @@ def to_vina(self): """ Generates a string of parameters for configuring a binding site in AutoDock Vina. """ - center = f"--center_x {self.cen_xyz[0]:.1f} --center_y {self.cen_xyz[1]:.1f} --center_z {self.cen_xyz[2]:.1f}" - size = f"--size_x {self.size_xyz[0]:.1f} --size_y {self.size_xyz[1]:.1f} --size_z {self.size_xyz[2]:.1f}" + center = ( + f"--center_x {self.cen_xyz[0]:.1f} --center_y {self.cen_xyz[1]:.1f} " f"--center_z {self.cen_xyz[2]:.1f}" + ) + size = f"--size_x {self.size_xyz[0]:.1f} --size_y {self.size_xyz[1]:.1f} " f"--size_z {self.size_xyz[2]:.1f}" return f"{center} {size}" @property @@ -316,10 +319,17 @@ def __repr__(self) -> str: """ Provides a string representation of the box, including its coordinates, size, and center. """ - return f"""CgoBox `{self.name}`: -Coordinates: {self.p1.x:.3f} - {self.p2.x:.3f}; {self.p1.y:.3f} - {self.p2.y:.3f}, {self.p1.z:.3f} - {self.p2.z:.3f} -Size: {self.size_xyz[0]:.3f} * {self.size_xyz[1]:.3f} * {self.size_xyz[2]:.3f} = {self.size_xyz[0] * self.size_xyz[1] * self.size_xyz[2]:.3f} -Center: {self.cen_xyz[0]:.3f}, {self.cen_xyz[1]:.3f}, {self.cen_xyz[2]:.3f}""" + coordinates = ( + f"Coordinates: {self.p1.x:.3f} - {self.p2.x:.3f}; " + f"{self.p1.y:.3f} - {self.p2.y:.3f}, {self.p1.z:.3f} - {self.p2.z:.3f}" + ) + size = ( + f"Size: {self.size_xyz[0]:.3f} * {self.size_xyz[1]:.3f} * " + f"{self.size_xyz[2]:.3f} = " + f"{self.size_xyz[0] * self.size_xyz[1] * self.size_xyz[2]:.3f}" + ) + center = f"Center: {self.cen_xyz[0]:.3f}, {self.cen_xyz[1]:.3f}, {self.cen_xyz[2]:.3f}" + return f"CgoBox `{self.name}`:\n{coordinates}\n{size}\n{center}" @classmethod def from_selection( @@ -382,6 +392,7 @@ def from_selection( return box +# fmt: off @overload def showbox(box: str, minX: float, maxX: float, minY: float, maxY: float, minZ: float, maxZ: float) -> CgoBox: ... @@ -398,6 +409,7 @@ def showbox( maxZ: float | str | None = None, ) -> CgoBox: ... +# fmt: on def showbox( @@ -434,11 +446,14 @@ def showbox( # Validate that all required coordinates are provided when box is a string if any(x is None for x in [minX, maxX, minY, maxY, minZ, maxZ]): raise ValueError( - "To make a box, you must specify minX, maxX, minY, maxY, minZ, maxZ as valid floats or float-like strings." + "To make a box, you must specify minX, maxX, minY, maxY, minZ, " + "maxZ as valid floats or float-like strings." ) # Create a new CgoBox object from provided parameters box = CgoBox( - name=box, p1=Point(float(minX), float(minY), float(minZ)), p2=Point(float(maxX), float(maxY), float(maxZ)) + name=box, + p1=Point(float(minX), float(minY), float(minZ)), + p2=Point(float(maxX), float(maxY), float(maxZ)), ) # Print box details in different formats @@ -543,9 +558,13 @@ def removeions(): """ Remove specified ions from the molecular model. - This function selects and removes ions such as PO4, SO4, ZN, CA, MG, and CL from the molecular model in the PyMOL environment. + This function selects and removes ions such as PO4, SO4, ZN, CA, MG, and CL + from the molecular model in the PyMOL environment. """ - cmd.select("Ions", "((resn PO4) | (resn SO4) | (resn ZN) | (resn CA) | (resn MG) | (resn CL)) & hetatm") + cmd.select( + "Ions", + "((resn PO4) | (resn SO4) | (resn ZN) | (resn CA) | (resn MG) | (resn CL)) & hetatm", + ) cmd.remove("Ions") cmd.delete("Ions") return @@ -692,7 +711,8 @@ def get_pca_box(selection="(sele)", new_box_name: str | None = None, extending=5 Parameters: - selection: A string defining the selection range for which the PCA box is generated. Defaults to "(sele)". - - new_box_name: An optional string specifying the name of the new box. If not provided, a unique name will be generated. + - new_box_name: An optional string specifying the name of the new box. If + not provided, a unique name will be generated. - extending: A float value representing the padding to be added to the oriented bounding box. Defaults to 5.0. Returns: diff --git a/src/REvoDesign/shortcuts/utils.py b/src/REvoDesign/shortcuts/utils.py index d13340b7..7924a7f7 100644 --- a/src/REvoDesign/shortcuts/utils.py +++ b/src/REvoDesign/shortcuts/utils.py @@ -160,7 +160,8 @@ def _build_asked_value(entry: dict) -> AskedValue: AskedValue: A fully constructed `AskedValue` object ready for use in a dialog. Raises: - ValueError: If there's an error resolving dynamic choices via [resolve_choice_from](file:///Users/yyy/Documents/protein_design/REvoDesign/src/REvoDesign/shortcuts/utils.py#L106-L159). + ValueError: If there's an error resolving dynamic choices via + `resolve_choice_from`. Any exceptions raised during callable execution will propagate up. """ # Get type From 879bdfec5be483c9364c0e9178ccef2ade4c50bb Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:05:26 +0800 Subject: [PATCH 28/63] docs(tests): document mini UniRef fixtures --- docs/dev-guide/testing.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/dev-guide/testing.md b/docs/dev-guide/testing.md index 52885e05..46a526b1 100644 --- a/docs/dev-guide/testing.md +++ b/docs/dev-guide/testing.md @@ -91,6 +91,42 @@ tests/ └── test_openkinetics_scorer.py # example test module ``` +### Mini UniRef Test Databases + +Server GREMLIN tests use tiny UniRef30/UniRef90 mocks under +`tests/data/msa/miniuc`. The raw inputs stay in `tests/data/msa` because +non-server evolution tests also read `2KL8.i90c75_aln.fas` directly. + +The shared `miniuc_databases` fixture in `tests/conftest.py` rebuilds the mock +databases on demand when all required binaries are available: + +- `makeblastdb` +- `psiblast` +- `hhblits` +- `cstranslate` +- `ffindex_from_fasta` + +Make sure the REvoDesign Conda environment is active and `CONDA_PREFIX` points +to it before running server tests that need these databases: + +```bash +conda activate REvoDesignTestFlight +echo "$CONDA_PREFIX" +``` + +To rebuild the mini databases manually, follow +`tests/data/msa/README.md`. The expected generated locations are: + +``` +tests/data/msa/miniuc/uc30/miniuc30* +tests/data/msa/miniuc/uc90/uniref90* +tests/data/msa/testminiuc/uc30/ +tests/data/msa/testminiuc/uc90/ +``` + +If those files are missing and any required binary is unavailable, the relevant +server tests skip with a message pointing back to the MSA README. + ## Running Tests Tests **must** run from a temporary directory. The Makefile targets handle this From 755e0b018673f74efbecf25a73d6df5aa10b42f1 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:13:34 +0800 Subject: [PATCH 29/63] docs(server): separate setup guide from audit checks --- docs/dev-guide/server.md | 277 ++++++++++++++++++++++++++++++++++++++ server/README.md | 282 ++++----------------------------------- 2 files changed, 301 insertions(+), 258 deletions(-) diff --git a/docs/dev-guide/server.md b/docs/dev-guide/server.md index 23fd0df8..9f5858d4 100644 --- a/docs/dev-guide/server.md +++ b/docs/dev-guide/server.md @@ -411,6 +411,283 @@ runner-launch changes: same IP, the login endpoint returns HTTP 429 with `retry_after_seconds`; the login page should disable the submit button and count down until retry. +### Docker Socket A/B Attack Test + +Run this test after Docker, Compose, user/group, or runner-launch changes. It +checks whether an authenticated low-privilege web user can turn task submission +into Docker daemon control. The expected secure result is not just "the app +works"; HTTP users must only submit normal FASTA tasks, must not reach Docker +API routes, must not choose Docker image/command/mounts, and must not use a +cookie-only CSRF request for task writes. + +Use an isolated staging instance. Do not run destructive Docker escape payloads +on a host that stores real data. + +Prepare: + +```bash +export BASE_URL=http://127.0.0.1:8080 +export ADMIN_USER=admin +export ADMIN_PASS='' + +docker ps --format '{{.ID}} {{.Names}} {{.Image}} {{.Ports}}' +docker inspect server-web-1 server-worker-1 \ + --format '{{.Name}} User={{.Config.User}} Groups={{json .HostConfig.GroupAdd}} Mounts={{json .Mounts}}' +docker exec server-web-1 id +docker exec server-worker-1 id +docker exec server-web-1 ls -ln /var/run/docker.sock +docker exec server-worker-1 ls -ln /var/run/docker.sock +``` + +A result: the socket is not accessible from web/worker. This proves the live +instance is not Docker-root-escape-capable, but Docker-backed task execution +will fail until permissions are fixed. Expected evidence: + +```text +/var/run/docker.sock -> srw-rw---- 1 0 0 ... +docker.from_env() from web/worker -> PermissionError(13, 'Permission denied') +submitted task -> failed with "Docker daemon unavailable" or PermissionError +``` + +B result: the socket is accessible from web/worker. This is operationally +required for runner containers, but it is host-root-equivalent if arbitrary code +ever runs in web/worker. In this case the test must prove the HTTP contract does +not expose Docker control. Expected evidence: + +```text +limited user /api/auth/admin/users -> 403 +cookie-only POST /PSSM_GREMLIN/api/post -> 403 Bearer token required +POST /containers/create and GET /containers/json on the Flask port -> 404 +path-like FASTA filename upload -> normal task only, sanitized filename, no host-path mount selection +server code fixes image, command, user, environment, and mounts from server config +``` + +The following probe creates a temporary non-admin user, checks the CSRF gate, +submits a harmless FASTA with a path-like filename, probes Docker-like HTTP +routes, and verifies API-key privilege limits: + +```bash +python - <<'PY' +import json +import os +import re +import uuid + +import requests + +base = os.environ["BASE_URL"] +admin_user = os.environ["ADMIN_USER"] +admin_pass = os.environ["ADMIN_PASS"] + + +def show(label, resp): + ctype = resp.headers.get("content-type", "") + body = resp.text[:220].replace("\n", " ") + if "application/json" in ctype: + try: + data = resp.json() + body = { + k: ("[redacted]" if k in {"token", "api_key"} else v) + for k, v in data.items() + } + except Exception: + pass + print( + json.dumps( + { + "label": label, + "status": resp.status_code, + "location": resp.headers.get("Location"), + "body": body, + }, + sort_keys=True, + ) + ) + + +admin = requests.Session() +r = admin.post( + f"{base}/PSSM_GREMLIN/api/auth/login", + json={"username": admin_user, "password": admin_pass}, + timeout=10, +) +show("admin_login", r) +r.raise_for_status() +admin_h = {"Authorization": "Bearer " + r.json()["token"]} + +name = "limited_" + uuid.uuid4().hex[:8] +pw = "Pw_" + uuid.uuid4().hex + "!aA1" +r = admin.post( + f"{base}/PSSM_GREMLIN/api/auth/admin/users", + headers=admin_h, + json={ + "username": name, + "email": name + "@audit.local", + "password": pw, + "affiliation": "audit", + "is_admin": False, + }, + timeout=10, +) +show("create_limited", r) + +limited = requests.Session() +r = limited.post( + f"{base}/PSSM_GREMLIN/api/auth/login", + json={"username": name, "password": pw}, + timeout=10, +) +show("limited_login", r) +r.raise_for_status() +limited_h = {"Authorization": "Bearer " + r.json()["token"]} + +r = limited.get( + f"{base}/PSSM_GREMLIN/api/auth/admin/users", + headers=limited_h, + timeout=10, +) +show("limited_admin_list", r) + +files = { + "file": ( + "../../../../var/run/docker.sock.fasta", + b">audit\nACDEFGHIKLMNPQRSTVWY\n", + ) +} +r = limited.post( + f"{base}/PSSM_GREMLIN/api/post", + files=files, + allow_redirects=False, + timeout=10, +) +show("cookie_only_upload", r) + +files = { + "file": ( + "../../../../--privileged--var-run-docker-sock.fasta", + b">audit\nACDEFGHIKLMNPQRSTVWY\n", + ) +} +r = limited.post( + f"{base}/PSSM_GREMLIN/api/post", + files=files, + headers=limited_h, + allow_redirects=False, + timeout=15, +) +show("bearer_upload_pathlike_name", r) +loc = r.headers.get("Location") or "" +match = re.search(r"/running/([0-9a-f]{32})", loc) +if match: + md5 = match.group(1) + show( + "running_after_upload", + limited.get( + f"{base}/PSSM_GREMLIN/api/running/{md5}", + headers=limited_h, + timeout=10, + ), + ) + show( + "delete_uploaded_task", + limited.delete( + f"{base}/PSSM_GREMLIN/api/delete/{md5}", + headers=limited_h, + timeout=10, + ), + ) + +for method, path in [ + ("get", "/containers/json"), + ("get", "/version"), + ("post", "/containers/create"), + ("get", "/PSSM_GREMLIN/api/docker"), + ("post", "/PSSM_GREMLIN/api/docker/run"), +]: + resp = getattr(limited, method)(base + path, headers=limited_h, timeout=10) + show(f"probe_{method}_{path}", resp) + +r = limited.post( + f"{base}/PSSM_GREMLIN/api/auth/me/api-key", + headers=limited_h, + timeout=10, +) +show("limited_generate_api_key", r) +if r.status_code == 201: + api_h = {"X-API-Key": r.json()["api_key"]} + clean = requests.Session() + show( + "apikey_admin_list", + clean.get( + f"{base}/PSSM_GREMLIN/api/auth/admin/users", + headers=api_h, + timeout=10, + ), + ) + files = {"file": ("api-key-task.fasta", b">audit\nACDEFGHIKLMNPQRSTVWY\n")} + show( + "apikey_upload_task", + clean.post( + f"{base}/PSSM_GREMLIN/api/post", + files=files, + headers=api_h, + allow_redirects=False, + timeout=15, + ), + ) +PY +``` + +Treat a failure in the expected status codes above as a security regression. If +B can access Docker and an HTTP user can influence image, command, privileged +mode, bind source paths, or socket mounts, this is a real host-root escape path. + +### Admin Self-Lockout Check + +Run this after admin user-management changes. The expected result is that an +admin cannot remove their own access, either through a single-user action or a +batch action. + +Expected results: + +```text +self PUT user_status=banned -> 400 Administrators cannot ban their own account +self DELETE -> 400 Administrators cannot delete their own account +batch disable [self, other] -> count 1, self stays active, other becomes banned +batch delete [self, other] -> count 1, self remains undeleted, other is deleted +``` + +### Banned-User Authentication Check + +Run this after account-status or token-auth changes. The check creates a +temporary user, confirms it can log in before the ban, bans it, and verifies +that new login, old Bearer token, and pre-existing API key are all rejected. + +Expected results: + +```text +pre_ban_login -> 200 +pre_ban_api_key -> 201 +ban_user -> 200 +post_ban_login -> 403 Account has been suspended +post_ban_old_bearer_me -> 401 Authentication required +post_ban_api_key_me -> 401 Authentication required +``` + +### Login Throttling Check + +Run this after changes to login, client IP extraction, CDN headers, or rate +limits. + +Expected results: + +```text +first 5 failed login attempts in one minute -> 401 +sixth failed login attempt from the same client identity -> 429 +429 JSON body includes retry_after_seconds +login page disables submit and counts down until retry +``` + ## REvoDesign Plugin Integration The REvoDesign plugin does **not** connect to the PSSM_GREMLIN server diff --git a/server/README.md b/server/README.md index 2ddbc564..a72959fb 100644 --- a/server/README.md +++ b/server/README.md @@ -1,22 +1,7 @@ # PSSM GREMLIN Server -The server is a pip-installable package (`pssm_gremlin_server`) providing a -Flask + Celery + Docker-runner web service for GREMLIN co-evolution analysis. - -## Development - -```bash -# Install in editable mode with test dependencies -pip install -e "server/[test]" - -# Run non-Docker tests from the repo root -pytest server/tests/ -v -k "not Docker and not docker" - -# Run the server directly (no Docker) -python -m pssm_gremlin_server.pssm_gremlin -``` - ---- +The server is a Docker-deployed Flask + Celery + Docker-runner web service for +GREMLIN co-evolution analysis. ## Production Deployment @@ -166,7 +151,6 @@ Fallback when `REVODESIGN_SERVER_ENV` is unset: | `WORKER_CONCURRENCY` | Celery worker concurrency. | | `GUNICORN_WORKERS` | Gunicorn worker count. | | `PORT` | Public HTTP port. | -| `ALLOWED_EMAIL_DOMAINS` | Comma-separated allowed email domains for registration (empty = all allowed). Also normalises plus-aliased addresses. | | `PUBLIC_DASHBOARD` | `false` by default; scopes task visibility to owner unless admin. | | `ADMIN_USERS` | Comma-separated admin usernames for cross-user management. | | `ADMIN_NOTIFY_EMAIL` | Comma-separated admin email addresses for new-user registration digests (default: empty = no notification). | @@ -440,254 +424,20 @@ The web and worker containers mount `/var/run/docker.sock` to spawn runner conta - Consider using a Docker socket proxy (e.g. `docker-socket-proxy`) to restrict API access in untrusted environments. - Never expose the Docker socket to a public network. -#### Docker socket A/B attack test - -Run this test after Docker, Compose, user/group, or runner-launch changes. It -checks whether an authenticated low-privilege web user can turn task submission -into Docker daemon control. The expected secure result is not just "the app -works"; it is that HTTP users can only submit normal FASTA tasks, cannot reach -Docker API routes, cannot choose Docker image/command/mounts, and cannot use a -cookie-only CSRF request for task writes. - -Use an isolated staging instance. Do not run destructive Docker escape payloads -on a host that stores real data. - -Prepare: - -```bash -export BASE_URL=http://127.0.0.1:8080 -export ADMIN_USER=admin -export ADMIN_PASS='' - -docker ps --format '{{.ID}} {{.Names}} {{.Image}} {{.Ports}}' -docker inspect server-web-1 server-worker-1 \ - --format '{{.Name}} User={{.Config.User}} Groups={{json .HostConfig.GroupAdd}} Mounts={{json .Mounts}}' -docker exec server-web-1 id -docker exec server-worker-1 id -docker exec server-web-1 ls -ln /var/run/docker.sock -docker exec server-worker-1 ls -ln /var/run/docker.sock -``` - -A result: the socket is not accessible from web/worker. This proves the live -instance is not Docker-root-escape-capable, but Docker-backed task execution -will fail until permissions are fixed. Expected evidence: - -```text -/var/run/docker.sock -> srw-rw---- 1 0 0 ... -docker.from_env() from web/worker -> PermissionError(13, 'Permission denied') -submitted task -> failed with "Docker daemon unavailable" or PermissionError -``` - -B result: the socket is accessible from web/worker. This is operationally -required for runner containers, but it is host-root-equivalent if arbitrary code -ever runs in web/worker. In this case the test must prove the HTTP contract does -not expose Docker control. Expected evidence: - -```text -limited user /api/auth/admin/users -> 403 -cookie-only POST /PSSM_GREMLIN/api/post -> 403 Bearer token required -POST /containers/create and GET /containers/json on the Flask port -> 404 -path-like FASTA filename upload -> normal task only, sanitized filename, no host-path mount selection -server code fixes image, command, user, environment, and mounts from server config -``` - -The following probe creates a temporary non-admin user, checks the CSRF gate, -submits a harmless FASTA with a path-like filename, probes Docker-like HTTP -routes, and verifies API-key privilege limits: - -```bash -python - <<'PY' -import json, re, uuid -import os -import requests - -base = os.environ["BASE_URL"] -admin_user = os.environ["ADMIN_USER"] -admin_pass = os.environ["ADMIN_PASS"] - -def show(label, resp): - ctype = resp.headers.get("content-type", "") - body = resp.text[:220].replace("\n", " ") - if "application/json" in ctype: - try: - data = resp.json() - body = {k: ("[redacted]" if k in {"token", "api_key"} else v) for k, v in data.items()} - except Exception: - pass - print(json.dumps({"label": label, "status": resp.status_code, "location": resp.headers.get("Location"), "body": body}, sort_keys=True)) - -admin = requests.Session() -r = admin.post(f"{base}/PSSM_GREMLIN/api/auth/login", json={"username": admin_user, "password": admin_pass}, timeout=10) -show("admin_login", r) -r.raise_for_status() -admin_h = {"Authorization": "Bearer " + r.json()["token"]} - -name = "limited_" + uuid.uuid4().hex[:8] -pw = "Pw_" + uuid.uuid4().hex + "!aA1" -r = admin.post( - f"{base}/PSSM_GREMLIN/api/auth/admin/users", - headers=admin_h, - json={"username": name, "email": name + "@audit.local", "password": pw, "affiliation": "audit", "is_admin": False}, - timeout=10, -) -show("create_limited", r) - -limited = requests.Session() -r = limited.post(f"{base}/PSSM_GREMLIN/api/auth/login", json={"username": name, "password": pw}, timeout=10) -show("limited_login", r) -r.raise_for_status() -limited_h = {"Authorization": "Bearer " + r.json()["token"]} - -r = limited.get(f"{base}/PSSM_GREMLIN/api/auth/admin/users", headers=limited_h, timeout=10) -show("limited_admin_list", r) - -files = {"file": ("../../../../var/run/docker.sock.fasta", b">audit\nACDEFGHIKLMNPQRSTVWY\n")} -r = limited.post(f"{base}/PSSM_GREMLIN/api/post", files=files, allow_redirects=False, timeout=10) -show("cookie_only_upload", r) - -files = {"file": ("../../../../--privileged--var-run-docker-sock.fasta", b">audit\nACDEFGHIKLMNPQRSTVWY\n")} -r = limited.post(f"{base}/PSSM_GREMLIN/api/post", files=files, headers=limited_h, allow_redirects=False, timeout=15) -show("bearer_upload_pathlike_name", r) -loc = r.headers.get("Location") or "" -match = re.search(r"/running/([0-9a-f]{32})", loc) -if match: - md5 = match.group(1) - show("running_after_upload", limited.get(f"{base}/PSSM_GREMLIN/api/running/{md5}", headers=limited_h, timeout=10)) - show("delete_uploaded_task", limited.delete(f"{base}/PSSM_GREMLIN/api/delete/{md5}", headers=limited_h, timeout=10)) - -for method, path in [ - ("get", "/containers/json"), - ("get", "/version"), - ("post", "/containers/create"), - ("get", "/PSSM_GREMLIN/api/docker"), - ("post", "/PSSM_GREMLIN/api/docker/run"), -]: - resp = getattr(limited, method)(base + path, headers=limited_h, timeout=10) - show(f"probe_{method}_{path}", resp) - -r = limited.post(f"{base}/PSSM_GREMLIN/api/auth/me/api-key", headers=limited_h, timeout=10) -show("limited_generate_api_key", r) -if r.status_code == 201: - api_h = {"X-API-Key": r.json()["api_key"]} - clean = requests.Session() - show("apikey_admin_list", clean.get(f"{base}/PSSM_GREMLIN/api/auth/admin/users", headers=api_h, timeout=10)) - files = {"file": ("api-key-task.fasta", b">audit\nACDEFGHIKLMNPQRSTVWY\n")} - show("apikey_upload_task", clean.post(f"{base}/PSSM_GREMLIN/api/post", files=files, headers=api_h, allow_redirects=False, timeout=15)) -PY -``` - -Treat a failure in the expected status codes above as a security regression. -If B can access Docker and an HTTP user can influence image, command, privileged -mode, bind source paths, or socket mounts, this is a real host-root escape path. - -#### Admin self-lockout check - -Run this after admin user-management changes. The expected result is that an -admin cannot remove their own access, either through a single-user action or a -batch action. - -Expected results: - -```text -self PUT user_status=banned -> 400 Administrators cannot ban their own account -self DELETE -> 400 Administrators cannot delete their own account -batch disable [self, other] -> count 1, self stays active, other becomes banned -batch delete [self, other] -> count 1, self remains undeleted, other is deleted -``` - -#### Banned-user authentication check - -Run this after account-status or token-auth changes. The check creates a -temporary user, confirms it can log in before the ban, bans it, and verifies -that new login, old Bearer token, and pre-existing API key are all rejected. - -Expected results: - -```text -pre_ban_login -> 200 -pre_ban_api_key -> 201 -ban_user -> 200 -post_ban_login -> 403 Account has been suspended -post_ban_old_bearer_me -> 401 Authentication required -post_ban_api_key_me -> 401 Authentication required -``` - -Command: - -```bash -python - <<'PY' -import json, os, uuid -import requests - -base = os.environ["BASE_URL"] -admin_user = os.environ["ADMIN_USER"] -admin_pass = os.environ["ADMIN_PASS"] - -def show(label, resp): - ctype = resp.headers.get("content-type", "") - body = resp.text[:180].replace("\n", " ") - if "application/json" in ctype: - try: - data = resp.json() - body = {k: ("[redacted]" if k in {"token", "api_key"} else v) for k, v in data.items()} - except Exception: - pass - print(json.dumps({"label": label, "status": resp.status_code, "body": body}, sort_keys=True)) - -admin = requests.Session() -r = admin.post(f"{base}/PSSM_GREMLIN/api/auth/login", json={"username": admin_user, "password": admin_pass}, timeout=10) -show("admin_login", r) -r.raise_for_status() -admin_h = {"Authorization": "Bearer " + r.json()["token"]} - -name = "bancheck_" + uuid.uuid4().hex[:8] -pw = "Pw_" + uuid.uuid4().hex + "!aA1" -r = admin.post( - f"{base}/PSSM_GREMLIN/api/auth/admin/users", - headers=admin_h, - json={"username": name, "email": name + "@audit.local", "password": pw, "affiliation": "audit", "is_admin": False}, - timeout=10, -) -show("create_user", r) - -r = admin.get(f"{base}/PSSM_GREMLIN/api/auth/admin/users", headers=admin_h, timeout=10) -r.raise_for_status() -user_id = next(u["id"] for u in r.json()["users"] if u["username"] == name) - -limited = requests.Session() -r = limited.post(f"{base}/PSSM_GREMLIN/api/auth/login", json={"username": name, "password": pw}, timeout=10) -show("pre_ban_login", r) -r.raise_for_status() -limited_h = {"Authorization": "Bearer " + r.json()["token"]} - -r = limited.post(f"{base}/PSSM_GREMLIN/api/auth/me/api-key", headers=limited_h, timeout=10) -show("pre_ban_api_key", r) -api_key = r.json().get("api_key") if r.status_code == 201 else None - -r = admin.put(f"{base}/PSSM_GREMLIN/api/auth/admin/users/{user_id}", headers=admin_h, json={"user_status": "banned"}, timeout=10) -show("ban_user", r) -show("post_ban_login", requests.post(f"{base}/PSSM_GREMLIN/api/auth/login", json={"username": name, "password": pw}, timeout=10)) -show("post_ban_old_bearer_me", requests.get(f"{base}/PSSM_GREMLIN/api/auth/me", headers=limited_h, timeout=10)) -if api_key: - show("post_ban_api_key_me", requests.get(f"{base}/PSSM_GREMLIN/api/auth/me", headers={"X-API-Key": api_key}, timeout=10)) -PY -``` +Security regression checks for Docker socket exposure, admin self-lockout, +banned users, and login throttling are maintained in +`docs/dev-guide/server.md#security-validation`. ### Authentication -- Tokens are signed with `itsdangerous.URLSafeTimedSerializer` (HMAC-SHA1). -- Gunicorn uses `--preload` so the auth secret key is generated once in the - arbiter; all workers share the same key and tokens validate consistently. - Set `AUTH_SECRET_KEY` to a fixed, high-entropy value in production; otherwise tokens are lost on restart. - Browser page navigations use an `HttpOnly`/`SameSite=Lax` cookie; JavaScript cannot read it, so logout requires the server endpoint (`POST /api/auth/logout`). - Rate limiting: 5 login attempts/minute/IP, 3 registrations/hour/IP. - All state-changing endpoints require a valid Bearer token or API key. - API keys have restricted privileges (task operations only) — Bearer tokens are required for profile changes and admin actions. -- CSRF is mitigated: state-changing endpoints require a Bearer token in the - `Authorization` header (browser same-origin policy prevents cross-origin - requests from setting custom headers). The `HttpOnly` cookie is used for - read-only page navigations only. +- Cookie-only writes are rejected; state-changing API calls require a Bearer + token or API key. ### Redis @@ -715,7 +465,23 @@ PY - Regularly back up sqlite and result archives. - If a task is deleted, result artifacts are removed, but the sqlite record remains for audit. -## 11. Troubleshooting +## 11. Local Development + +```bash +# Install in editable mode with test dependencies +pip install -e "server/[test]" + +# Run non-Docker tests from the repo root +pytest server/tests/ -v -k "not Docker and not docker" + +# Run the server directly without Docker +python -m pssm_gremlin_server.pssm_gremlin +``` + +Full test and security validation guidance is maintained in +`docs/dev-guide/server.md`. + +## 12. Troubleshooting ### Network issues From 31476e306059a39aa8e159a27c80d2b1e4e8eb70 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:15:47 +0800 Subject: [PATCH 30/63] docs(server): record security test status --- docs/dev-guide/server.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/dev-guide/server.md b/docs/dev-guide/server.md index 9f5858d4..12f1175f 100644 --- a/docs/dev-guide/server.md +++ b/docs/dev-guide/server.md @@ -383,6 +383,44 @@ Flask test clients with temporary SQLite databases and environment variables. A dedicated CI workflow (``.github/workflows/server-test.yml``) runs non-Docker tests on every push touching ``server/**``. +### Current Security Test Coverage + +The current automated security coverage is concentrated in: + +- `server/tests/test_security.py`: login throttling, enumeration resistance, + cookie-only write rejection, token tampering, password policy, admin + self-protection, batch-user authorization, upload/path handling, SQL/XSS + payload handling, plus-alias registration policy, task-id validation, and + endpoint method/content-type rejection. +- `server/tests/test_security_advanced.py`: upload/download path traversal, + CRLF/header-injection probes, SSTI/RCE-style payloads, command-injection + filenames, eval/prototype-pollution payloads, YAML/pickle body rejection, API + key brute-force rejection, and binary upload rejection. +- `server/tests/test_auth.py`: auth/session behavior that security checks rely + on, including Bearer-vs-cookie gates, API key lifecycle and privilege limits, + guest restrictions, CAPTCHA/token validation, forgot-password throttling, + deleted/pending/unverified-user rejection, registration/upload rate limits, + upload validation, task ownership separation, and schema validation. + +Current local evaluation snapshot, after `make clean`: + +```bash +conda run -n REvoDesignTestFlight python -m pytest \ + server/tests/test_security.py \ + server/tests/test_security_advanced.py \ + server/tests/test_auth.py \ + -k "security or attack or api_key or rate_limit or deleted_user or pending_user or unverified_email or admin" \ + --no-cov +``` + +Result on 2026-07-17: `66 passed, 43 deselected in 22.93s`. + +This is a non-Docker Flask test-client evaluation. It does not prove live Docker +socket behavior, Docker runtime group mapping, reverse-proxy header handling, or +staging deployment isolation. Use the manual security validation checks below +after Docker Compose, user/group, runner-launch, auth, or account-status +changes. + ## Security Validation Run these checks after auth, account-status, Docker Compose, user/group, or From ec8181598f0f15f96cf94dc54547d6a80fa9d75d Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:17:21 +0800 Subject: [PATCH 31/63] fix(deepsource): wrap sidechain and diffusion lines --- server/scripts/GREMLIN_TFv1.py | 9 +++++- src/REvoDesign/shortcuts/tools/represents.py | 4 ++- .../shortcuts/tools/rfdiffusion_tasks.py | 31 +++++++++++++++++-- .../sidechain/mutate_runner/DLPacker.py | 18 +++++++++-- .../mutate_runner/DunbrackRotamerLib.py | 23 ++++++++++++-- .../mutate_runner/RosettaMutateRelax.py | 21 +++++++++++-- 6 files changed, 93 insertions(+), 13 deletions(-) diff --git a/server/scripts/GREMLIN_TFv1.py b/server/scripts/GREMLIN_TFv1.py index 3238ef55..1b3238b2 100644 --- a/server/scripts/GREMLIN_TFv1.py +++ b/server/scripts/GREMLIN_TFv1.py @@ -327,7 +327,14 @@ def get_loss(): # ## Explore the contact map # ### Contact prediction: # -# For contact prediction, the W matrix is reduced from LxLx21x21 to LxL matrix (by taking the L2norm for each of the 20x20). In the code below, you can access this as mtx["raw"]. Further correction (average product correction) is then performed to the mtx["raw"] to remove the effects of entropy, mtx["apc"]. The relative ranking of mtx["apc"] is used to assess importance. When there are enough effective sequences (>1000), we find that the top 1.0L contacts are ~90% accurate! When the number of effective sequences is lower, NN can help clean noise and fill in missing contacts. +# For contact prediction, the W matrix is reduced from LxLx21x21 to LxL matrix +# by taking the L2 norm for each of the 20x20 slices. In the code below, you can +# access this as mtx["raw"]. Further correction (average product correction) is +# then performed to remove entropy effects, producing mtx["apc"]. The relative +# ranking of mtx["apc"] is used to assess importance. When there are enough +# effective sequences (>1000), we find that the top 1.0L contacts are ~90% +# accurate. When the number of effective sequences is lower, NN can help clean +# noise and fill in missing contacts. # # ## Functions for extracting contacts from MRF diff --git a/src/REvoDesign/shortcuts/tools/represents.py b/src/REvoDesign/shortcuts/tools/represents.py index 7d681bec..149d8b91 100644 --- a/src/REvoDesign/shortcuts/tools/represents.py +++ b/src/REvoDesign/shortcuts/tools/represents.py @@ -802,7 +802,9 @@ def _load_to_one_chain(chain_id: str, state: int = 0): bfact_ori = bf_chain.get(pos) except (IndexError, KeyError, issues.InvalidInputError): logging.warning( - f"Position {pos} (zero-indexed {pos - 1}) exceeds the length of new B-factor data ({len(bf_chain.bfactor_data)}); setting B-factor to -1.0" + f"Position {pos} (zero-indexed {pos - 1}) exceeds the length of " + f"new B-factor data ({len(bf_chain.bfactor_data)}); setting " + "B-factor to -1.0" ) continue bfacts_orignal.append(bfact_ori) diff --git a/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py b/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py index 55d3ea48..9a846827 100644 --- a/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py +++ b/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py @@ -77,7 +77,9 @@ def fetch_cuda_version_before_install(self): self.cuda_version = "" warnings.warn( issues.PlatformNotSupportedWarning( - f"CUDA version {cuda_version} is not supported by DGL. Please install CUDA version >= 11.8 if you need to use DGL with CUDA support." + f"CUDA version {cuda_version} is not supported by DGL. " + "Please install CUDA version >= 11.8 if you need to use " + "DGL with CUDA support." ), stacklevel=2, ) @@ -89,7 +91,9 @@ def install(self): else: index_link = "https://data.dgl.ai/wheels/repo.html" - c = run_command([sys.executable, "-m", "pip", "install", "dgl==2.2.1", "-f", index_link]) + c = run_command( + [sys.executable, "-m", "pip", "install", "dgl==2.2.1", "-f", index_link] + ) if c.returncode != 0: logging.error(f"Failed to install DGL: {c.stderr}") raise RuntimeError(f"Failed to install DGL: {c.stderr}") @@ -451,7 +455,28 @@ def main(self) -> None: volume={620}, number={7976}, pages={1089-1100}, -abstract={There has been considerable recent progress in designing new proteins using deep-learning methods1--9. Despite this progress, a general deep-learning framework for protein design that enables solution of a wide range of design challenges, including de novo binder design and design of higher-order symmetric architectures, has yet to be described. Diffusion models10,11 have had considerable success in image and language generative modelling but limited success when applied to protein modelling, probably due to the complexity of protein backbone geometry and sequence--structure relationships. Here we show that by fine-tuning the RoseTTAFold structure prediction network on protein structure denoising tasks, we obtain a generative model of protein backbones that achieves outstanding performance on unconditional and topology-constrained protein monomer design, protein binder design, symmetric oligomer design, enzyme active site scaffolding and symmetric motif scaffolding for therapeutic and metal-binding protein design. We demonstrate the power and generality of the method, called RoseTTAFold diffusion (RFdiffusion), by experimentally characterizing the structures and functions of hundreds of designed symmetric assemblies, metal-binding proteins and protein binders. The accuracy of RFdiffusion is confirmed by the cryogenic electron microscopy structure of a designed binder in complex with influenza haemagglutinin that is nearly identical to the design model. In a manner analogous to networks that produce images from user-specified inputs, RFdiffusion enables the design of diverse functional proteins from simple molecular specifications.}, +abstract={There has been considerable recent progress in designing new proteins +using deep-learning methods1--9. Despite this progress, a general deep-learning +framework for protein design that enables solution of a wide range of design +challenges, including de novo binder design and design of higher-order symmetric +architectures, has yet to be described. Diffusion models10,11 have had +considerable success in image and language generative modelling but limited +success when applied to protein modelling, probably due to the complexity of +protein backbone geometry and sequence--structure relationships. Here we show +that by fine-tuning the RoseTTAFold structure prediction network on protein +structure denoising tasks, we obtain a generative model of protein backbones +that achieves outstanding performance on unconditional and topology-constrained +protein monomer design, protein binder design, symmetric oligomer design, enzyme +active site scaffolding and symmetric motif scaffolding for therapeutic and +metal-binding protein design. We demonstrate the power and generality of the +method, called RoseTTAFold diffusion (RFdiffusion), by experimentally +characterizing the structures and functions of hundreds of designed symmetric +assemblies, metal-binding proteins and protein binders. The accuracy of +RFdiffusion is confirmed by the cryogenic electron microscopy structure of a +designed binder in complex with influenza haemagglutinin that is nearly +identical to the design model. In a manner analogous to networks that produce +images from user-specified inputs, RFdiffusion enables the design of diverse +functional proteins from simple molecular specifications.}, issn={1476-4687}, doi={10.1038/s41586-023-06415-8}, url={https://doi.org/10.1038/s41586-023-06415-8} diff --git a/src/REvoDesign/sidechain/mutate_runner/DLPacker.py b/src/REvoDesign/sidechain/mutate_runner/DLPacker.py index 37f15cf1..ddbc0234 100644 --- a/src/REvoDesign/sidechain/mutate_runner/DLPacker.py +++ b/src/REvoDesign/sidechain/mutate_runner/DLPacker.py @@ -212,7 +212,9 @@ def run_mutate_parallel( logging.warning(f"Fixed {nproc=} to {num_task=}") nproc = num_task - results = Parallel(n_jobs=nproc, return_as="list")(delayed(self.run_mutate)(mutant) for mutant in mutants) + results = Parallel(n_jobs=nproc, return_as="list")( + delayed(self.run_mutate)(mutant) for mutant in mutants + ) gc.collect() return list(results) # type: ignore @@ -229,7 +231,19 @@ def run_mutate_parallel( doi = {https://doi.org/10.1002/prot.26311}, url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/prot.26311}, eprint = {https://onlinelibrary.wiley.com/doi/pdf/10.1002/prot.26311}, -abstract = {Abstract Prediction of side chain conformations of amino acids in proteins (also termed “packing”) is an important and challenging part of protein structure prediction with many interesting applications in protein design. A variety of methods for packing have been developed but more accurate ones are still needed. Machine learning (ML) methods have recently become a powerful tool for solving various problems in diverse areas of science, including structural biology. In this study, we evaluate the potential of deep neural networks (DNNs) for prediction of amino acid side chain conformations. We formulate the problem as image-to-image transformation and train a U-net style DNN to solve the problem. We show that our method outperforms other physics-based methods by a significant margin: reconstruction RMSDs for most amino acids are about 20\% smaller compared to SCWRL4 and Rosetta Packer with RMSDs for bulky hydrophobic amino acids Phe, Tyr, and Trp being up to 50\% smaller.}, +abstract = {Abstract Prediction of side chain conformations of amino acids in proteins +(also termed “packing”) is an important and challenging part of protein +structure prediction with many interesting applications in protein design. A +variety of methods for packing have been developed but more accurate ones are +still needed. Machine learning (ML) methods have recently become a powerful tool +for solving various problems in diverse areas of science, including structural +biology. In this study, we evaluate the potential of deep neural networks (DNNs) +for prediction of amino acid side chain conformations. We formulate the problem +as image-to-image transformation and train a U-net style DNN to solve the +problem. We show that our method outperforms other physics-based methods by a +significant margin: reconstruction RMSDs for most amino acids are about 20\% +smaller compared to SCWRL4 and Rosetta Packer with RMSDs for bulky hydrophobic +amino acids Phe, Tyr, and Trp being up to 50\% smaller.}, year = {2022} } diff --git a/src/REvoDesign/sidechain/mutate_runner/DunbrackRotamerLib.py b/src/REvoDesign/sidechain/mutate_runner/DunbrackRotamerLib.py index a8771ca6..4dd3906d 100644 --- a/src/REvoDesign/sidechain/mutate_runner/DunbrackRotamerLib.py +++ b/src/REvoDesign/sidechain/mutate_runner/DunbrackRotamerLib.py @@ -111,7 +111,9 @@ def run_mutate_parallel(self, mutants: list[Mutant], nproc: int = 2) -> list[str - List of paths to the mutated PDB files """ - results = Parallel(n_jobs=nproc, return_as="list")(delayed(self.run_mutate)(mutant) for mutant in mutants) + results = Parallel(n_jobs=nproc, return_as="list")( + delayed(self.run_mutate)(mutant) for mutant in mutants + ) return list(results) # type: ignore # https://www.bruot.org/ris2bib/ @@ -119,7 +121,8 @@ def run_mutate_parallel(self, mutants: list[Mutant], nproc: int = 2) -> list[str "Dunbrack Rotamer Library": """@Article{Shapovalov2011, author={Shapovalov, Maxim V. and Dunbrack Jr., Roland L.,}, -title={A Smoothed Backbone-Dependent Rotamer Library for Proteins Derived from Adaptive Kernel Density Estimates and Regressions}, +title={A Smoothed Backbone-Dependent Rotamer Library for Proteins Derived from +Adaptive Kernel Density Estimates and Regressions}, journal={Structure}, year={2011}, month={Jun}, @@ -128,7 +131,21 @@ def run_mutate_parallel(self, mutants: list[Mutant], nproc: int = 2) -> list[str volume={19}, number={6}, pages={844-858}, -abstract={Rotamer libraries are used in protein structure determination, prediction, and design. The backbone-dependent rotamer library consists of rotamer frequencies, mean dihedral angles, and variances as a function of the backbone dihedral angles. Structure prediction and design methods that employ backbone flexibility would strongly benefit from smoothly varying probabilities and angles. A new version of the backbone-dependent rotamer library has been developed using adaptive kernel density estimates for the rotamer frequencies and adaptive kernel regression for the mean dihedral angles and variances. This formulation allows for evaluation of the rotamer probabilities, mean angles, and variances as a smooth and continuous function of phi and psi. Continuous probability density estimates for the nonrotameric degrees of freedom of amides, carboxylates, and aromatic side chains have been modeled as a function of the backbone dihedrals and rotamers of the remaining degrees of freedom. New backbone-dependent rotamer libraries at varying levels of smoothing are available from http://dunbrack.fccc.edu.}, +abstract={Rotamer libraries are used in protein structure determination, prediction, +and design. The backbone-dependent rotamer library consists of rotamer +frequencies, mean dihedral angles, and variances as a function of the backbone +dihedral angles. Structure prediction and design methods that employ backbone +flexibility would strongly benefit from smoothly varying probabilities and +angles. A new version of the backbone-dependent rotamer library has been +developed using adaptive kernel density estimates for the rotamer frequencies +and adaptive kernel regression for the mean dihedral angles and variances. This +formulation allows for evaluation of the rotamer probabilities, mean angles, and +variances as a smooth and continuous function of phi and psi. Continuous +probability density estimates for the nonrotameric degrees of freedom of +amides, carboxylates, and aromatic side chains have been modeled as a function +of the backbone dihedrals and rotamers of the remaining degrees of freedom. New +backbone-dependent rotamer libraries at varying levels of smoothing are +available from http://dunbrack.fccc.edu.}, issn={0969-2126}, doi={10.1016/j.str.2011.03.019}, url={https://doi.org/10.1016/j.str.2011.03.019} diff --git a/src/REvoDesign/sidechain/mutate_runner/RosettaMutateRelax.py b/src/REvoDesign/sidechain/mutate_runner/RosettaMutateRelax.py index 7562538b..e42c4f55 100644 --- a/src/REvoDesign/sidechain/mutate_runner/RosettaMutateRelax.py +++ b/src/REvoDesign/sidechain/mutate_runner/RosettaMutateRelax.py @@ -209,12 +209,27 @@ def run_mutate_parallel( List[str]: List of output PDB file paths for all mutants. """ # Refresh node configuration before each run - self.mutate_relax_instance.node = self.node_hint, rosetta_utils.read_rosetta_node_config() + self.mutate_relax_instance.node = ( + self.node_hint, + rosetta_utils.read_rosetta_node_config(), + ) self.mutate_relax_instance.run(mutants, opts=list(self.rosetta_general_opts)) - return [os.path.join(self.temp_dir, f"{mutant.short_mutant_id}.pdb") for mutant in mutants] + return [ + os.path.join(self.temp_dir, f"{mutant.short_mutant_id}.pdb") + for mutant in mutants + ] __bibtex__ = rosetta_utils.copy_rosetta_citation( { - "Relax": """@article{10.1002/pro.2389, author = {Conway, P. and Tyka, M. D. and DiMaio, F. and Konerding, D. E. and Baker, D.}, title = {Relaxation of backbone bond geometry improves protein energy landscape modeling}, journal = {Protein Science}, year = {2013}, volume = {23}, issue = {1}, pages = {47-55}, doi = {10.1002/pro.2389} }""" + "Relax": """@article{10.1002/pro.2389, +author = {Conway, P. and Tyka, M. D. and DiMaio, F. and Konerding, D. E. and Baker, D.}, +title = {Relaxation of backbone bond geometry improves protein energy landscape modeling}, +journal = {Protein Science}, +year = {2013}, +volume = {23}, +issue = {1}, +pages = {47-55}, +doi = {10.1002/pro.2389} +}""" } ) From 04255e6e0aa17f32635d9814570066690ff76b3f Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:22:00 +0800 Subject: [PATCH 32/63] fix(deepsource): wrap rosetta citation lines --- .../shortcuts/tools/rosetta_tasks.py | 56 +++++++++- src/REvoDesign/tools/rosetta_utils.py | 102 ++++++++++++++++-- 2 files changed, 146 insertions(+), 12 deletions(-) diff --git a/src/REvoDesign/shortcuts/tools/rosetta_tasks.py b/src/REvoDesign/shortcuts/tools/rosetta_tasks.py index acfc539c..0655c729 100644 --- a/src/REvoDesign/shortcuts/tools/rosetta_tasks.py +++ b/src/REvoDesign/shortcuts/tools/rosetta_tasks.py @@ -45,7 +45,27 @@ class RosettaLigand(RosettaLigandOriginal, CitableModuleAbstract): doi = {https://doi.org/10.1002/prot.21086}, url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/prot.21086}, eprint = {https://onlinelibrary.wiley.com/doi/pdf/10.1002/prot.21086}, -abstract = {Abstract Protein–small molecule docking algorithms provide a means to model the structure of protein–small molecule complexes in structural detail and play an important role in drug development. In recent years the necessity of simulating protein side-chain flexibility for an accurate prediction of the protein–small molecule interfaces has become apparent, and an increasing number of docking algorithms probe different approaches to include protein flexibility. Here we describe a new method for docking small molecules into protein binding sites employing a Monte Carlo minimization procedure in which the rigid body position and orientation of the small molecule and the protein side-chain conformations are optimized simultaneously. The energy function comprises van der Waals (VDW) interactions, an implicit solvation model, an explicit orientation hydrogen bonding potential, and an electrostatics model. In an evaluation of the scoring function the computed energy correlated with experimental small molecule binding energy with a correlation coefficient of 0.63 across a diverse set of 229 protein– small molecule complexes. The docking method produced lowest energy models with a root mean square deviation (RMSD) smaller than 2 Å in 71 out of 100 protein–small molecule crystal structure complexes (self-docking). In cross-docking calculations in which both protein side-chain and small molecule internal degrees of freedom were varied the lowest energy predictions had RMSDs less than 2 Å in 14 of 20 test cases. Proteins 2006. © 2006 Wiley-Liss, Inc.}, +abstract = {Abstract Protein–small molecule docking algorithms provide a means to model +the structure of protein–small molecule complexes in structural detail and play +an important role in drug development. In recent years the necessity of +simulating protein side-chain flexibility for an accurate prediction of the +protein–small molecule interfaces has become apparent, and an increasing number +of docking algorithms probe different approaches to include protein flexibility. +Here we describe a new method for docking small molecules into protein binding +sites employing a Monte Carlo minimization procedure in which the rigid body +position and orientation of the small molecule and the protein side-chain +conformations are optimized simultaneously. The energy function comprises van +der Waals (VDW) interactions, an implicit solvation model, an explicit +orientation hydrogen bonding potential, and an electrostatics model. In an +evaluation of the scoring function the computed energy correlated with +experimental small molecule binding energy with a correlation coefficient of +0.63 across a diverse set of 229 protein– small molecule complexes. The docking +method produced lowest energy models with a root mean square deviation (RMSD) +smaller than 2 Å in 71 out of 100 protein–small molecule crystal structure +complexes (self-docking). In cross-docking calculations in which both protein +side-chain and small molecule internal degrees of freedom were varied the lowest +energy predictions had RMSDs less than 2 Å in 14 of 20 test cases. Proteins +2006. © 2006 Wiley-Liss, Inc.}, year = {2006} } """, @@ -60,7 +80,14 @@ class RosettaLigand(RosettaLigandOriginal, CitableModuleAbstract): publisher="Springer New York", address="New York, NY", pages="143--155", -abstract="RosettaLigand is premiere software for predicting how a protein and a small molecule interact. Benchmark studies demonstrate that 70{\%} of the top scoring RosettaLigand predicted interfaces are within 2{\AA} RMSD from the crystal structure [1]. The latest release of Rosetta ligand software includes many new features, such as (1) docking of multiple ligands simultaneously, (2) representing ligands as fragments for greater flexibility, (3) redesign of the interface during docking, and (4) an XML script based interface that gives the user full control of the ligand docking protocol.", +abstract="RosettaLigand is premiere software for predicting how a protein and a +small molecule interact. Benchmark studies demonstrate that 70{\%} of the top +scoring RosettaLigand predicted interfaces are within 2{\AA} RMSD from the +crystal structure [1]. The latest release of Rosetta ligand software includes +many new features, such as (1) docking of multiple ligands simultaneously, (2) +representing ligands as fragments for greater flexibility, (3) redesign of the +interface during docking, and (4) an XML script based interface that gives the +user full control of the ligand docking protocol.", isbn="978-1-61779-465-0", doi="10.1007/978-1-61779-465-0_10", url="https://doi.org/10.1007/978-1-61779-465-0_10" @@ -84,7 +111,15 @@ class PROSS(PROSS_Original, CitableModuleAbstract): pages = {123-125}, year = {2020}, month = {12}, -abstract = {Many natural and designed proteins are only marginally stable limiting their usefulness in research and applications. Recently, we described an automated structure and sequence-based design method, called PROSS, for optimizing protein stability and heterologous expression levels that has since been validated on dozens of proteins. Here, we introduce improvements to the method, workflow and presentation, including more accurate sequence analysis, error handling and automated analysis of the quality of the sequence alignment that is used in design calculations.PROSS2 is freely available for academic use at https://pross.weizmann.ac.il. }, +abstract = {Many natural and designed proteins are only marginally stable limiting +their usefulness in research and applications. Recently, we described an +automated structure and sequence-based design method, called PROSS, for +optimizing protein stability and heterologous expression levels that has since +been validated on dozens of proteins. Here, we introduce improvements to the +method, workflow and presentation, including more accurate sequence analysis, +error handling and automated analysis of the quality of the sequence alignment +that is used in design calculations.PROSS2 is freely available for academic use +at https://pross.weizmann.ac.il. }, issn = {1367-4803}, doi = {10.1093/bioinformatics/btaa1071}, url = {https://doi.org/10.1093/bioinformatics/btaa1071}, @@ -93,7 +128,20 @@ class PROSS(PROSS_Original, CitableModuleAbstract): """, "PROSS": """ -@article{10.1016/j.molcel.2016.06.012, author = {Goldenzweig, A. and Goldsmith, M. and Hill, S. E. and Gertman, O. and Laurino, P. and Ashani, Y. and Dym, O. and Unger, T. and Albeck, S. and Prilusky, J. and Lieberman, R. L. and Aharoni, A. and Silman, I. and Sussman, J. L. and Tawfik, D. S. and Fleishman, S. J.}, title = {Automated structure- and sequence-based design of proteins for high bacterial expression and stability}, journal = {Molecular Cell}, year = {2016}, volume = {63}, issue = {2}, pages = {337-346}, doi = {10.1016/j.molcel.2016.06.012} }""", +@article{10.1016/j.molcel.2016.06.012, +author = {Goldenzweig, A. and Goldsmith, M. and Hill, S. E. and Gertman, O. +and Laurino, P. and Ashani, Y. and Dym, O. and Unger, T. and Albeck, S. +and Prilusky, J. and Lieberman, R. L. and Aharoni, A. and Silman, I. +and Sussman, J. L. and Tawfik, D. S. and Fleishman, S. J.}, +title = {Automated structure- and sequence-based design of proteins for high +bacterial expression and stability}, +journal = {Molecular Cell}, +year = {2016}, +volume = {63}, +issue = {2}, +pages = {337-346}, +doi = {10.1016/j.molcel.2016.06.012} +}""", } ) diff --git a/src/REvoDesign/tools/rosetta_utils.py b/src/REvoDesign/tools/rosetta_utils.py index e41a81d2..d50e4431 100644 --- a/src/REvoDesign/tools/rosetta_utils.py +++ b/src/REvoDesign/tools/rosetta_utils.py @@ -291,10 +291,48 @@ def read_rosetta_node_config() -> dict[str, Any]: ROSETTA_COMMON_CITATION: dict[str, str | tuple] = { - "Rosetta": """@article{10.1038/s41592-020-0848-2, author = {Leman, J. K. and Weitzner, B. D. and Lewis, S. M. and Adolf‐Bryfogle, J. and Alam, N. and Alford, R. F. and Aprahamian, M. L. and Baker, D. and Barlow, K. A. and Barth, P. and Basanta, B. and Bender, B. J. and Blacklock, K. and Bonet, J. and Boyken, S. E. and Bradley, P. and Bystroff, C. and Conway, P. and Cooper, S. and Correia, B. E. and Coventry, B. and Das, R. and Jong, R. M. d. and DiMaio, F. and Dsilva, L. and Dunbrack, R. L. and Ford, A. S. and Frenz, B. and Fu, D. and Geniesse, C. and Goldschmidt, L. and Gowthaman, R. and Gray, J. J. and Gront, D. and Guffy, S. L. and Horowitz, S. and Huang, P. and Huber, T. and Jacobs, T. M. and Jeliazkov, J. R. and Johnson, D. K. and Kappel, K. and Karanicolas, J. and Khakzad, H. and Khar, K. R. and Khare, S. D. and Khatib, F. and Khramushin, A. and King, C. and Kleffner, R. and Koepnick, B. and Kortemme, T. and Kuenze, G. and Kuhlman, B. and Kuroda, D. and Labonte, J. W. and Lai, J. and Lapidoth, G. and Leaver‐Fay, A. and Lindert, S. and Linsky, T. W. and London, N. and Lubin, J. H. and Lyskov, S. and Maguire, J. B. and Malmström, L. and Marcos, E. and Marcu, O. and Marze, N. and Meiler, J. and Moretti, R. and Mulligan, V. K. and Nerli, S. and Norn, C. and Ó’Conchúir, S. and Ollikainen, N. and Ovchinnikov, S. and Pacella, M. S. and Pan, X. and Park, H. and Pavlovicz, R. E. and Pethe, M. A. and Pierce, B. G. and Pilla, K. B. and Raveh, B. and Renfrew, P. D. and Burman, S. S. R. and Rubenstein, A. B. and Sauer, M. F. and Scheck, A. and Schief, W. R. and Schueler‐Furman, O. and Sedan, Y. and Sevy, A. M. and Sgourakis, N. G. and Shi, L. and Siegel, J. B. and Silva, D. and Smith, S. T. and Song, Y. and Stein, A. and Szegedy, M. and Teets, F. D. and Thyme, S. B. and Wang, R. Y. and Watkins, A. M. and Zimmerman, L. and Bonneau, R.}, title = {Macromolecular modeling and design in rosetta: recent methods and frameworks}, journal = {Nature Methods}, year = {2020}, volume = {17}, issue = {7}, pages = {665-680}, doi = {10.1038/s41592-020-0848-2} }""", + "Rosetta": """@article{10.1038/s41592-020-0848-2, +author = {Leman, J. K. and Weitzner, B. D. and Lewis, S. M. +and Adolf‐Bryfogle, J. and Alam, N. and Alford, R. F. +and Aprahamian, M. L. and Baker, D. and Barlow, K. A. and Barth, P. +and Basanta, B. and Bender, B. J. and Blacklock, K. and Bonet, J. +and Boyken, S. E. and Bradley, P. and Bystroff, C. and Conway, P. +and Cooper, S. and Correia, B. E. and Coventry, B. and Das, R. +and Jong, R. M. d. and DiMaio, F. and Dsilva, L. and Dunbrack, R. L. +and Ford, A. S. and Frenz, B. and Fu, D. and Geniesse, C. +and Goldschmidt, L. and Gowthaman, R. and Gray, J. J. and Gront, D. +and Guffy, S. L. and Horowitz, S. and Huang, P. and Huber, T. +and Jacobs, T. M. and Jeliazkov, J. R. and Johnson, D. K. +and Kappel, K. and Karanicolas, J. and Khakzad, H. and Khar, K. R. +and Khare, S. D. and Khatib, F. and Khramushin, A. and King, C. +and Kleffner, R. and Koepnick, B. and Kortemme, T. and Kuenze, G. +and Kuhlman, B. and Kuroda, D. and Labonte, J. W. and Lai, J. +and Lapidoth, G. and Leaver‐Fay, A. and Lindert, S. and Linsky, T. W. +and London, N. and Lubin, J. H. and Lyskov, S. and Maguire, J. B. +and Malmström, L. and Marcos, E. and Marcu, O. and Marze, N. +and Meiler, J. and Moretti, R. and Mulligan, V. K. and Nerli, S. +and Norn, C. and Ó’Conchúir, S. and Ollikainen, N. and Ovchinnikov, S. +and Pacella, M. S. and Pan, X. and Park, H. and Pavlovicz, R. E. +and Pethe, M. A. and Pierce, B. G. and Pilla, K. B. and Raveh, B. +and Renfrew, P. D. and Burman, S. S. R. and Rubenstein, A. B. +and Sauer, M. F. and Scheck, A. and Schief, W. R. +and Schueler‐Furman, O. and Sedan, Y. and Sevy, A. M. +and Sgourakis, N. G. and Shi, L. and Siegel, J. B. and Silva, D. +and Smith, S. T. and Song, Y. and Stein, A. and Szegedy, M. +and Teets, F. D. and Thyme, S. B. and Wang, R. Y. +and Watkins, A. M. and Zimmerman, L. and Bonneau, R.}, +title = {Macromolecular modeling and design in rosetta: recent methods and frameworks}, +journal = {Nature Methods}, +year = {2020}, +volume = {17}, +issue = {7}, +pages = {665-680}, +doi = {10.1038/s41592-020-0848-2} +}""", "Rosetta3": """ @incollection{LEAVERFAY2011545, -title = {Chapter nineteen - Rosetta3: An Object-Oriented Software Suite for the Simulation and Design of Macromolecules}, +title = {Chapter nineteen - Rosetta3: An Object-Oriented Software Suite for the +Simulation and Design of Macromolecules}, editor = {Michael L. Johnson and Ludwig Brand}, series = {Methods in Enzymology}, publisher = {Academic Press}, @@ -305,27 +343,75 @@ def read_rosetta_node_config() -> dict[str, Any]: issn = {0076-6879}, doi = {https://doi.org/10.1016/B978-0-12-381270-4.00019-6}, url = {https://www.sciencedirect.com/science/article/pii/B9780123812704000196}, -author = {Andrew Leaver-Fay and Michael Tyka and Steven M. Lewis and Oliver F. Lange and James Thompson and Ron Jacak and Kristian W. Kaufman and P. Douglas Renfrew and Colin A. Smith and Will Sheffler and Ian W. Davis and Seth Cooper and Adrien Treuille and Daniel J. Mandell and Florian Richter and Yih-En Andrew Ban and Sarel J. Fleishman and Jacob E. Corn and David E. Kim and Sergey Lyskov and Monica Berrondo and Stuart Mentzer and Zoran Popović and James J. Havranek and John Karanicolas and Rhiju Das and Jens Meiler and Tanja Kortemme and Jeffrey J. Gray and Brian Kuhlman and David Baker and Philip Bradley}, -abstract = {We have recently completed a full rearchitecturing of the Rosetta molecular modeling program, generalizing and expanding its existing functionality. The new architecture enables the rapid prototyping of novel protocols by providing easy-to-use interfaces to powerful tools for molecular modeling. The source code of this rearchitecturing has been released as Rosetta3 and is freely available for academic use. At the time of its release, it contained 470,000 lines of code. Counting currently unpublished protocols at the time of this writing, the source includes 1,285,000 lines. Its rapid growth is a testament to its ease of use. This chapter describes the requirements for our new architecture, justifies the design decisions, sketches out central classes, and highlights a few of the common tasks that the new software can perform.} +author = {Andrew Leaver-Fay and Michael Tyka and Steven M. Lewis +and Oliver F. Lange and James Thompson and Ron Jacak and Kristian W. Kaufman +and P. Douglas Renfrew and Colin A. Smith and Will Sheffler and Ian W. Davis +and Seth Cooper and Adrien Treuille and Daniel J. Mandell and Florian Richter +and Yih-En Andrew Ban and Sarel J. Fleishman and Jacob E. Corn +and David E. Kim and Sergey Lyskov and Monica Berrondo and Stuart Mentzer +and Zoran Popović and James J. Havranek and John Karanicolas and Rhiju Das +and Jens Meiler and Tanja Kortemme and Jeffrey J. Gray and Brian Kuhlman +and David Baker and Philip Bradley}, +abstract = {We have recently completed a full rearchitecturing of the Rosetta +molecular modeling program, generalizing and expanding its existing +functionality. The new architecture enables the rapid prototyping of novel +protocols by providing easy-to-use interfaces to powerful tools for molecular +modeling. The source code of this rearchitecturing has been released as Rosetta3 +and is freely available for academic use. At the time of its release, it +contained 470,000 lines of code. Counting currently unpublished protocols at the +time of this writing, the source includes 1,285,000 lines. Its rapid growth is a +testament to its ease of use. This chapter describes the requirements for our +new architecture, justifies the design decisions, sketches out central classes, +and highlights a few of the common tasks that the new software can perform.} }""", "RosettaScripts": """ @article{10.1371/journal.pone.0020161, doi = {10.1371/journal.pone.0020161}, - author = {Fleishman, Sarel J. AND Leaver-Fay, Andrew AND Corn, Jacob E. AND Strauch, Eva-Maria AND Khare, Sagar D. AND Koga, Nobuyasu AND Ashworth, Justin AND Murphy, Paul AND Richter, Florian AND Lemmon, Gordon AND Meiler, Jens AND Baker, David}, + author = {Fleishman, Sarel J. AND Leaver-Fay, Andrew AND Corn, Jacob E. + AND Strauch, Eva-Maria AND Khare, Sagar D. AND Koga, Nobuyasu + AND Ashworth, Justin AND Murphy, Paul AND Richter, Florian + AND Lemmon, Gordon AND Meiler, Jens AND Baker, David}, journal = {PLOS ONE}, publisher = {Public Library of Science}, - title = {RosettaScripts: A Scripting Language Interface to the Rosetta Macromolecular Modeling Suite}, + title = {RosettaScripts: A Scripting Language Interface to the Rosetta + Macromolecular Modeling Suite}, year = {2011}, month = {06}, volume = {6}, url = {https://doi.org/10.1371/journal.pone.0020161}, pages = {1-10}, - abstract = {Macromolecular modeling and design are increasingly useful in basic research, biotechnology, and teaching. However, the absence of a user-friendly modeling framework that provides access to a wide range of modeling capabilities is hampering the wider adoption of computational methods by non-experts. RosettaScripts is an XML-like language for specifying modeling tasks in the Rosetta framework. RosettaScripts provides access to protocol-level functionalities, such as rigid-body docking and sequence redesign, and allows fast testing and deployment of complex protocols without need for modifying or recompiling the underlying C++ code. We illustrate these capabilities with RosettaScripts protocols for the stabilization of proteins, the generation of computationally constrained libraries for experimental selection of higher-affinity binding proteins, loop remodeling, small-molecule ligand docking, design of ligand-binding proteins, and specificity redesign in DNA-binding proteins.}, + abstract = {Macromolecular modeling and design are increasingly useful in basic + research, biotechnology, and teaching. However, the absence of a + user-friendly modeling framework that provides access to a wide range of + modeling capabilities is hampering the wider adoption of computational + methods by non-experts. RosettaScripts is an XML-like language for + specifying modeling tasks in the Rosetta framework. RosettaScripts provides + access to protocol-level functionalities, such as rigid-body docking and + sequence redesign, and allows fast testing and deployment of complex + protocols without need for modifying or recompiling the underlying C++ code. + We illustrate these capabilities with RosettaScripts protocols for the + stabilization of proteins, the generation of computationally constrained + libraries for experimental selection of higher-affinity binding proteins, + loop remodeling, small-molecule ligand docking, design of ligand-binding + proteins, and specificity redesign in DNA-binding proteins.}, number = {6}, }""", "Rosetta All-Atom Energy Function": """ -@article{10.1021/acs.jctc.7b00125, author = {Alford, R. F. and Leaver‐Fay, A. and Jeliazkov, J. R. and O’Meara, M. J. and DiMaio, F. and Park, H. and Shapovalov, M. V. and Renfrew, P. D. and Mulligan, V. K. and Kappel, K. and Labonte, J. W. and Pacella, M. S. and Bonneau, R. and Bradley, P. and Dunbrack, R. L. and Das, R. and Baker, D. and Kuhlman, B. and Kortemme, T. and Gray, J. J.}, title = {The rosetta all-atom energy function for macromolecular modeling and design}, journal = {Journal of Chemical Theory and Computation}, year = {2017}, volume = {13}, issue = {6}, pages = {3031-3048}, doi = {10.1021/acs.jctc.7b00125} }""", +@article{10.1021/acs.jctc.7b00125, +author = {Alford, R. F. and Leaver‐Fay, A. and Jeliazkov, J. R. +and O’Meara, M. J. and DiMaio, F. and Park, H. and Shapovalov, M. V. +and Renfrew, P. D. and Mulligan, V. K. and Kappel, K. and Labonte, J. W. +and Pacella, M. S. and Bonneau, R. and Bradley, P. and Dunbrack, R. L. +and Das, R. and Baker, D. and Kuhlman, B. and Kortemme, T. and Gray, J. J.}, +title = {The rosetta all-atom energy function for macromolecular modeling and design}, +journal = {Journal of Chemical Theory and Computation}, +year = {2017}, +volume = {13}, +issue = {6}, +pages = {3031-3048}, +doi = {10.1021/acs.jctc.7b00125} +}""", } # TODO: refactor as general citation merging function From f044164cc13e8e833600a201b1fdbf4008e56202 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:23:57 +0800 Subject: [PATCH 33/63] fix(deepsource): wrap design citation lines --- src/REvoDesign/magician/designers/cart_ddg.py | 7 ++-- .../magician/designers/colabdesign.py | 36 +++++++++++++++++-- src/REvoDesign/phylogenetics/gremlin_tools.py | 22 ++++++++++-- 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/src/REvoDesign/magician/designers/cart_ddg.py b/src/REvoDesign/magician/designers/cart_ddg.py index 5eb695d8..a3709ce6 100644 --- a/src/REvoDesign/magician/designers/cart_ddg.py +++ b/src/REvoDesign/magician/designers/cart_ddg.py @@ -149,8 +149,11 @@ def scorer(self, mutant: Mutant | RosettaPyProteinSequence, **kwargs) -> float: __bibtex__ = rosetta_utils.copy_rosetta_citation( { "Cartesian-ddG": """@article{doi:10.1021/acs.jctc.6b00819, -author = {Park, Hahnbeom and Bradley, Philip and Greisen, Per Jr. and Liu, Yuan and Mulligan, Vikram Khipple and Kim, David E. and Baker, David and DiMaio, Frank}, -title = {Simultaneous Optimization of Biomolecular Energy Functions on Features from Small Molecules and Macromolecules}, +author = {Park, Hahnbeom and Bradley, Philip and Greisen, Per Jr. +and Liu, Yuan and Mulligan, Vikram Khipple and Kim, David E. and Baker, David +and DiMaio, Frank}, +title = {Simultaneous Optimization of Biomolecular Energy Functions on Features +from Small Molecules and Macromolecules}, journal = {Journal of Chemical Theory and Computation}, volume = {12}, number = {12}, diff --git a/src/REvoDesign/magician/designers/colabdesign.py b/src/REvoDesign/magician/designers/colabdesign.py index c5580801..319e66d2 100644 --- a/src/REvoDesign/magician/designers/colabdesign.py +++ b/src/REvoDesign/magician/designers/colabdesign.py @@ -134,7 +134,12 @@ def designer(self, *args, **kwargs): __bibtex__ = { "ProteinMPNN": r"""@article{ doi:10.1126/science.add2187, -author = {J. Dauparas and I. Anishchenko and N. Bennett and H. Bai and R. J. Ragotte and L. F. Milles and B. I. M. Wicky and A. Courbet and R. J. de Haas and N. Bethel and P. J. Y. Leung and T. F. Huddy and S. Pellock and D. Tischer and F. Chan and B. Koepnick and H. Nguyen and A. Kang and B. Sankaran and A. K. Bera and N. P. King and D. Baker }, +author = {J. Dauparas and I. Anishchenko and N. Bennett and H. Bai +and R. J. Ragotte and L. F. Milles and B. I. M. Wicky and A. Courbet +and R. J. de Haas and N. Bethel and P. J. Y. Leung and T. F. Huddy +and S. Pellock and D. Tischer and F. Chan and B. Koepnick +and H. Nguyen and A. Kang and B. Sankaran and A. K. Bera +and N. P. King and D. Baker }, title = {Robust deep learning–based protein sequence design using ProteinMPNN}, journal = {Science}, volume = {378}, @@ -144,7 +149,34 @@ def designer(self, *args, **kwargs): doi = {10.1126/science.add2187}, URL = {https://www.science.org/doi/abs/10.1126/science.add2187}, eprint = {https://www.science.org/doi/pdf/10.1126/science.add2187}, -abstract = {Although deep learning has revolutionized protein structure prediction, almost all experimentally characterized de novo protein designs have been generated using physically based approaches such as Rosetta. Here, we describe a deep learning–based protein sequence design method, ProteinMPNN, that has outstanding performance in both in silico and experimental tests. On native protein backbones, ProteinMPNN has a sequence recovery of 52.4\% compared with 32.9\% for Rosetta. The amino acid sequence at different positions can be coupled between single or multiple chains, enabling application to a wide range of current protein design challenges. We demonstrate the broad utility and high accuracy of ProteinMPNN using x-ray crystallography, cryo–electron microscopy, and functional studies by rescuing previously failed designs, which were made using Rosetta or AlphaFold, of protein monomers, cyclic homo-oligomers, tetrahedral nanoparticles, and target-binding proteins. Deep learning approaches such as Alphafold and Rosettafold have made reliable protein structure prediction broadly accessible. For the inverse problem, finding a sequence that folds to a desired structure, most approaches remain based on energy optimization. In two papers, a range of protein design problems were addressed through deep learning methods. Dauparas et al. built on recent deep learning protein design approaches to develop a method called ProteinMPNN. They validated designs experimentally and showed that ProteinMPNN can rescue previously failed designs made using Rosetta or Alphafold. Wicky et al. started from a random sequence and used Monte Carlo sequence search coupled with structure prediction by Alphafold to design cyclic homo-oligomers. Although the designs were generated to achieve stable expression, the sequences had to be regenerated using ProteinMPNN. This approach allowed for the design of a range of experimentally validated cyclic oligomers and paves the way for the design of increasingly complex assemblies. —VV A network-based protein design enables the generation of cyclic homo-oligomers across the nanoscopic scale.}} +abstract = {Although deep learning has revolutionized protein structure prediction, +almost all experimentally characterized de novo protein designs have been +generated using physically based approaches such as Rosetta. Here, we describe a +deep learning–based protein sequence design method, ProteinMPNN, that has +outstanding performance in both in silico and experimental tests. On native +protein backbones, ProteinMPNN has a sequence recovery of 52.4\% compared with +32.9\% for Rosetta. The amino acid sequence at different positions can be +coupled between single or multiple chains, enabling application to a wide range +of current protein design challenges. We demonstrate the broad utility and high +accuracy of ProteinMPNN using x-ray crystallography, cryo–electron microscopy, +and functional studies by rescuing previously failed designs, which were made +using Rosetta or AlphaFold, of protein monomers, cyclic homo-oligomers, +tetrahedral nanoparticles, and target-binding proteins. Deep learning approaches +such as Alphafold and Rosettafold have made reliable protein structure +prediction broadly accessible. For the inverse problem, finding a sequence that +folds to a desired structure, most approaches remain based on energy +optimization. In two papers, a range of protein design problems were addressed +through deep learning methods. Dauparas et al. built on recent deep learning +protein design approaches to develop a method called ProteinMPNN. They validated +designs experimentally and showed that ProteinMPNN can rescue previously failed +designs made using Rosetta or Alphafold. Wicky et al. started from a random +sequence and used Monte Carlo sequence search coupled with structure prediction +by Alphafold to design cyclic homo-oligomers. Although the designs were +generated to achieve stable expression, the sequences had to be regenerated +using ProteinMPNN. This approach allowed for the design of a range of +experimentally validated cyclic oligomers and paves the way for the design of +increasingly complex assemblies. —VV A network-based protein design enables the +generation of cyclic homo-oligomers across the nanoscopic scale.}} """ } diff --git a/src/REvoDesign/phylogenetics/gremlin_tools.py b/src/REvoDesign/phylogenetics/gremlin_tools.py index c5ea7cc6..4cb4ee56 100644 --- a/src/REvoDesign/phylogenetics/gremlin_tools.py +++ b/src/REvoDesign/phylogenetics/gremlin_tools.py @@ -476,7 +476,8 @@ def plot_w_o2a(self, resi) -> tuple[CoevolvedPair]: "GREMLIN": r"""@article{ doi:10.1073/pnas.1314045110, author = {Hetunandan Kamisetty and Sergey Ovchinnikov and David Baker }, -title = {Assessing the utility of coevolution-based residue–residue contact predictions in a sequence- and structure-rich era}, +title = {Assessing the utility of coevolution-based residue–residue contact +predictions in a sequence- and structure-rich era}, journal = {Proceedings of the National Academy of Sciences}, volume = {110}, number = {39}, @@ -485,5 +486,22 @@ def plot_w_o2a(self, resi) -> tuple[CoevolvedPair]: doi = {10.1073/pnas.1314045110}, URL = {https://www.pnas.org/doi/abs/10.1073/pnas.1314045110}, eprint = {https://www.pnas.org/doi/pdf/10.1073/pnas.1314045110}, -abstract = {Recently developed methods have shown considerable promise in predicting residue–residue contacts in protein 3D structures using evolutionary covariance information. However, these methods require large numbers of evolutionarily related sequences to robustly assess the extent of residue covariation, and the larger the protein family, the more likely that contact information is unnecessary because a reasonable model can be built based on the structure of a homolog. Here we describe a method that integrates sequence coevolution and structural context information using a pseudolikelihood approach, allowing more accurate contact predictions from fewer homologous sequences. We rigorously assess the utility of predicted contacts for protein structure prediction using large and representative sequence and structure databases from recent structure prediction experiments. We find that contact predictions are likely to be accurate when the number of aligned sequences (with sequence redundancy reduced to 90\%) is greater than five times the length of the protein, and that accurate predictions are likely to be useful for structure modeling if the aligned sequences are more similar to the protein of interest than to the closest homolog of known structure. These conditions are currently met by 422 of the protein families collected in the Pfam database.}}""", +abstract = {Recently developed methods have shown considerable promise in predicting +residue–residue contacts in protein 3D structures using evolutionary covariance +information. However, these methods require large numbers of evolutionarily +related sequences to robustly assess the extent of residue covariation, and the +larger the protein family, the more likely that contact information is +unnecessary because a reasonable model can be built based on the structure of a +homolog. Here we describe a method that integrates sequence coevolution and +structural context information using a pseudolikelihood approach, allowing more +accurate contact predictions from fewer homologous sequences. We rigorously +assess the utility of predicted contacts for protein structure prediction using +large and representative sequence and structure databases from recent structure +prediction experiments. We find that contact predictions are likely to be +accurate when the number of aligned sequences (with sequence redundancy reduced +to 90\%) is greater than five times the length of the protein, and that accurate +predictions are likely to be useful for structure modeling if the aligned +sequences are more similar to the protein of interest than to the closest +homolog of known structure. These conditions are currently met by 422 of the +protein families collected in the Pfam database.}}""", } From 6cb3db42e47b76874bc48ff41e2c3c987c17a490 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:28:10 +0800 Subject: [PATCH 34/63] fix(config): remove local workspace paths --- server/.env.example | 8 +- src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui | 2 +- src/REvoDesign/UI/UI.pyproject.user | 375 -------------------- src/REvoDesign/tools/rfdiffusion_tools.py | 2 +- 4 files changed, 6 insertions(+), 381 deletions(-) delete mode 100644 src/REvoDesign/UI/UI.pyproject.user diff --git a/server/.env.example b/server/.env.example index a3ef1155..9f2d72c5 100644 --- a/server/.env.example +++ b/server/.env.example @@ -15,19 +15,19 @@ RUNNER_IMAGE=revodesign-pssm-gremlin-non-root # Host paths — must exist on the Docker host # ============================================================================ # Root directory for uploads, results, and the SQLite task database. -SERVER_DIR=/Users/yyy/Documents/protein_design/REvoDesign/playground/server_test +SERVER_DIR=/srv/revodesign/server # Directory for Gunicorn and Celery log files. -LOG_DIR=/Users/yyy/Documents/protein_design/REvoDesign/playground/server_test/revodesign/server/logs +LOG_DIR=/srv/revodesign/logs # ============================================================================ # MSA databases — large read-only datasets mounted into the runner container # ============================================================================ # Prefix for the UniRef30 HHblits database (the directory containing the .cs219 # and .a3m files; omit extensions). -DB_UNIREF30=/Users/yyy/Documents/protein_design/REvoDesign/playground/miniuc/uc30/miniuc30 +DB_UNIREF30=/srv/revodesign/databases/uniref30/UniRef30_2023_02 # Prefix for the UniRef90 BLAST database (the directory containing the .pin/.phr # etc. files; omit extensions). -DB_UNIREF90=/Users/yyy/Documents/protein_design/REvoDesign/playground/miniuc/uc90/uniref90 +DB_UNIREF90=/srv/revodesign/databases/uniref90/uniref90 # ============================================================================ # Authentication — token signing and user database diff --git a/src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui b/src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui index 5e474170..bedd70f1 100644 --- a/src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui +++ b/src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui @@ -151,7 +151,7 @@ - /Users/yyy/Documents/protein_design/REvoDesign + diff --git a/src/REvoDesign/UI/UI.pyproject.user b/src/REvoDesign/UI/UI.pyproject.user deleted file mode 100644 index 82ab870d..00000000 --- a/src/REvoDesign/UI/UI.pyproject.user +++ /dev/null @@ -1,375 +0,0 @@ - - - - - - EnvironmentId - {0214636d-c5aa-4424-985a-d088db583ed6} - - - ProjectExplorer.Project.ActiveTarget - 0 - - - ProjectExplorer.Project.EditorSettings - - true - false - true - - Cpp - - CppGlobal - - - - QmlJS - - QmlJSGlobal - - - 2 - UTF-8 - false - 4 - false - 80 - true - true - 1 - false - true - false - 0 - true - true - 0 - 8 - true - false - 1 - true - true - true - *.md, *.MD, Makefile - false - true - true - - - - ProjectExplorer.Project.PluginSettings - - - true - false - true - true - true - true - - - 0 - true - - true - true - Builtin.DefaultTidyAndClazy - 5 - true - - - - true - - - - - ProjectExplorer.Project.Target.0 - - Desktop - Desktop (x86-darwin-generic-mach_o-64bit) - Desktop (x86-darwin-generic-mach_o-64bit) - {1e4bf9bd-468b-4836-ba8c-71a0373db4bd} - 0 - 0 - 0 - - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI - - - true - Python.PysideBuildStep - - 1 - Build - Build - ProjectExplorer.BuildSteps.Build - - - 0 - Clean - Clean - ProjectExplorer.BuildSteps.Clean - - 2 - false - - false - - build - Python.PySideBuildConfiguration - - 1 - - - 0 - Deploy - Deploy - ProjectExplorer.BuildSteps.Deploy - - 1 - - false - ProjectExplorer.DefaultDeployConfiguration - - 1 - - true - true - true - - 2 - - false - Ui_REvoDesign.py - PythonEditor.RunConfiguration./Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - true - true - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI - - 1 - - - - ProjectExplorer.Project.Target.1 - - Desktop - Desktop (arm-darwin-generic-mach_o-64bit) - Desktop (arm-darwin-generic-mach_o-64bit) - {ce734272-b7d7-4c06-994a-976506a526c1} - 0 - 0 - 0 - - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI - - - true - Python.PysideBuildStep - - 1 - Build - Build - ProjectExplorer.BuildSteps.Build - - - 0 - Clean - Clean - ProjectExplorer.BuildSteps.Clean - - 2 - false - - false - - build - Python.PySideBuildConfiguration - - 1 - - - 0 - Deploy - Deploy - ProjectExplorer.BuildSteps.Deploy - - 1 - - false - ProjectExplorer.DefaultDeployConfiguration - - 1 - - true - true - true - - 2 - - false - Ui_REvoDesign.py - PythonEditor.RunConfiguration./Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - true - true - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI - - 1 - - - - ProjectExplorer.Project.Target.2 - - Desktop - Desktop (arm-darwin-generic-mach_o-32bit) - Desktop (arm-darwin-generic-mach_o-32bit) - {1c5021eb-a3b5-4932-90c6-41451ea69647} - 0 - 0 - 0 - - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI - - - true - Python.PysideBuildStep - - 1 - Build - Build - ProjectExplorer.BuildSteps.Build - - - 0 - Clean - Clean - ProjectExplorer.BuildSteps.Clean - - 2 - false - - false - - build - Python.PySideBuildConfiguration - - 1 - - - 0 - Deploy - Deploy - ProjectExplorer.BuildSteps.Deploy - - 1 - - false - ProjectExplorer.DefaultDeployConfiguration - - 1 - - true - true - true - - 2 - - false - Ui_REvoDesign.py - PythonEditor.RunConfiguration./Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - true - true - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI - - 1 - - - - ProjectExplorer.Project.Target.3 - - Desktop - Desktop (x86-darwin-generic-mach_o-32bit) - Desktop (x86-darwin-generic-mach_o-32bit) - {3ad391aa-4d57-4fd8-94ae-5f8e44f65cc1} - 0 - 0 - 0 - - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI - - - true - Python.PysideBuildStep - - 1 - Build - Build - ProjectExplorer.BuildSteps.Build - - - 0 - Clean - Clean - ProjectExplorer.BuildSteps.Clean - - 2 - false - - false - - build - Python.PySideBuildConfiguration - - 1 - - - 0 - Deploy - Deploy - ProjectExplorer.BuildSteps.Deploy - - 1 - - false - ProjectExplorer.DefaultDeployConfiguration - - 1 - - true - true - true - - 2 - - false - Ui_REvoDesign.py - PythonEditor.RunConfiguration./Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI/Ui_REvoDesign.py - true - true - /Users/yyy/Documents/protein_design/REvoDesign/REvoDesign/UI - - 1 - - - - ProjectExplorer.Project.TargetCount - 4 - - - ProjectExplorer.Project.Updater.FileVersion - 22 - - - Version - 22 - - diff --git a/src/REvoDesign/tools/rfdiffusion_tools.py b/src/REvoDesign/tools/rfdiffusion_tools.py index b1a43fc1..8b0d8829 100644 --- a/src/REvoDesign/tools/rfdiffusion_tools.py +++ b/src/REvoDesign/tools/rfdiffusion_tools.py @@ -278,6 +278,6 @@ def plot_potential_field(self, grid_size=200, margin=10, save_to: str = "default # # Example usage: # visualizer = SubstratePotentialVisualizer( -# pdb_path="/Users/yyy/Documents/protein_design/REvoDesign-test-data/1SUO.pdb", lig_key="HEM" +# pdb_path="/path/to/1SUO.pdb", lig_key="HEM" # ) # visualizer.plot_potential_field(save_to='HEM.png') From d04ec835822289d7c181a9d220926e84fc09b8c5 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:39:24 +0800 Subject: [PATCH 35/63] fix(deepsource): close file handles deterministically --- server/scripts/GREMLIN_TFv1.py | 24 ++++----- server/scripts/make_paired_MSA_simple.py | 53 +++++++++++-------- src/REvoDesign/citations/citation_manager.py | 3 +- src/REvoDesign/clusters/cluster_sequence.py | 4 +- src/REvoDesign/common/mutant_visualise.py | 17 +++--- src/REvoDesign/common/profile_parsers.py | 11 ++-- src/REvoDesign/logger/logger.py | 4 +- src/REvoDesign/phylogenetics/revo_designer.py | 3 +- .../shortcuts/tools/ligand_converters.py | 3 +- src/REvoDesign/tools/customized_widgets.py | 14 ++--- src/REvoDesign/tools/mutant_tools.py | 3 +- 11 files changed, 77 insertions(+), 62 deletions(-) diff --git a/server/scripts/GREMLIN_TFv1.py b/server/scripts/GREMLIN_TFv1.py index 1b3238b2..acef3472 100644 --- a/server/scripts/GREMLIN_TFv1.py +++ b/server/scripts/GREMLIN_TFv1.py @@ -83,17 +83,16 @@ def parse_fasta(filename, limit=-1): """function to parse fasta""" header = [] sequence = [] - lines = open(filename) - for line in lines: - line = line.rstrip() - if line[0] == ">": - if len(header) == limit: - break - header.append(line[1:]) - sequence.append([]) - else: - sequence[-1].append(line) - lines.close() + with open(filename) as lines: + for line in lines: + line = line.rstrip() + if line[0] == ">": + if len(header) == limit: + break + header.append(line[1:]) + sequence.append([]) + else: + sequence[-1].append(line) sequence = ["".join(seq) for seq in sequence] return np.array(header), np.array(sequence) @@ -373,7 +372,8 @@ def plot_mtx(mtx, key="zscore", vmin=1, vmax=3): # save mtx file -pickle.dump(mrf, open(f"{pth}/{instance}.GREMLIN.mrf.pkl", "wb")) +with open(f"{pth}/{instance}.GREMLIN.mrf.pkl", "wb") as mrf_file: + pickle.dump(mrf, mrf_file) mtx = get_mtx(mrf) plot_mtx(mtx) diff --git a/server/scripts/make_paired_MSA_simple.py b/server/scripts/make_paired_MSA_simple.py index b5bbd72d..5093474d 100644 --- a/server/scripts/make_paired_MSA_simple.py +++ b/server/scripts/make_paired_MSA_simple.py @@ -34,13 +34,38 @@ def calc_seqID(query, cand): def read_a3m(fn): # read sequences in a3m file # only take one (having the highest seqID to query) per each taxID + if fn.split(".")[-1] == "gz": + with gzip.open(fn, "rt") as fp: + query, tmp = _read_a3m_taxa(fp) + else: + with open(fn) as fp: + query, tmp = _read_a3m_taxa(fp) + + query_in_num = seq2number(query) + a3m = {} + for TaxID in tmp: + if len(tmp[TaxID]) < 1: + continue + if len(tmp[TaxID]) < 2: + a3m[TaxID] = tmp[TaxID][0] + continue + # Get the best sequence only + score_s = list() + for _seqID, seq in tmp[TaxID]: + seq_in_num = seq2number(seq) + score = calc_seqID(query_in_num, seq_in_num) + score_s.append(score) + # + idx = np.argmax(score_s) + a3m[TaxID] = tmp[TaxID][idx] + + return query, a3m + + +def _read_a3m_taxa(fp): is_first = True is_ignore = False tmp = {} - if fn.split(".")[-1] == "gz": - fp = gzip.open(fn, "rt") - else: - fp = open(fn) for line in fp: if line[0] == ">": @@ -66,25 +91,7 @@ def read_a3m(fn): else: tmp[TaxID].append((seqID, line.strip())) - query_in_num = seq2number(query) - a3m = {} - for TaxID in tmp: - if len(tmp[TaxID]) < 1: - continue - if len(tmp[TaxID]) < 2: - a3m[TaxID] = tmp[TaxID][0] - continue - # Get the best sequence only - score_s = list() - for _seqID, seq in tmp[TaxID]: - seq_in_num = seq2number(seq) - score = calc_seqID(query_in_num, seq_in_num) - score_s.append(score) - # - idx = np.argmax(score_s) - a3m[TaxID] = tmp[TaxID][idx] - - return query, a3m + return query, tmp if len(sys.argv) == 1: diff --git a/src/REvoDesign/citations/citation_manager.py b/src/REvoDesign/citations/citation_manager.py index 99855f0f..3bafa319 100644 --- a/src/REvoDesign/citations/citation_manager.py +++ b/src/REvoDesign/citations/citation_manager.py @@ -99,7 +99,8 @@ def output(self, cwd: str = "."): f'{time.strftime("%Y%m%d", time.localtime())}.bib', ) os.makedirs(os.path.dirname(citation_output), exist_ok=True) - bibtexparser.write_file(file=open(citation_output, "w", encoding="utf8"), library=library) + with open(citation_output, "w", encoding="utf8") as citation_file: + bibtexparser.write_file(file=citation_file, library=library) logging.info(f"Citation is created at {citation_output}") def dismiss(self, modulename: str): diff --git a/src/REvoDesign/clusters/cluster_sequence.py b/src/REvoDesign/clusters/cluster_sequence.py index bf1d5053..f584fba4 100644 --- a/src/REvoDesign/clusters/cluster_sequence.py +++ b/src/REvoDesign/clusters/cluster_sequence.py @@ -243,8 +243,8 @@ def override_cluster_centers_with_rosetta(self, rosetta_results): def _calculate_pairwise_scores(self, progressbar) -> np.ndarray: from REvoDesign.tools.customized_widgets import QtParallelExecutor - handle = open(self.fastafile) - self.records = list(SeqIO.parse(handle, "fasta")) + with open(self.fastafile) as handle: + self.records = list(SeqIO.parse(handle, "fasta")) if self.shuffle_variant: # Deterministic clustering order is controlled by the UI random-seed setting. rng = random.Random(self.random_seed) # nosec B311 diff --git a/src/REvoDesign/common/mutant_visualise.py b/src/REvoDesign/common/mutant_visualise.py index 077f430e..09be02c1 100644 --- a/src/REvoDesign/common/mutant_visualise.py +++ b/src/REvoDesign/common/mutant_visualise.py @@ -218,14 +218,15 @@ def _get_mutation_data(self): return pd.read_fwf(self.mutfile) if self.mutfile.lower().endswith((".fasta", ".fas", ".fa")): # Read mutant data from fasta file. - _mutation_objs = [ - extract_mutant_from_sequences( - mutant_sequence=str(mut_record.seq), - wt_sequences=self.designable_sequences, - chain_id=self.chain_id, - ) - for mut_record in SeqIO.parse(open(self.mutfile), format="fasta") - ] + with open(self.mutfile) as mutant_file: + _mutation_objs = [ + extract_mutant_from_sequences( + mutant_sequence=str(mut_record.seq), + wt_sequences=self.designable_sequences, + chain_id=self.chain_id, + ) + for mut_record in SeqIO.parse(mutant_file, format="fasta") + ] return pd.DataFrame.from_dict( {self.key_col: [mut_obj.short_mutant_id for mut_obj in _mutation_objs if mut_obj is not None]} diff --git a/src/REvoDesign/common/profile_parsers.py b/src/REvoDesign/common/profile_parsers.py index 6b43fb9b..768c984f 100644 --- a/src/REvoDesign/common/profile_parsers.py +++ b/src/REvoDesign/common/profile_parsers.py @@ -153,11 +153,12 @@ def convert_PSSM_file_to_df(input_pssm_file): PSSM_Alphabet = "ARNDCQEGHILKMFPSTWYV" # Fetch table header of PSSM c = 0 - for line in open(input_pssm_file): - pssm_header = line - c += 1 - if c == 3: - break + with open(input_pssm_file) as pssm_file: + for line in pssm_file: + pssm_header = line + c += 1 + if c == 3: + break logging.info(pssm_header) diff --git a/src/REvoDesign/logger/logger.py b/src/REvoDesign/logger/logger.py index e526c188..f7a16062 100644 --- a/src/REvoDesign/logger/logger.py +++ b/src/REvoDesign/logger/logger.py @@ -330,11 +330,11 @@ def get_current_logger_level(channel: str = "root"): key = f"handlers.{channel}.level" try: return ConfigBus().get_value(key, str, reject_none=True, cfg="logger") - except Exception as e: + except Exception: logging.error(f"Logger channel {channel} does not exist like key: {key}") logging.error(f"Available channels: {list_all_logger_channels()}") logging.error(f'All config: {ConfigBus().cfg_group["logger"]}') - raise e + raise def list_all_logger_levels(): diff --git a/src/REvoDesign/phylogenetics/revo_designer.py b/src/REvoDesign/phylogenetics/revo_designer.py index fdfd6c18..0ba6d658 100644 --- a/src/REvoDesign/phylogenetics/revo_designer.py +++ b/src/REvoDesign/phylogenetics/revo_designer.py @@ -218,7 +218,8 @@ def plot_custom_indices_segments( mutation_json_fp = f"{self.pwd}/mutations_design_profile/{file_name}.json" mutation_png_fp = f"{self.pwd}/mutations_design_profile/{file_name}.png" - json.dump(mutation_candidates, open(mutation_json_fp, "w"), indent=2) + with open(mutation_json_fp, "w") as mutation_json_file: + json.dump(mutation_candidates, mutation_json_file, indent=2) plt.savefig(mutation_png_fp) plt.close() diff --git a/src/REvoDesign/shortcuts/tools/ligand_converters.py b/src/REvoDesign/shortcuts/tools/ligand_converters.py index b9a928f1..4f2451e2 100644 --- a/src/REvoDesign/shortcuts/tools/ligand_converters.py +++ b/src/REvoDesign/shortcuts/tools/ligand_converters.py @@ -62,7 +62,8 @@ def shortcut_smiles_conformer_batch( Runs the smiles_conformer_batch function with parameters collected from the dialog. """ - smi = json.load(open(smiles)) + with open(smiles) as smiles_file: + smi = json.load(smiles_file) with timing("Get SMILES Conformers (Many)"): smiles_conformer_batch( smi=smi, diff --git a/src/REvoDesign/tools/customized_widgets.py b/src/REvoDesign/tools/customized_widgets.py index 33dd4e5d..933959e3 100644 --- a/src/REvoDesign/tools/customized_widgets.py +++ b/src/REvoDesign/tools/customized_widgets.py @@ -2514,7 +2514,8 @@ def _load_json_file(self, selected_file): from REvoDesign import __version__ # load back all asked values from a json file - contents_to_load: dict[str, dict[str, Any]] = json.load(open(selected_file)) + with open(selected_file) as selected_file_handle: + contents_to_load: dict[str, dict[str, Any]] = json.load(selected_file_handle) if contents_to_load["metadata"]["__window__"] != self.windowTitle(): logging.error( f"The recipe is made for Dialog `{contents_to_load['metadata']['__window__']}`, " @@ -2744,11 +2745,12 @@ def ask_for_multiple_values_as_json() -> str: json_fp = os.path.join("json_multi_input", f"{data_id}.json") os.makedirs(os.path.dirname(json_fp), exist_ok=True) - json.dump( - obj=data.asdict, - fp=open(json_fp, "w"), - indent=4, - ) + with open(json_fp, "w") as json_file: + json.dump( + obj=data.asdict, + fp=json_file, + indent=4, + ) return json_fp diff --git a/src/REvoDesign/tools/mutant_tools.py b/src/REvoDesign/tools/mutant_tools.py index 95c635f7..953b4e65 100644 --- a/src/REvoDesign/tools/mutant_tools.py +++ b/src/REvoDesign/tools/mutant_tools.py @@ -487,7 +487,8 @@ def process_mutations(data): def read_profile_design_mutations(filename): - data = json.load(open(filename)) + with open(filename) as mutation_file: + data = json.load(mutation_file) return process_mutations(data) From aecdce8456779a6aacd1875e7e9d938482e89c65 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:42:45 +0800 Subject: [PATCH 36/63] fix(deepsource): route tracebacks through logger --- src/REvoDesign/REvoDesign.py | 13 +++++-------- src/REvoDesign/clients/QtSocketConnector.py | 7 ++----- src/REvoDesign/phylogenetics/evo_mutator.py | 6 ++---- src/REvoDesign/phylogenetics/gremlin_tools.py | 9 ++++++--- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/REvoDesign/REvoDesign.py b/src/REvoDesign/REvoDesign.py index 172bd2f2..e4776ed6 100644 --- a/src/REvoDesign/REvoDesign.py +++ b/src/REvoDesign/REvoDesign.py @@ -11,7 +11,6 @@ import gc import os import tempfile -import traceback import warnings # using partial module to reduce duplicate code. @@ -1190,7 +1189,7 @@ def multi_mutagenesis_design_auto(self): self.multi_mutagenesis_design_stop_design() self.multi_mutagenesis_design_save_design() except Exception: - traceback.print_exc() + self.logging.exception("Failed to generate multi-mutagenesis designs.") # Tab Interact via GREMLIN def load_gremlin_mrf(self): @@ -1317,9 +1316,8 @@ def toggle_ws_server_mode(self): self.logging.warning("Server is already stopped. Do nothing.") return self.ws_server.stop_server() - except Exception as e: - self.logging.warning(e) - traceback.print_exc() + except Exception: + self.logging.exception("Failed to toggle WebSocket server connection.") self.logging.warning(f'Server status: {"ON" if self.ws_server.is_running else "OFF"}') @@ -1363,9 +1361,8 @@ def toggle_ws_client_connection(self, connect: bool = True): self.ws_client_connect_to_server() else: self.ws_client_disconnect_from_server() - except Exception as e: - self.logging.warning(e) - traceback.print_exc() + except Exception: + self.logging.exception("Failed to toggle WebSocket client connection.") self.logging.warning(f'Client status: {"ON" if self.ws_client.connected else "OFF"}') diff --git a/src/REvoDesign/clients/QtSocketConnector.py b/src/REvoDesign/clients/QtSocketConnector.py index 5e03d32e..86ba6621 100644 --- a/src/REvoDesign/clients/QtSocketConnector.py +++ b/src/REvoDesign/clients/QtSocketConnector.py @@ -20,7 +20,6 @@ import pickle # nosec B403 import socket import time -import traceback import warnings from collections.abc import Iterable, Mapping from dataclasses import dataclass, field @@ -810,8 +809,7 @@ def connect_to_server(self): logging.info("Connection established.") except Exception: - logging.error("Unexpected error during connection:") - traceback.print_exc() + logging.exception("Unexpected error during connection.") @property def connected(self): @@ -829,8 +827,7 @@ def close_connection(self): refresh_tree_widget(user_tree={}, treeWidget_ws_peers=self.treeWidget_ws_peers) except Exception: - traceback.print_exc() - logging.error("Client disconnection failed.") + logging.exception("Client disconnection failed.") @property def server_is_reachable(self): diff --git a/src/REvoDesign/phylogenetics/evo_mutator.py b/src/REvoDesign/phylogenetics/evo_mutator.py index 29ec685e..281ed6d7 100644 --- a/src/REvoDesign/phylogenetics/evo_mutator.py +++ b/src/REvoDesign/phylogenetics/evo_mutator.py @@ -15,7 +15,6 @@ import asyncio import itertools import os -import traceback import warnings from dataclasses import dataclass from functools import partial @@ -212,7 +211,7 @@ def run_mutant_loading_from_profile(self): cmd.save(output_pse) except Exception: - traceback.print_exc() + logging.exception("Error while running the design workflow.") finally: CitationManager().output() @@ -324,8 +323,7 @@ def visualize_mutants(self): cmd.save(output_pse) except Exception: - logging.error("Error while running the visualization: ") - traceback.print_exc() + logging.exception("Error while running the visualization.") finally: CitationManager().output() diff --git a/src/REvoDesign/phylogenetics/gremlin_tools.py b/src/REvoDesign/phylogenetics/gremlin_tools.py index 4cb4ee56..b0c94f03 100644 --- a/src/REvoDesign/phylogenetics/gremlin_tools.py +++ b/src/REvoDesign/phylogenetics/gremlin_tools.py @@ -21,7 +21,6 @@ """ import os -import traceback import warnings from dataclasses import dataclass, field from typing import Literal @@ -379,8 +378,12 @@ def plot_w(self, i: int, j: int, i_aa: str, j_aa: str, idx: int = 0) -> Coevolve wt_i_index = self.alphabet.index(wt_i_aa) wt_j_index = self.alphabet.index(wt_j_aa) except ValueError: - traceback.print_exc() - logging.error(f"Error occured while processing '{wt_i_aa=}' or '{wt_j_aa=}' from {self.alphabet=}") + logging.exception( + "Error occured while processing %r or %r from %s", + wt_i_aa, + wt_j_aa, + self.alphabet, + ) # early return to skip ploting warnings.warn(issues.BadDataWarning(f"Bad pair: {str(a_pair)}"), stacklevel=2) return None From af708b2f166674ea018dc29fc36c970c3c243931 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:49:53 +0800 Subject: [PATCH 37/63] fix(shortcuts): clean up conformer preview processes --- dev/TODO.md | 36 ++++---- src/REvoDesign/shortcuts/function_utils.py | 32 ++++++- tests/shortcuts/test_function_utils.py | 99 ++++++++++++++++++++++ 3 files changed, 147 insertions(+), 20 deletions(-) create mode 100644 tests/shortcuts/test_function_utils.py diff --git a/dev/TODO.md b/dev/TODO.md index 85e50a29..8a1f8fb0 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -24,39 +24,37 @@ `src/REvoDesign/logger/logger.py:1-120` configures logging, creates directories under `user_log_path`, and spins up a background `QueueListener` during import. Simply touching `REvoDesign.logger` writes to the host filesystem and leaves live threads around; lazily initialize logging from the entrypoint instead. 12. **High – WorkerThread drops exceptions** `src/REvoDesign/tools/package_manager.py:2296-2345` executes `self.func` in `QThread.run()` without trapping exceptions or emitting an error signal. Failures just print to stderr while the UI thinks work succeeded. Catch exceptions, forward them through a signal, and surface them via `notify_signal`. -13. **Medium – Conformer preview leaks PyMOL processes** - `src/REvoDesign/shortcuts/function_utils.py:17-57` launches a brand-new `pymol -xi` process for every preview and never registers or terminates it, so repeated clicks accumulate orphan PyMOL instances. Reuse the existing PyMOL session or track the subprocess via `RunningProcessRegistry` so abort/exit can kill it. -14. **High – Tests still touch the real config tree** +13. **High – Tests still touch the real config tree** `tests/conftest.py:53-124` calls `set_REvoDesign_config_file()` at import time, which may pop up Qt dialogs and can delete `/Users/yyy/Library/Application Support/REvoDesign/config` if the user confirms. Mock `platformdirs.user_data_dir`/`user_cache_dir` so tests operate entirely inside the repo. -15. **High – RFdiffusion helper forces Qt5Agg backend globally** +14. **High – RFdiffusion helper forces Qt5Agg backend globally** `src/REvoDesign/tools/rfdiffusion_tools.py:1-7` unconditionally runs `matplotlib.use("Qt5Agg")` on import, breaking headless builds and notebooks that rely on Agg. Only set the backend when a GUI feature is invoked and honor `MPLBACKEND`. -16. **High – Download registry accepts unsigned payloads** +15. **High – Download registry accepts unsigned payloads** `src/REvoDesign/tools/download_registry.py:133-152` leaves registry entries as `None`, so `pooch` skips hash verification and trusts whatever the server returns. Require hashes for every asset or refuse to download; otherwise a MITM can ship arbitrary binaries. -17. **High – RFdiffusion weight downloads block the UI** +16. **High – RFdiffusion weight downloads block the UI** `src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py:90-151` fetches multi‑GB checkpoints synchronously inside `RfDiffusion.pick_model`, freezing PyMOL while `pooch` downloads. Move the download to `run_worker_thread_in_pool` and provide progress/abort hooks. -18. **High – DglSolver silently runs pip install** +17. **High – DglSolver silently runs pip install** `src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py:33-70` detects CUDA and immediately executes `python -m pip install dgl==2.2.1` in the current interpreter without prompting or isolating the environment. Prompt the user and install inside a managed env/container so you don’t mutate the host Python unexpectedly. -19. **High – Server keeps plaintext credentials in repo** +18. **High – Server keeps plaintext credentials in repo** `server/pssm_gremlin/pssm_gremlin.py:17-36` parses `users.txt` where each line is `username:password` and only hashes them at runtime, so the file on disk stores the raw passwords. Store hashed secrets (or use a proper auth backend) to avoid leaking credentials if the repo/server is compromised. -20. **High – GREMLIN web service hardcodes cluster-only paths** +19. **High – GREMLIN web service hardcodes cluster-only paths** `server/pssm_gremlin/pssm_gremlin.py:38-120` assumes `/mnt/data/...`, `/mnt/db/...`, and `redis://localhost` exist and derives Docker mounts from `/home/{os.getlogin()}`. With no configuration hooks the service cannot run off that exact cluster. Externalize broker paths, DBs, and output dirs via a config file/env vars. -21. **Medium – Config-driven env overrides clobber user variables** +20. **Medium – Config-driven env overrides clobber user variables** `src/REvoDesign/driver/environ_register.py:8-39` iterates `environ.yaml` and unconditionally sets each key in `os.environ`, overwriting whatever the shell provided (proxy settings, CUDA flags, etc.). Respect existing values or mark overrides explicitly in config. -22. **High – Archive extraction is vulnerable to path traversal** +21. **High – Archive extraction is vulnerable to path traversal** `src/REvoDesign/tools/utils.py:592-627` calls `tarfile.ZipFile(...).extractall()` on user-selected archives with no member filtering, so a malicious tar can overwrite arbitrary files. Validate members (reject `..`/absolute paths) or use Python’s newer `tarfile` filters. -23. **High – SSL certificate manager writes weak certs** +22. **High – SSL certificate manager writes weak certs** `src/REvoDesign/tools/ssl_certificates.py:34-115` generates self-signed certs with a fixed serial number, week-long validity, and default filesystem permissions under a shared cache dir. Anyone on the box can read or overwrite the keys. Randomize serials, tighten permissions, and allow the validity to be configured. -24. **High – “Background” workers busy-wait on the GUI thread** +23. **High – “Background” workers busy-wait on the GUI thread** `src/REvoDesign/tools/package_manager.py:2635-2661` starts a `WorkerThread` but then loops on `work_thread.isFinished()` calling `refresh_window()`/`time.sleep`, so the main thread stays blocked and abort buttons can’t repaint. Switch to signal-driven completion instead of polling. -25. **High – Aborting a worker interrupts the whole PyMOL session** +24. **High – Aborting a worker interrupts the whole PyMOL session** `src/REvoDesign/tools/package_manager.py:2223-2244` handles every abort click by calling `cmd.interrupt()`, which cancels whatever PyMOL is currently doing—even unrelated jobs. Terminate only the subprocesses registered for that worker rather than nuking the global interpreter. -26. **High – Monaco editor logs its auth token** +25. **High – Monaco editor logs its auth token** `src/REvoDesign/editor/monaco/server.py:71-78` generates a `SECRET_TOKEN` and immediately logs it, so anyone with read access to the log file can hijack the editor backend. Never log secrets and rotate them when the server restarts. -27. **High – WebSocket clients hand over the shared auth key on request** +26. **High – WebSocket clients hand over the shared auth key on request** `src/REvoDesign/clients/QtSocketConnector.py:813-863` responds to a `RequireKey` message by sending `self.authentication_key` before the server proves its identity. A malicious host can simply ask for the key and impersonate everyone. Authenticate the server (TLS + pinned cert) and only send a key through an already-authenticated channel. -28. **Medium – Git tag fetches are synchronous and unauthenticated** +27. **Medium – Git tag fetches are synchronous and unauthenticated** `src/REvoDesign/tools/package_manager.py:2349-2380` hits the public GitHub tags API on startup without auth, caching, or error UI. When the rate limit is exceeded the manager freezes for ~10s and users see nothing. Add caching, a timeout, and a friendly error surface. -29. **Medium – pip installs run on the UI thread** +28. **Medium – pip installs run on the UI thread** `src/REvoDesign/tools/package_manager.py:861-925` calls `pip install/uninstall` directly from menu actions, blocking PyMOL for minutes during heavy installs. Dispatch these calls to `run_worker_thread_in_pool` and stream progress to the dashboard. -30. **Medium – Archive browsing keeps every extraction forever** +29. **Medium – Archive browsing keeps every extraction forever** `src/REvoDesign/driver/file_dialog.py:240-269` always expands compressed inputs into `/expanded_compressed_files/` with no cleanup or quota. Repeated browsing bloats the session dir and a malicious 20 GB tar can fill the disk. Extract into a temp dir and delete it once the dialog closes. diff --git a/src/REvoDesign/shortcuts/function_utils.py b/src/REvoDesign/shortcuts/function_utils.py index 47c96095..28be7b13 100644 --- a/src/REvoDesign/shortcuts/function_utils.py +++ b/src/REvoDesign/shortcuts/function_utils.py @@ -7,6 +7,7 @@ Utils for shortcuts """ +import atexit import os import shutil @@ -20,6 +21,33 @@ from REvoDesign import ROOT_LOGGER logging = ROOT_LOGGER.getChild(__name__) +_PREVIEW_PROCESSES: set[subprocess.Popen[bytes]] = set() + + +def _prune_finished_preview_processes() -> None: + for process in tuple(_PREVIEW_PROCESSES): + if process.poll() is not None: + _PREVIEW_PROCESSES.discard(process) + + +def cleanup_conformer_preview_processes(timeout: float = 2.0) -> None: + """Terminate PyMOL preview processes launched by this module.""" + for process in tuple(_PREVIEW_PROCESSES): + try: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=timeout) + except Exception as exc: + logging.debug("Could not stop PyMOL preview process %s: %s", process, exc) + finally: + _PREVIEW_PROCESSES.discard(process) + + +atexit.register(cleanup_conformer_preview_processes) def _resolve_pymol_executable() -> str: @@ -66,12 +94,14 @@ def visualize_conformer_sdf(sdf_file_path: str, show_conformer: Literal["New Win # Explicitly call a new PyMOL instance in the background pymol_command = [_resolve_pymol_executable(), "-xi", pml_file_path] # Launches a user-requested PyMOL preview with an argv list and a resolved executable. - subprocess.Popen( # nosec B603 + _prune_finished_preview_processes() + process = subprocess.Popen( # nosec B603 pymol_command, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) + _PREVIEW_PROCESSES.add(process) logging.info("PyMOL launched in the background with %s.", sdf_file_path) diff --git a/tests/shortcuts/test_function_utils.py b/tests/shortcuts/test_function_utils.py new file mode 100644 index 00000000..592a58ae --- /dev/null +++ b/tests/shortcuts/test_function_utils.py @@ -0,0 +1,99 @@ +# Copyright (c) 2026 The REvoDesign Developers. +# Distributed under the terms of the GNU General Public License v3.0. +# SPDX-License-Identifier: GPL-3.0-only + +from __future__ import annotations + +import subprocess + +from REvoDesign.shortcuts import function_utils + + +class DummyProcess: + def __init__(self, returncode: int | None = None): + self.returncode = returncode + self.terminated = False + self.killed = False + self.wait_calls: list[float | None] = [] + + def poll(self): + return self.returncode + + def terminate(self): + self.terminated = True + self.returncode = 0 + + def kill(self): + self.killed = True + self.returncode = 0 + + def wait(self, timeout=None): + self.wait_calls.append(timeout) + return self.returncode + + +class HangingProcess(DummyProcess): + def wait(self, timeout=None): + self.wait_calls.append(timeout) + if not self.killed: + raise subprocess.TimeoutExpired(cmd="pymol", timeout=timeout) + return 0 + + +def test_visualize_conformer_sdf_tracks_preview_process(monkeypatch, tmp_path): + sdf_file = tmp_path / "ligand.sdf" + sdf_file.write_text("mock", encoding="utf-8") + process = DummyProcess() + calls = [] + + monkeypatch.setattr(function_utils.shutil, "which", lambda name: f"/usr/bin/{name}") + + def fake_popen(*args, **kwargs): + calls.append((args, kwargs)) + return process + + monkeypatch.setattr(function_utils.subprocess, "Popen", fake_popen) + function_utils._PREVIEW_PROCESSES.clear() + + function_utils.visualize_conformer_sdf(str(sdf_file), "New Window") + + assert process in function_utils._PREVIEW_PROCESSES + assert calls[0][0][0] == [ + "/usr/bin/pymol", + "-xi", + str(tmp_path / "ligand_load_to_preview.pml"), + ] + assert calls[0][1]["stdin"] is subprocess.DEVNULL + + function_utils.cleanup_conformer_preview_processes() + + +def test_visualize_conformer_sdf_prunes_finished_preview_processes(monkeypatch, tmp_path): + sdf_file = tmp_path / "ligand.sdf" + sdf_file.write_text("mock", encoding="utf-8") + finished = DummyProcess(returncode=0) + running = DummyProcess() + + monkeypatch.setattr(function_utils.shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr(function_utils.subprocess, "Popen", lambda *args, **kwargs: running) + function_utils._PREVIEW_PROCESSES.clear() + function_utils._PREVIEW_PROCESSES.add(finished) + + function_utils.visualize_conformer_sdf(str(sdf_file), "New Window") + + assert finished not in function_utils._PREVIEW_PROCESSES + assert running in function_utils._PREVIEW_PROCESSES + + function_utils.cleanup_conformer_preview_processes() + + +def test_cleanup_conformer_preview_processes_kills_stubborn_process(): + process = HangingProcess() + function_utils._PREVIEW_PROCESSES.clear() + function_utils._PREVIEW_PROCESSES.add(process) + + function_utils.cleanup_conformer_preview_processes(timeout=0.01) + + assert process.terminated is True + assert process.killed is True + assert process not in function_utils._PREVIEW_PROCESSES From f02ccc354a9054c39156ed226e6c997d39f26536 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:56:43 +0800 Subject: [PATCH 38/63] fix(security): harden archive member validation --- dev/TODO.md | 18 ++++++++---------- src/REvoDesign/tools/utils.py | 14 ++++++++++++-- tests/data/test_data.py | 7 ++----- tests/tools/test_utils.py | 26 +++++++++++++++++++++++++- 4 files changed, 47 insertions(+), 18 deletions(-) diff --git a/dev/TODO.md b/dev/TODO.md index 8a1f8fb0..7e7fb115 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -40,21 +40,19 @@ `server/pssm_gremlin/pssm_gremlin.py:38-120` assumes `/mnt/data/...`, `/mnt/db/...`, and `redis://localhost` exist and derives Docker mounts from `/home/{os.getlogin()}`. With no configuration hooks the service cannot run off that exact cluster. Externalize broker paths, DBs, and output dirs via a config file/env vars. 20. **Medium – Config-driven env overrides clobber user variables** `src/REvoDesign/driver/environ_register.py:8-39` iterates `environ.yaml` and unconditionally sets each key in `os.environ`, overwriting whatever the shell provided (proxy settings, CUDA flags, etc.). Respect existing values or mark overrides explicitly in config. -21. **High – Archive extraction is vulnerable to path traversal** - `src/REvoDesign/tools/utils.py:592-627` calls `tarfile.ZipFile(...).extractall()` on user-selected archives with no member filtering, so a malicious tar can overwrite arbitrary files. Validate members (reject `..`/absolute paths) or use Python’s newer `tarfile` filters. -22. **High – SSL certificate manager writes weak certs** +21. **High – SSL certificate manager writes weak certs** `src/REvoDesign/tools/ssl_certificates.py:34-115` generates self-signed certs with a fixed serial number, week-long validity, and default filesystem permissions under a shared cache dir. Anyone on the box can read or overwrite the keys. Randomize serials, tighten permissions, and allow the validity to be configured. -23. **High – “Background” workers busy-wait on the GUI thread** +22. **High – “Background” workers busy-wait on the GUI thread** `src/REvoDesign/tools/package_manager.py:2635-2661` starts a `WorkerThread` but then loops on `work_thread.isFinished()` calling `refresh_window()`/`time.sleep`, so the main thread stays blocked and abort buttons can’t repaint. Switch to signal-driven completion instead of polling. -24. **High – Aborting a worker interrupts the whole PyMOL session** +23. **High – Aborting a worker interrupts the whole PyMOL session** `src/REvoDesign/tools/package_manager.py:2223-2244` handles every abort click by calling `cmd.interrupt()`, which cancels whatever PyMOL is currently doing—even unrelated jobs. Terminate only the subprocesses registered for that worker rather than nuking the global interpreter. -25. **High – Monaco editor logs its auth token** +24. **High – Monaco editor logs its auth token** `src/REvoDesign/editor/monaco/server.py:71-78` generates a `SECRET_TOKEN` and immediately logs it, so anyone with read access to the log file can hijack the editor backend. Never log secrets and rotate them when the server restarts. -26. **High – WebSocket clients hand over the shared auth key on request** +25. **High – WebSocket clients hand over the shared auth key on request** `src/REvoDesign/clients/QtSocketConnector.py:813-863` responds to a `RequireKey` message by sending `self.authentication_key` before the server proves its identity. A malicious host can simply ask for the key and impersonate everyone. Authenticate the server (TLS + pinned cert) and only send a key through an already-authenticated channel. -27. **Medium – Git tag fetches are synchronous and unauthenticated** +26. **Medium – Git tag fetches are synchronous and unauthenticated** `src/REvoDesign/tools/package_manager.py:2349-2380` hits the public GitHub tags API on startup without auth, caching, or error UI. When the rate limit is exceeded the manager freezes for ~10s and users see nothing. Add caching, a timeout, and a friendly error surface. -28. **Medium – pip installs run on the UI thread** +27. **Medium – pip installs run on the UI thread** `src/REvoDesign/tools/package_manager.py:861-925` calls `pip install/uninstall` directly from menu actions, blocking PyMOL for minutes during heavy installs. Dispatch these calls to `run_worker_thread_in_pool` and stream progress to the dashboard. -29. **Medium – Archive browsing keeps every extraction forever** +28. **Medium – Archive browsing keeps every extraction forever** `src/REvoDesign/driver/file_dialog.py:240-269` always expands compressed inputs into `/expanded_compressed_files/` with no cleanup or quota. Repeated browsing bloats the session dir and a malicious 20 GB tar can fill the disk. Extract into a temp dir and delete it once the dialog closes. diff --git a/src/REvoDesign/tools/utils.py b/src/REvoDesign/tools/utils.py index 4e324cb1..7d52711a 100644 --- a/src/REvoDesign/tools/utils.py +++ b/src/REvoDesign/tools/utils.py @@ -23,6 +23,7 @@ from collections.abc import Callable, Iterable from functools import partial, wraps from itertools import pairwise +from pathlib import PurePosixPath, PureWindowsPath from typing import Any, Literal, ParamSpec, TypeVar, cast, overload import matplotlib @@ -602,9 +603,18 @@ def data_generator(): def _safe_member_target(base_dir: str, member_name: str) -> str: normalized = member_name.replace("\\", "/") - if normalized.startswith("/") or normalized.startswith("../") or "/../" in f"/{normalized}": + posix_path = PurePosixPath(normalized) + windows_path = PureWindowsPath(normalized) + if ( + not normalized + or normalized == "." + or posix_path.is_absolute() + or windows_path.is_absolute() + or windows_path.drive + or any(part == ".." for part in posix_path.parts) + ): raise ValueError(f"Unsafe archive member path: {member_name}") - target_path = os.path.realpath(os.path.join(base_dir, member_name)) + target_path = os.path.realpath(os.path.join(base_dir, *posix_path.parts)) base_path = os.path.realpath(base_dir) try: if os.path.commonpath([base_path, target_path]) != base_path: diff --git a/tests/data/test_data.py b/tests/data/test_data.py index 2060045d..ef73df22 100644 --- a/tests/data/test_data.py +++ b/tests/data/test_data.py @@ -6,8 +6,6 @@ import os from dataclasses import dataclass -import pytest - @dataclass class TestData: @@ -384,10 +382,9 @@ def expand_zip(self, compressed_file: str): expanded_dirs = os.listdir(dist_dir) if not expanded_dirs: - import zipfile + from REvoDesign.tools.utils import extract_archive - with zipfile.ZipFile(compressed_file, mode="r") as z: - z.extractall(path=dist_dir) + extract_archive(archive_file=compressed_file, extract_to=dist_dir) extracted_files = os.listdir(dist_dir) return dist_dir, extracted_files diff --git a/tests/tools/test_utils.py b/tests/tools/test_utils.py index b61e5f70..e1c63b9f 100644 --- a/tests/tools/test_utils.py +++ b/tests/tools/test_utils.py @@ -231,6 +231,16 @@ def test_extract_zip_rejects_path_traversal(temp_extract_dir): assert not os.path.exists(outside_path) +@pytest.mark.parametrize("member_name", ["/tmp/evil.txt", r"C:\tmp\evil.txt"]) +def test_extract_zip_rejects_absolute_member_paths(temp_extract_dir, member_name): + archive_path = os.path.join(temp_extract_dir, "unsafe_absolute.zip") + with zipfile.ZipFile(archive_path, "w") as zipf: + zipf.writestr(member_name, "malicious") + + with pytest.raises(ValueError, match="Unsafe archive member path"): + extract_archive(archive_path, temp_extract_dir) + + def test_extract_tar_rejects_path_traversal(temp_extract_dir): archive_path = os.path.join(temp_extract_dir, "unsafe.tar.gz") payload = b"malicious" @@ -246,6 +256,19 @@ def test_extract_tar_rejects_path_traversal(temp_extract_dir): assert not os.path.exists(outside_path) +@pytest.mark.parametrize("member_name", ["/tmp/evil.txt", r"C:\tmp\evil.txt"]) +def test_extract_tar_rejects_absolute_member_paths(temp_extract_dir, member_name): + archive_path = os.path.join(temp_extract_dir, "unsafe_absolute.tar.gz") + payload = b"malicious" + info = tarfile.TarInfo(name=member_name) + info.size = len(payload) + with tarfile.open(archive_path, "w:gz") as tar: + tar.addfile(info, io.BytesIO(payload)) + + with pytest.raises(ValueError, match="Unsafe archive member path"): + extract_archive(archive_path, temp_extract_dir) + + def test_get_color_uniform_range(): # Test when min_value equals max_value color = get_color("viridis", 0.5, 0.5, 0.5) @@ -496,7 +519,8 @@ class TestClass: class CitableClass(CitableModuleAbstract): - def __init__(self): ... + def __init__(self): + ... def config(self): print("Awesome module is under configuration!") From 0bd32df28326981489328a47a35b5836f271cc0d Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:58:43 +0800 Subject: [PATCH 39/63] docs(dev): drop resolved monaco token item --- dev/TODO.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dev/TODO.md b/dev/TODO.md index 7e7fb115..afe2352d 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -46,13 +46,11 @@ `src/REvoDesign/tools/package_manager.py:2635-2661` starts a `WorkerThread` but then loops on `work_thread.isFinished()` calling `refresh_window()`/`time.sleep`, so the main thread stays blocked and abort buttons can’t repaint. Switch to signal-driven completion instead of polling. 23. **High – Aborting a worker interrupts the whole PyMOL session** `src/REvoDesign/tools/package_manager.py:2223-2244` handles every abort click by calling `cmd.interrupt()`, which cancels whatever PyMOL is currently doing—even unrelated jobs. Terminate only the subprocesses registered for that worker rather than nuking the global interpreter. -24. **High – Monaco editor logs its auth token** - `src/REvoDesign/editor/monaco/server.py:71-78` generates a `SECRET_TOKEN` and immediately logs it, so anyone with read access to the log file can hijack the editor backend. Never log secrets and rotate them when the server restarts. -25. **High – WebSocket clients hand over the shared auth key on request** +24. **High – WebSocket clients hand over the shared auth key on request** `src/REvoDesign/clients/QtSocketConnector.py:813-863` responds to a `RequireKey` message by sending `self.authentication_key` before the server proves its identity. A malicious host can simply ask for the key and impersonate everyone. Authenticate the server (TLS + pinned cert) and only send a key through an already-authenticated channel. -26. **Medium – Git tag fetches are synchronous and unauthenticated** +25. **Medium – Git tag fetches are synchronous and unauthenticated** `src/REvoDesign/tools/package_manager.py:2349-2380` hits the public GitHub tags API on startup without auth, caching, or error UI. When the rate limit is exceeded the manager freezes for ~10s and users see nothing. Add caching, a timeout, and a friendly error surface. -27. **Medium – pip installs run on the UI thread** +26. **Medium – pip installs run on the UI thread** `src/REvoDesign/tools/package_manager.py:861-925` calls `pip install/uninstall` directly from menu actions, blocking PyMOL for minutes during heavy installs. Dispatch these calls to `run_worker_thread_in_pool` and stream progress to the dashboard. -28. **Medium – Archive browsing keeps every extraction forever** +27. **Medium – Archive browsing keeps every extraction forever** `src/REvoDesign/driver/file_dialog.py:240-269` always expands compressed inputs into `/expanded_compressed_files/` with no cleanup or quota. Repeated browsing bloats the session dir and a malicious 20 GB tar can fill the disk. Extract into a temp dir and delete it once the dialog closes. From 146c06a17e5bda6fb1d04e77b0e3b609c3113967 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:04:25 +0800 Subject: [PATCH 40/63] test(deepsource): lock resolved security audit checks --- dev/TODO.md | 4 ---- server/tests/test_runner_script_static.py | 28 +++++++++++++++++++++++ tests/tools/test_utils.py | 15 ++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 server/tests/test_runner_script_static.py diff --git a/dev/TODO.md b/dev/TODO.md index afe2352d..8aa528f2 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -16,10 +16,6 @@ The Makefile and tests refer to `/Users/yyy/...` (e.g., `Makefile:113`, `tests/data/test_data.py:393`), so running scripts on any other machine wipes or looks for that specific user’s home directory. Replace these with `$HOME`/config values or derive from repo-relative paths. 8. **Medium – PIPPack runner calls private APIs of its dependency** `src/REvoDesign/sidechain/mutate_runner/PIPPack.py:64-105` invokes `_initialize_with_a_model`, `_run_repack_single`, and `_run_repack_batch` on the third-party `PIPPack` object. These underscore-prefixed methods are not part of the public API and break whenever upstream refactors; switch to supported entrypoints or wrap the CLI. -9. **High – Shell pipeline script evaluates user input** - `server/REvoDesign_PSSM_GREMLIN.sh:158-220` builds shell commands containing user-provided file paths and runs them via `eval "$cmd"`. A crafted FASTA path or output directory injects arbitrary shell code into cluster jobs. Use arrays/`"$@"` instead of eval and properly quote user input. -10. **High – WebSocket keys use non-cryptographic randomness** - `src/REvoDesign/tools/utils.py:743-765` uses `random.choice` to generate passwords, and `src/REvoDesign/REvoDesign.py:1158-1167` feeds those into `ui.socket.input.key`. The default PRNG is predictable and unsuitable for authentication secrets; switch to `secrets.choice`/`secrets.token_urlsafe`. 11. **High – Importing the logger mutates user state** `src/REvoDesign/logger/logger.py:1-120` configures logging, creates directories under `user_log_path`, and spins up a background `QueueListener` during import. Simply touching `REvoDesign.logger` writes to the host filesystem and leaves live threads around; lazily initialize logging from the entrypoint instead. 12. **High – WorkerThread drops exceptions** diff --git a/server/tests/test_runner_script_static.py b/server/tests/test_runner_script_static.py new file mode 100644 index 00000000..10044d11 --- /dev/null +++ b/server/tests/test_runner_script_static.py @@ -0,0 +1,28 @@ +# Copyright (c) 2026 The REvoDesign Developers. +# Distributed under the terms of the GNU General Public License v3.0. +# SPDX-License-Identifier: GPL-3.0-only + +from __future__ import annotations + +from pathlib import Path + + +RUNNER_SCRIPT = Path(__file__).resolve().parents[1] / "REvoDesign_PSSM_GREMLIN.sh" + + +def test_runner_script_does_not_eval_user_controlled_commands(): + script = RUNNER_SCRIPT.read_text() + + assert 'eval "$cmd"' not in script + assert "eval $cmd" not in script + assert "bash -c" not in script + assert "sh -c" not in script + + +def test_runner_script_executes_pipeline_commands_as_arrays(): + script = RUNNER_SCRIPT.read_text() + + for command in ("hhblits", "hhfilter", "GREMLIN_TFv1.py", "psiblast"): + assert command in script + + assert '"${cmd[@]}"' in script diff --git a/tests/tools/test_utils.py b/tests/tools/test_utils.py index e1c63b9f..a410e5f6 100644 --- a/tests/tools/test_utils.py +++ b/tests/tools/test_utils.py @@ -462,6 +462,21 @@ def test_generate_strong_password_character_set(): assert set(password).issubset(allowed_characters) +def test_generate_strong_password_uses_cryptographic_rng(monkeypatch): + """Password generation must not fall back to the predictable random module.""" + choices = [] + + def fake_choice(characters): + choices.append(characters) + return "A" + + monkeypatch.setattr("REvoDesign.tools.utils.secrets.choice", fake_choice) + + assert generate_strong_password(16) == "A" * 16 + assert len(choices) == 16 + assert all("A" in characters for characters in choices) + + def test_generate_strong_password_value_error(): """Test that a ValueError is raised for invalid lengths.""" with pytest.raises(ValueError): From 6920cfeb9a8286ae669e5957e613c6ddfe84f085 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:08:53 +0800 Subject: [PATCH 41/63] fix(config): remove workstation-specific paths --- Makefile | 4 ++-- dev/TODO.md | 4 +--- server/tests/test_tasks.py | 6 +++--- src/REvoDesign/config/environ.yaml | 2 +- src/REvoDesign/tools/cgo_utils.py | 2 +- tests/conftest.py | 6 +++--- tests/data/ddg_csv.json | 6 +++--- tests/data/kinetics/openkinetics_1SUO/manifest.json | 4 ++-- tests/data/test_data.py | 5 ++++- 9 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 5036d08f..6db71348 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ install-pytorch-cpu-non-mac: # local dev reinstall: make clean - make black;rm -r /Users/yyy/.REvoDesign/config/; pip install . -U + make black; rm -rf "$${HOME}/.REvoDesign/config/"; python -m pip install . -U compile-ui: stage=compile bash tools/translate.sh @@ -145,7 +145,7 @@ test: # ui-test-pymol: # # Run a tmp folder to make sure the tests are run on the installed version # mkdir -p $(TESTDIR) -# cd $(TESTDIR); pymol -ckqy /Users/yyy/miniconda_py39_arm64/lib/python3.10/site-packages/REvoDesign/tests/cases/tabs/; +# cd $(TESTDIR); pymol -ckqy $(PYTEST_CASES_PATH)/tabs/; # all test all-test: diff --git a/dev/TODO.md b/dev/TODO.md index 8aa528f2..a5a29b13 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -12,8 +12,6 @@ `src/REvoDesign/REvoDesign.py:103-145` calls `os.chdir(self.PWD)` every time a window opens, mutating the process-wide CWD for all other PyMOL plugins and threads. Persisting state should rely on explicit paths stored in the bus, not on moving the interpreter’s CWD. 6. **Medium – ConfigBus is a mutable global singleton storing Qt widget handles** `src/REvoDesign/driver/ui_driver.py:365-381` binds the singleton instance to the active UI and exposes widgets to every module. This leaks Qt objects across tests, prevents multiple windows, and requires manual resets (see `tests/conftest.reset_singletons`). Break ConfigBus into a pure configuration store plus scoped UI adapters so state does not persist globally. -7. **Medium – Developer-only absolute paths are committed** - The Makefile and tests refer to `/Users/yyy/...` (e.g., `Makefile:113`, `tests/data/test_data.py:393`), so running scripts on any other machine wipes or looks for that specific user’s home directory. Replace these with `$HOME`/config values or derive from repo-relative paths. 8. **Medium – PIPPack runner calls private APIs of its dependency** `src/REvoDesign/sidechain/mutate_runner/PIPPack.py:64-105` invokes `_initialize_with_a_model`, `_run_repack_single`, and `_run_repack_batch` on the third-party `PIPPack` object. These underscore-prefixed methods are not part of the public API and break whenever upstream refactors; switch to supported entrypoints or wrap the CLI. 11. **High – Importing the logger mutates user state** @@ -21,7 +19,7 @@ 12. **High – WorkerThread drops exceptions** `src/REvoDesign/tools/package_manager.py:2296-2345` executes `self.func` in `QThread.run()` without trapping exceptions or emitting an error signal. Failures just print to stderr while the UI thinks work succeeded. Catch exceptions, forward them through a signal, and surface them via `notify_signal`. 13. **High – Tests still touch the real config tree** - `tests/conftest.py:53-124` calls `set_REvoDesign_config_file()` at import time, which may pop up Qt dialogs and can delete `/Users/yyy/Library/Application Support/REvoDesign/config` if the user confirms. Mock `platformdirs.user_data_dir`/`user_cache_dir` so tests operate entirely inside the repo. + `tests/conftest.py:53-124` calls `set_REvoDesign_config_file()` at import time, which may pop up Qt dialogs and can delete the user's `~/Library/Application Support/REvoDesign/config` tree if the user confirms. Mock `platformdirs.user_data_dir`/`user_cache_dir` so tests operate entirely inside the repo. 14. **High – RFdiffusion helper forces Qt5Agg backend globally** `src/REvoDesign/tools/rfdiffusion_tools.py:1-7` unconditionally runs `matplotlib.use("Qt5Agg")` on import, breaking headless builds and notebooks that rely on Agg. Only set the backend when a GUI feature is invoked and honor `MPLBACKEND`. 15. **High – Download registry accepts unsigned payloads** diff --git a/server/tests/test_tasks.py b/server/tests/test_tasks.py index f7e0d8cd..20c2c5e8 100644 --- a/server/tests/test_tasks.py +++ b/server/tests/test_tasks.py @@ -482,7 +482,7 @@ def test_dashboard_masks_host_file_paths_on_read_errors(monkeypatch, tmp_path): md5sum = uuid.uuid4().hex result_dir = tmp_path / "result" result_dir.mkdir(parents=True, exist_ok=True) - leaked_host_path = "/Users/yyy/Documents/protein_design/REvoDesign/playground/server_test/upload/2KL8.fasta" + leaked_host_path = "/home/server-user/REvoDesign/playground/server_test/upload/2KL8.fasta" _upsert_task_for_user( module, @@ -517,7 +517,7 @@ def test_failed_status_masks_host_paths_in_api_error(monkeypatch, tmp_path): md5sum = uuid.uuid4().hex result_dir = tmp_path / "result" result_dir.mkdir(parents=True, exist_ok=True) - leaked_host_path = "/Users/yyy/Documents/protein_design/REvoDesign/playground/server_test/upload/2KL8.fasta" + leaked_host_path = "/home/server-user/REvoDesign/playground/server_test/upload/2KL8.fasta" _upsert_task_for_user( module, @@ -538,7 +538,7 @@ def test_failed_status_masks_host_paths_in_api_error(monkeypatch, tmp_path): payload = response.get_json() assert payload["status"] == "failed" assert "/srv/REvoDesign/PSSM_GREMLIN/upload/2KL8.fasta" in payload["error"] - assert "/Users/yyy/Documents/protein_design/REvoDesign" not in payload["error"] + assert "/home/server-user/REvoDesign" not in payload["error"] def test_private_dashboard_blocks_non_owner_access(monkeypatch, tmp_path): diff --git a/src/REvoDesign/config/environ.yaml b/src/REvoDesign/config/environ.yaml index a3b781f1..36d33bab 100644 --- a/src/REvoDesign/config/environ.yaml +++ b/src/REvoDesign/config/environ.yaml @@ -1,5 +1,5 @@ # register all variables here variables: - # ROSETTA_BIN: /Users/yyy/rosetta/rosetta_binary/rosetta.source.release-371/main/source/bin/ + # ROSETTA_BIN: /path/to/rosetta/source/bin/ # OPENKINETICS_API_KEY: your-openkinetics-api-key diff --git a/src/REvoDesign/tools/cgo_utils.py b/src/REvoDesign/tools/cgo_utils.py index 4011dced..3496a479 100644 --- a/src/REvoDesign/tools/cgo_utils.py +++ b/src/REvoDesign/tools/cgo_utils.py @@ -2503,7 +2503,7 @@ def rebuild(self): # with timing('writing'): # test_text() -# font_path = "/Users/yyy/Downloads/simhei.ttf" +# font_path = "/path/to/simhei.ttf" # text='Silver Bullet\n\nCool Kid' diff --git a/tests/conftest.py b/tests/conftest.py index 8987b62c..1fd60a59 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -116,9 +116,9 @@ def check_real_config_dir(i: int): """ A checkpoint to check whether the test suite has created the real config dir. """ - the_dir = "/Users/yyy/Library/Application Support/REvoDesign/config" - if os.path.exists(the_dir): - raise RuntimeError(f"[{i}]The test suite has created this dir! {the_dir}") + real_config_dir = Path.home() / "Library" / "Application Support" / "REvoDesign" / "config" + if real_config_dir.exists(): + raise RuntimeError(f"[{i}]The test suite has created this dir! {real_config_dir}") # check_real_config_dir(cck.i) diff --git a/tests/data/ddg_csv.json b/tests/data/ddg_csv.json index 22069a10..dd2764fa 100644 --- a/tests/data/ddg_csv.json +++ b/tests/data/ddg_csv.json @@ -5,12 +5,12 @@ "__date__": "2025-03-12 17:20:50" }, "__asked_values__": { - "profile": "/Users/yyy/Documents/protein_design/REvoDesign/tests/data/pytia_ddg/1SUO_pred_mask.csv", + "profile": "tests/data/pytia_ddg/1SUO_pred_mask.csv", "profile_type": "CSV", "prefer_lower_score": true, "keep_missing": true, - "residue_range": "/Users/yyy/Documents/protein_design/REvoDesign/tests/data/surface_residue_records/1SUO_residues_cutoff_30.0.txt", + "residue_range": "tests/data/surface_residue_records/1SUO_residues_cutoff_30.0.txt", "view_highlight": "orient", "view_highlight_nbr": 6 } -} \ No newline at end of file +} diff --git a/tests/data/kinetics/openkinetics_1SUO/manifest.json b/tests/data/kinetics/openkinetics_1SUO/manifest.json index 47ac3d48..363867f8 100644 --- a/tests/data/kinetics/openkinetics_1SUO/manifest.json +++ b/tests/data/kinetics/openkinetics_1SUO/manifest.json @@ -19,8 +19,8 @@ "revodesign_commit_hash": "83e1f78f8349efad62af8c01aab88625ecd6d237", "schema_assumption_note": "Official API docs could not be fetched automatically during implementation; the collector uses the documented fallback endpoint pattern from plan/openkinetics.md.", "secrets_policy": "Authorization headers and API keys are redacted and never stored", - "source_mutation_file": "/Users/yyy/Documents/protein_design/REvoDesign/tests/data/mutations/1SUO.surf.entro.mutagenesis.besthits.mut.txt", - "source_pdb_file": "/Users/yyy/Documents/protein_design/REvoDesign/tests/data/pdb/1SUO.pdb", + "source_mutation_file": "tests/data/mutations/1SUO.surf.entro.mutagenesis.besthits.mut.txt", + "source_pdb_file": "tests/data/pdb/1SUO.pdb", "source_structure_id": "1SUO", "substrate_metadata_source": "rdkit", "substrate_smiles": "ClC1CCC(C2CNCN2)CC1" diff --git a/tests/data/test_data.py b/tests/data/test_data.py index ef73df22..ca52a40d 100644 --- a/tests/data/test_data.py +++ b/tests/data/test_data.py @@ -392,7 +392,10 @@ def expand_zip(self, compressed_file: str): @dataclass() class TestDataOnLocalMac(TestData): - test_data_repo: str = "/Users/yyy/Documents/protein_design/REvoDesignTutorial/" + test_data_repo: str = os.environ.get( + "REVODESIGN_TUTORIAL_DATA", + os.path.expanduser("~/Documents/protein_design/REvoDesignTutorial"), + ) if __name__ == "__main__": From e53fc5e036b4e4111347bf3fbd003d754e34b846 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:13:07 +0800 Subject: [PATCH 42/63] fix(security): harden generated ssl certificates --- dev/TODO.md | 2 - src/REvoDesign/tools/ssl_certificates.py | 56 +++++++++++++++--- tests/tools/test_ssl_certificates.py | 74 ++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 tests/tools/test_ssl_certificates.py diff --git a/dev/TODO.md b/dev/TODO.md index a5a29b13..ea5f1293 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -34,8 +34,6 @@ `server/pssm_gremlin/pssm_gremlin.py:38-120` assumes `/mnt/data/...`, `/mnt/db/...`, and `redis://localhost` exist and derives Docker mounts from `/home/{os.getlogin()}`. With no configuration hooks the service cannot run off that exact cluster. Externalize broker paths, DBs, and output dirs via a config file/env vars. 20. **Medium – Config-driven env overrides clobber user variables** `src/REvoDesign/driver/environ_register.py:8-39` iterates `environ.yaml` and unconditionally sets each key in `os.environ`, overwriting whatever the shell provided (proxy settings, CUDA flags, etc.). Respect existing values or mark overrides explicitly in config. -21. **High – SSL certificate manager writes weak certs** - `src/REvoDesign/tools/ssl_certificates.py:34-115` generates self-signed certs with a fixed serial number, week-long validity, and default filesystem permissions under a shared cache dir. Anyone on the box can read or overwrite the keys. Randomize serials, tighten permissions, and allow the validity to be configured. 22. **High – “Background” workers busy-wait on the GUI thread** `src/REvoDesign/tools/package_manager.py:2635-2661` starts a `WorkerThread` but then loops on `work_thread.isFinished()` calling `refresh_window()`/`time.sleep`, so the main thread stays blocked and abort buttons can’t repaint. Switch to signal-driven completion instead of polling. 23. **High – Aborting a worker interrupts the whole PyMOL session** diff --git a/src/REvoDesign/tools/ssl_certificates.py b/src/REvoDesign/tools/ssl_certificates.py index 67c4444c..9d3b8bb8 100644 --- a/src/REvoDesign/tools/ssl_certificates.py +++ b/src/REvoDesign/tools/ssl_certificates.py @@ -11,6 +11,7 @@ import datetime import os import platform +import secrets import ssl from dataclasses import dataclass from typing import Literal @@ -22,6 +23,39 @@ logging = ROOT_LOGGER.getChild(__name__) +DEFAULT_CERT_VALIDITY_DAYS = 7 +CERT_VALIDITY_ENV = "REVODESIGN_SSL_CERT_VALIDITY_DAYS" +PRIVATE_DIR_MODE = 0o700 +PRIVATE_KEY_MODE = 0o600 +CERTIFICATE_MODE = 0o644 + + +def _certificate_validity_days() -> int: + raw_days = os.environ.get(CERT_VALIDITY_ENV) + if raw_days is None: + return DEFAULT_CERT_VALIDITY_DAYS + + try: + days = int(raw_days) + except ValueError as exc: + raise ValueError(f"{CERT_VALIDITY_ENV} must be a positive integer.") from exc + + if days <= 0: + raise ValueError(f"{CERT_VALIDITY_ENV} must be a positive integer.") + + return days + + +def _write_pem_file(path: str, payload: bytes, mode: int) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + fd = os.open(path, flags, mode) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(payload) + finally: + os.chmod(path, mode) + + @dataclass class SSLCertificateManager: peer_roles = Literal["server", "client"] @@ -33,7 +67,8 @@ def __post_init__(self): self.cache_dir: str = set_cache_dir() self.crt_dir: str = os.path.join(self.cache_dir, "crts") - os.makedirs(self.crt_dir, exist_ok=True) + os.makedirs(self.crt_dir, mode=PRIVATE_DIR_MODE, exist_ok=True) + os.chmod(self.crt_dir, PRIVATE_DIR_MODE) self.crt_path = os.path.join(self.crt_dir, f"{self.role}.crt") self.key_path = os.path.join(self.crt_dir, f"{self.role}.key") @@ -95,6 +130,7 @@ def get_certificate(self): logging.warning("Certificate has expired. Generating a new certificate.") self.create_new_certificate() else: + self._enforce_file_permissions() logging.info("Certificate is still valid.") def create_new_certificate(self): @@ -109,6 +145,7 @@ def create_new_certificate(self): - None """ role = self.role + validity_days = _certificate_validity_days() # Get node information from OS or set to 'Unknown' if not available @@ -135,15 +172,20 @@ def create_new_certificate(self): cert.get_subject().CN = _cn[:64] # Set serial number, validity period, issuer, public key, and sign the certificate - cert.set_serial_number(1000) + cert.set_serial_number(secrets.randbits(159) + 1) cert.gmtime_adj_notBefore(0) - cert.gmtime_adj_notAfter(7 * 24 * 60 * 60) + cert.gmtime_adj_notAfter(validity_days * 24 * 60 * 60) cert.set_issuer(cert.get_subject()) cert.set_pubkey(k) cert.sign(k, "sha256") # Write the certificate and private key to files in PEM format - with open(self.crt_path, "wb") as f: - f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert)) - with open(self.key_path, "wb") as f: - f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k)) + _write_pem_file(self.crt_path, crypto.dump_certificate(crypto.FILETYPE_PEM, cert), CERTIFICATE_MODE) + _write_pem_file(self.key_path, crypto.dump_privatekey(crypto.FILETYPE_PEM, k), PRIVATE_KEY_MODE) + + def _enforce_file_permissions(self): + os.chmod(self.crt_dir, PRIVATE_DIR_MODE) + if os.path.exists(self.crt_path): + os.chmod(self.crt_path, CERTIFICATE_MODE) + if os.path.exists(self.key_path): + os.chmod(self.key_path, PRIVATE_KEY_MODE) diff --git a/tests/tools/test_ssl_certificates.py b/tests/tools/test_ssl_certificates.py new file mode 100644 index 00000000..004c6420 --- /dev/null +++ b/tests/tools/test_ssl_certificates.py @@ -0,0 +1,74 @@ +# Copyright (c) 2026 The REvoDesign Developers. +# Distributed under the terms of the GNU General Public License v3.0. +# SPDX-License-Identifier: GPL-3.0-only + +from __future__ import annotations + +import datetime +import stat + +import pytest +from OpenSSL import crypto + +import REvoDesign +from REvoDesign.tools.ssl_certificates import ( + CERT_VALIDITY_ENV, + CERTIFICATE_MODE, + PRIVATE_DIR_MODE, + PRIVATE_KEY_MODE, + SSLCertificateManager, +) + + +def _mode(path): + return stat.S_IMODE(path.stat().st_mode) + + +def _load_cert(path): + return crypto.load_certificate(crypto.FILETYPE_PEM, path.read_bytes()) + + +def test_new_certificate_uses_random_serial_and_private_permissions(monkeypatch, tmp_path): + monkeypatch.setattr(REvoDesign, "set_cache_dir", lambda: str(tmp_path)) + + manager = SSLCertificateManager(role="server") + manager.create_new_certificate() + first_serial = _load_cert(tmp_path / "crts" / "server.crt").get_serial_number() + + manager.create_new_certificate() + second_serial = _load_cert(tmp_path / "crts" / "server.crt").get_serial_number() + + assert first_serial != 1000 + assert second_serial != 1000 + assert first_serial != second_serial + assert _mode(tmp_path / "crts") == PRIVATE_DIR_MODE + assert _mode(tmp_path / "crts" / "server.key") == PRIVATE_KEY_MODE + assert _mode(tmp_path / "crts" / "server.crt") == CERTIFICATE_MODE + + +def test_certificate_validity_can_be_configured(monkeypatch, tmp_path): + monkeypatch.setattr(REvoDesign, "set_cache_dir", lambda: str(tmp_path)) + monkeypatch.setenv(CERT_VALIDITY_ENV, "3") + + manager = SSLCertificateManager(role="server") + manager.create_new_certificate() + + cert = _load_cert(tmp_path / "crts" / "server.crt") + expires_at = datetime.datetime.strptime( + cert.get_notAfter().decode("utf-8"), + "%Y%m%d%H%M%SZ", + ).replace(tzinfo=datetime.UTC) + now = datetime.datetime.now(datetime.UTC) + + assert datetime.timedelta(days=2, hours=23) < expires_at - now <= datetime.timedelta(days=3, minutes=1) + + +@pytest.mark.parametrize("value", ["0", "-1", "invalid"]) +def test_certificate_validity_rejects_invalid_values(monkeypatch, tmp_path, value): + monkeypatch.setattr(REvoDesign, "set_cache_dir", lambda: str(tmp_path)) + monkeypatch.setenv(CERT_VALIDITY_ENV, value) + + manager = SSLCertificateManager(role="server") + + with pytest.raises(ValueError, match=CERT_VALIDITY_ENV): + manager.create_new_certificate() From 4d2f3f474c491c3989d0e8881373dc109807a488 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:15:48 +0800 Subject: [PATCH 43/63] fix(rfdiffusion): defer matplotlib backend selection --- dev/TODO.md | 2 - src/REvoDesign/tools/rfdiffusion_tools.py | 21 ++++++++--- tests/tools/test_rfdiffusion_tools.py | 46 +++++++++++++++++++++++ 3 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 tests/tools/test_rfdiffusion_tools.py diff --git a/dev/TODO.md b/dev/TODO.md index ea5f1293..fda1681f 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -20,8 +20,6 @@ `src/REvoDesign/tools/package_manager.py:2296-2345` executes `self.func` in `QThread.run()` without trapping exceptions or emitting an error signal. Failures just print to stderr while the UI thinks work succeeded. Catch exceptions, forward them through a signal, and surface them via `notify_signal`. 13. **High – Tests still touch the real config tree** `tests/conftest.py:53-124` calls `set_REvoDesign_config_file()` at import time, which may pop up Qt dialogs and can delete the user's `~/Library/Application Support/REvoDesign/config` tree if the user confirms. Mock `platformdirs.user_data_dir`/`user_cache_dir` so tests operate entirely inside the repo. -14. **High – RFdiffusion helper forces Qt5Agg backend globally** - `src/REvoDesign/tools/rfdiffusion_tools.py:1-7` unconditionally runs `matplotlib.use("Qt5Agg")` on import, breaking headless builds and notebooks that rely on Agg. Only set the backend when a GUI feature is invoked and honor `MPLBACKEND`. 15. **High – Download registry accepts unsigned payloads** `src/REvoDesign/tools/download_registry.py:133-152` leaves registry entries as `None`, so `pooch` skips hash verification and trusts whatever the server returns. Require hashes for every asset or refuse to download; otherwise a MITM can ship arbitrary binaries. 16. **High – RFdiffusion weight downloads block the UI** diff --git a/src/REvoDesign/tools/rfdiffusion_tools.py b/src/REvoDesign/tools/rfdiffusion_tools.py index 8b0d8829..443f2ce6 100644 --- a/src/REvoDesign/tools/rfdiffusion_tools.py +++ b/src/REvoDesign/tools/rfdiffusion_tools.py @@ -2,8 +2,8 @@ # Distributed under the terms of the GNU General Public License v3.0. # SPDX-License-Identifier: GPL-3.0-only +import os -import matplotlib import numpy as np from REvoDesign.basic.abc_third_party_module import ThirdPartyModuleAbstract @@ -13,10 +13,19 @@ logging = ROOT_LOGGER.getChild(__name__) -try: - matplotlib.use("QtAgg") -except Exception as exc: - logging.debug("Falling back to default matplotlib backend: %s", exc) +PREFERRED_INTERACTIVE_BACKEND = "QtAgg" + + +def _prefer_interactive_backend_for_plotting() -> None: + if os.environ.get("MPLBACKEND"): + return + + import matplotlib + + try: + matplotlib.use(PREFERRED_INTERACTIVE_BACKEND, force=False) + except Exception as exc: + logging.debug("Falling back to default matplotlib backend: %s", exc) @require_installed @@ -223,6 +232,8 @@ def plot_potential_field(self, grid_size=200, margin=10, save_to: str = "default Path to save the plot. If 'default.png', the plot will be saved in the current working directory. """ + _prefer_interactive_backend_for_plotting() + import matplotlib.pyplot as plt X, Y, P = self.compute_potential_field(grid_size=grid_size, margin=margin) diff --git a/tests/tools/test_rfdiffusion_tools.py b/tests/tools/test_rfdiffusion_tools.py new file mode 100644 index 00000000..4918f9be --- /dev/null +++ b/tests/tools/test_rfdiffusion_tools.py @@ -0,0 +1,46 @@ +# Copyright (c) 2026 The REvoDesign Developers. +# Distributed under the terms of the GNU General Public License v3.0. +# SPDX-License-Identifier: GPL-3.0-only + +from __future__ import annotations + +import importlib +import sys + +import matplotlib + + +def _reload_rfdiffusion_tools(): + sys.modules.pop("REvoDesign.tools.rfdiffusion_tools", None) + return importlib.import_module("REvoDesign.tools.rfdiffusion_tools") + + +def test_import_does_not_force_matplotlib_backend(monkeypatch): + calls = [] + monkeypatch.setattr(matplotlib, "use", lambda *args, **kwargs: calls.append((args, kwargs))) + + _reload_rfdiffusion_tools() + + assert calls == [] + + +def test_plotting_can_prefer_interactive_backend_lazily(monkeypatch): + module = _reload_rfdiffusion_tools() + calls = [] + monkeypatch.delenv("MPLBACKEND", raising=False) + monkeypatch.setattr(matplotlib, "use", lambda *args, **kwargs: calls.append((args, kwargs))) + + module._prefer_interactive_backend_for_plotting() + + assert calls == [((module.PREFERRED_INTERACTIVE_BACKEND,), {"force": False})] + + +def test_plotting_honors_existing_mplbackend(monkeypatch): + module = _reload_rfdiffusion_tools() + calls = [] + monkeypatch.setenv("MPLBACKEND", "Agg") + monkeypatch.setattr(matplotlib, "use", lambda *args, **kwargs: calls.append((args, kwargs))) + + module._prefer_interactive_backend_for_plotting() + + assert calls == [] From 1c6221c97ade67d5f8f6e0de4662bdd70e85085a Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:18:11 +0800 Subject: [PATCH 44/63] docs(downloads): require registry hashes --- dev/TODO.md | 2 -- docs/api/download-registry.md | 6 +++++- tests/tools/test_file_downloader.py | 13 +++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/dev/TODO.md b/dev/TODO.md index fda1681f..4dcc300e 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -20,8 +20,6 @@ `src/REvoDesign/tools/package_manager.py:2296-2345` executes `self.func` in `QThread.run()` without trapping exceptions or emitting an error signal. Failures just print to stderr while the UI thinks work succeeded. Catch exceptions, forward them through a signal, and surface them via `notify_signal`. 13. **High – Tests still touch the real config tree** `tests/conftest.py:53-124` calls `set_REvoDesign_config_file()` at import time, which may pop up Qt dialogs and can delete the user's `~/Library/Application Support/REvoDesign/config` tree if the user confirms. Mock `platformdirs.user_data_dir`/`user_cache_dir` so tests operate entirely inside the repo. -15. **High – Download registry accepts unsigned payloads** - `src/REvoDesign/tools/download_registry.py:133-152` leaves registry entries as `None`, so `pooch` skips hash verification and trusts whatever the server returns. Require hashes for every asset or refuse to download; otherwise a MITM can ship arbitrary binaries. 16. **High – RFdiffusion weight downloads block the UI** `src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py:90-151` fetches multi‑GB checkpoints synchronously inside `RfDiffusion.pick_model`, freezing PyMOL while `pooch` downloads. Move the download to `run_worker_thread_in_pool` and provide progress/abort hooks. 17. **High – DglSolver silently runs pip install** diff --git a/docs/api/download-registry.md b/docs/api/download-registry.md index 0a5f712e..cd5be13e 100644 --- a/docs/api/download-registry.md +++ b/docs/api/download-registry.md @@ -26,7 +26,7 @@ registry = FileDownloadRegistry( base_url="https://example.com/data/", registry={ "file1.pdb": "md5:abc123def456", - "file2.pkl": None, # no hash verification + "file2.pkl": "sha256:0123456789abcdef...", }, alternative_base_urls=["https://mirror.example.com/data/"], retry_count=3, @@ -65,6 +65,10 @@ local filesystem path. The archive extraction feature (`flatten_archive`) can automatically unpack `.tar.gz` and `.zip` archives into a `_flatten/` subdirectory. +Every registry entry must include an `md5:`, `sha256:`, or other Pooch-supported +hash string. Entries with `None` or an empty hash are rejected before any +download starts. + ## Creating Registries from MD5 Checksums ```python diff --git a/tests/tools/test_file_downloader.py b/tests/tools/test_file_downloader.py index a1c78009..08f9c314 100644 --- a/tests/tools/test_file_downloader.py +++ b/tests/tools/test_file_downloader.py @@ -48,6 +48,19 @@ def test_preprocess_registry_rejects_missing_hashes(self): with pytest.raises(ValueError, match="file1.txt, file2.txt"): FileDownloadRegistry.preprocess_registry({"file1.txt": None, "file2.txt": ""}) + @patch("REvoDesign.tools.download_registry.pooch.create") + def test_init_rejects_registry_entries_without_hashes(self, mock_pooch_create): + """Test unsigned registry entries fail before Pooch can download them.""" + with pytest.raises(ValueError, match="unsafe.pkl"): + FileDownloadRegistry( + name="test_module", + base_url="http://example.com", + registry={"safe.txt": "md5:abc123", "unsafe.pkl": None}, + customized_directory="/custom/path", + ) + + mock_pooch_create.assert_not_called() + @pytest.mark.parametrize( "md5_contents, expected", [ From d2a108cc626ddcda57fa8aa2ec61a473301e9682 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:21:23 +0800 Subject: [PATCH 45/63] fix(workers): propagate worker thread errors --- dev/TODO.md | 2 -- src/REvoDesign/tools/package_manager.py | 16 ++++++++++++ tests/tools/test_package_manager.py | 34 +++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/dev/TODO.md b/dev/TODO.md index 4dcc300e..5eb8827e 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -16,8 +16,6 @@ `src/REvoDesign/sidechain/mutate_runner/PIPPack.py:64-105` invokes `_initialize_with_a_model`, `_run_repack_single`, and `_run_repack_batch` on the third-party `PIPPack` object. These underscore-prefixed methods are not part of the public API and break whenever upstream refactors; switch to supported entrypoints or wrap the CLI. 11. **High – Importing the logger mutates user state** `src/REvoDesign/logger/logger.py:1-120` configures logging, creates directories under `user_log_path`, and spins up a background `QueueListener` during import. Simply touching `REvoDesign.logger` writes to the host filesystem and leaves live threads around; lazily initialize logging from the entrypoint instead. -12. **High – WorkerThread drops exceptions** - `src/REvoDesign/tools/package_manager.py:2296-2345` executes `self.func` in `QThread.run()` without trapping exceptions or emitting an error signal. Failures just print to stderr while the UI thinks work succeeded. Catch exceptions, forward them through a signal, and surface them via `notify_signal`. 13. **High – Tests still touch the real config tree** `tests/conftest.py:53-124` calls `set_REvoDesign_config_file()` at import time, which may pop up Qt dialogs and can delete the user's `~/Library/Application Support/REvoDesign/config` tree if the user confirms. Mock `platformdirs.user_data_dir`/`user_cache_dir` so tests operate entirely inside the repo. 16. **High – RFdiffusion weight downloads block the UI** diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index f21af547..77caeac5 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -2422,6 +2422,7 @@ class WorkerThread(QtCore.QThread): """Custom worker thread for executing a function in a separate thread.""" result_signal = QtCore.pyqtSignal(list) + error_signal = QtCore.pyqtSignal(object) finished_signal = QtCore.pyqtSignal() interrupt_signal = QtCore.pyqtSignal() notify_signal = QtCore.pyqtSignal(str) @@ -2434,6 +2435,7 @@ def __init__(self, func, args=None, kwargs=None): self.args = args or () self.kwargs = kwargs or {} self.results = None + self.error: BaseException | None = None self.interrupt_signal.connect(self._handle_interrupt) def run(self): @@ -2445,6 +2447,14 @@ def run(self): self.results = [self.func(*self.args, **self.kwargs)] if self.results: self.result_signal.emit(self.results) + except BaseException as exc: # noqa: B036 - worker errors are re-raised by the caller after thread join + self.error = exc + logging.exception("WorkerThread failed while running %s", self.func) + try: + self.error_signal.emit(exc) + self.notify_signal.emit(f"{type(exc).__name__}: {exc}") + except RuntimeError: + pass finally: try: self.finished_signal.emit() @@ -2462,6 +2472,9 @@ def run(self): def handle_result(self): return self.results + def handle_error(self): + return self.error + def interrupt(self): self.requestInterruption() RunningProcessRegistry.terminate(self) @@ -2802,6 +2815,9 @@ def run_worker_thread_in_pool( time.sleep(0.01) result = work_thread.handle_result() + error = work_thread.handle_error() + if error is not None: + raise error return result[0] if result else None diff --git a/tests/tools/test_package_manager.py b/tests/tools/test_package_manager.py index 6cae257f..72cdb62a 100644 --- a/tests/tools/test_package_manager.py +++ b/tests/tools/test_package_manager.py @@ -762,3 +762,37 @@ def cancellable_task(): assert manager.abort_overlays == [] assert mock_cmd.interrupt.called assert result is None or result[0] == "aborted" + + +def test_worker_thread_emits_and_records_errors(qtbot): + def failing_task(): + raise RuntimeError("worker exploded") + + worker_thread = package_manager.WorkerThread(failing_task) + errors = [] + notifications = [] + finished = [] + worker_thread.error_signal.connect(errors.append) + worker_thread.notify_signal.connect(notifications.append) + worker_thread.finished_signal.connect(lambda: finished.append(True)) + + worker_thread.start() + + qtbot.waitUntil(lambda: bool(finished), timeout=2000) + worker_thread.wait(2000) + + assert worker_thread.handle_result() is None + assert isinstance(worker_thread.handle_error(), RuntimeError) + assert len(errors) == 1 + assert isinstance(errors[0], RuntimeError) + assert notifications == ["RuntimeError: worker exploded"] + + +def test_run_worker_thread_in_pool_reraises_worker_errors(qtbot, monkeypatch): + monkeypatch.setattr(package_manager, "refresh_window", lambda: None) + + def failing_task(): + raise RuntimeError("worker exploded") + + with pytest.raises(RuntimeError, match="worker exploded"): + package_manager.run_worker_thread_in_pool(failing_task, notify_slot=lambda _message: None) From 29daae4d4d50ce4aced0f1fc8cbb8ff9d9a0a3bf Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:29:18 +0800 Subject: [PATCH 46/63] fix(rfdiffusion): require approval before dgl install --- dev/TODO.md | 2 - .../shortcuts/tools/rfdiffusion_tasks.py | 45 +++++++++- tests/shortcuts/test_rfdiffusion_tasks.py | 83 +++++++++++++++++++ 3 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 tests/shortcuts/test_rfdiffusion_tasks.py diff --git a/dev/TODO.md b/dev/TODO.md index 5eb8827e..64381a77 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -20,8 +20,6 @@ `tests/conftest.py:53-124` calls `set_REvoDesign_config_file()` at import time, which may pop up Qt dialogs and can delete the user's `~/Library/Application Support/REvoDesign/config` tree if the user confirms. Mock `platformdirs.user_data_dir`/`user_cache_dir` so tests operate entirely inside the repo. 16. **High – RFdiffusion weight downloads block the UI** `src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py:90-151` fetches multi‑GB checkpoints synchronously inside `RfDiffusion.pick_model`, freezing PyMOL while `pooch` downloads. Move the download to `run_worker_thread_in_pool` and provide progress/abort hooks. -17. **High – DglSolver silently runs pip install** - `src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py:33-70` detects CUDA and immediately executes `python -m pip install dgl==2.2.1` in the current interpreter without prompting or isolating the environment. Prompt the user and install inside a managed env/container so you don’t mutate the host Python unexpectedly. 18. **High – Server keeps plaintext credentials in repo** `server/pssm_gremlin/pssm_gremlin.py:17-36` parses `users.txt` where each line is `username:password` and only hashes them at runtime, so the file on disk stores the raw passwords. Store hashed secrets (or use a proper auth backend) to avoid leaking credentials if the repo/server is compromised. 19. **High – GREMLIN web service hardcodes cluster-only paths** diff --git a/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py b/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py index 9a846827..03028778 100644 --- a/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py +++ b/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py @@ -19,13 +19,14 @@ import numpy as np from hydra import errors as hydra_errors from omegaconf import DictConfig, OmegaConf +from pymol.Qt import QtWidgets from REvoDesign import ROOT_LOGGER, issues from REvoDesign.basic.abc_third_party_module import ThirdPartyModuleAbstract, TorchModuleAbstract from REvoDesign.bootstrap import REVODESIGN_CONFIG_FILE from REvoDesign.bootstrap.set_config import is_package_installed, reload_config_file from REvoDesign.tools.download_registry import FileDownloadRegistry -from REvoDesign.tools.package_manager import run_command +from REvoDesign.tools.package_manager import decide, run_command from REvoDesign.tools.rfdiffusion_tools import SubstratePotentialVisualizer from REvoDesign.tools.utils import device_picker, get_cited, require_installed, timing @@ -53,6 +54,12 @@ 6f4d00394d34f6a9072d70976f6c8777 RF_structure_prediction_weights.pt """ +DGL_INSTALL_APPROVAL_ENV = "REVODESIGN_ALLOW_DGL_INSTALL" + + +def _has_qapplication() -> bool: + return QtWidgets.QApplication.instance() is not None + @dataclass class DglSolver: @@ -84,6 +91,25 @@ def fetch_cuda_version_before_install(self): stacklevel=2, ) + def _install_approved(self, index_link: str) -> bool: + if os.environ.get(DGL_INSTALL_APPROVAL_ENV) == "1": + return True + if not _has_qapplication(): + logging.warning( + "DGL installation requires interactive approval; no QApplication is available." + ) + return False + + return decide( + title="Install DGL?", + description=( + "RFdiffusion requires DGL, but it is not installed. " + "Install dgl==2.2.1 into the current Python environment?\n\n" + f"Package index: {index_link}" + ), + rich=True, + ) + def install(self): self.fetch_cuda_version_before_install() if self.cuda_version: @@ -91,6 +117,10 @@ def install(self): else: index_link = "https://data.dgl.ai/wheels/repo.html" + if not self._install_approved(index_link): + logging.info("DGL installation was not approved.") + return False + c = run_command( [sys.executable, "-m", "pip", "install", "dgl==2.2.1", "-f", index_link] ) @@ -98,6 +128,7 @@ def install(self): logging.error(f"Failed to install DGL: {c.stderr}") raise RuntimeError(f"Failed to install DGL: {c.stderr}") self.installed = True + return True RFD_WEIGHTS = FileDownloadRegistry( @@ -216,10 +247,18 @@ def ensure_dgl(): if (dgl_solver := DglSolver()).installed: return - warnings.warn(issues.MissingExternalTool("DGL is not installed. Now try to install it."), stacklevel=2) + warnings.warn( + issues.MissingExternalTool("DGL is not installed. User approval is required before installing it."), + stacklevel=2, + ) - dgl_solver.install() + install_started = dgl_solver.install() if not dgl_solver.installed: + if not install_started: + raise issues.MissingExternalToolError( + "DGL installation was cancelled. Install dgl==2.2.1 manually or approve installation " + f"by setting {DGL_INSTALL_APPROVAL_ENV}=1." + ) raise issues.MissingExternalToolError( "DGL may not be installed. Please install it manually or take a restart to take effect." ) diff --git a/tests/shortcuts/test_rfdiffusion_tasks.py b/tests/shortcuts/test_rfdiffusion_tasks.py new file mode 100644 index 00000000..214a668a --- /dev/null +++ b/tests/shortcuts/test_rfdiffusion_tasks.py @@ -0,0 +1,83 @@ +# Copyright (c) 2026 The REvoDesign Developers. +# Distributed under the terms of the GNU General Public License v3.0. +# SPDX-License-Identifier: GPL-3.0-only + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from REvoDesign import issues +from REvoDesign.shortcuts.tools import rfdiffusion_tasks + + +def test_dgl_solver_does_not_run_pip_without_approval(monkeypatch): + solver = rfdiffusion_tasks.DglSolver(installed=False) + monkeypatch.setattr(solver, "fetch_cuda_version_before_install", lambda: None) + monkeypatch.setattr(rfdiffusion_tasks, "_has_qapplication", lambda: True) + monkeypatch.delenv(rfdiffusion_tasks.DGL_INSTALL_APPROVAL_ENV, raising=False) + monkeypatch.setattr(rfdiffusion_tasks, "decide", lambda **_kwargs: False) + + def fail_run_command(_cmd): + raise AssertionError("pip install should not run without approval") + + monkeypatch.setattr(rfdiffusion_tasks, "run_command", fail_run_command) + + assert solver.install() is False + assert solver.installed is False + + +def test_dgl_solver_declines_without_qapplication(monkeypatch): + solver = rfdiffusion_tasks.DglSolver(installed=False) + monkeypatch.setattr(solver, "fetch_cuda_version_before_install", lambda: None) + monkeypatch.setattr(rfdiffusion_tasks, "_has_qapplication", lambda: False) + monkeypatch.delenv(rfdiffusion_tasks.DGL_INSTALL_APPROVAL_ENV, raising=False) + + def fail_decide(**_kwargs): + raise AssertionError("headless DGL install approval should not open a dialog") + + def fail_run_command(_cmd): + raise AssertionError("pip install should not run without approval") + + monkeypatch.setattr(rfdiffusion_tasks, "decide", fail_decide) + monkeypatch.setattr(rfdiffusion_tasks, "run_command", fail_run_command) + + assert solver.install() is False + assert solver.installed is False + + +def test_dgl_solver_env_approval_runs_pip(monkeypatch): + solver = rfdiffusion_tasks.DglSolver(installed=False) + commands = [] + monkeypatch.setattr(solver, "fetch_cuda_version_before_install", lambda: None) + monkeypatch.setenv(rfdiffusion_tasks.DGL_INSTALL_APPROVAL_ENV, "1") + monkeypatch.setattr(rfdiffusion_tasks, "decide", lambda **_kwargs: False) + monkeypatch.setattr( + rfdiffusion_tasks, + "run_command", + lambda cmd: commands.append(cmd) or SimpleNamespace(returncode=0, stderr=""), + ) + + assert solver.install() is True + assert solver.installed is True + assert commands == [ + [ + rfdiffusion_tasks.sys.executable, + "-m", + "pip", + "install", + "dgl==2.2.1", + "-f", + "https://data.dgl.ai/wheels/repo.html", + ] + ] + + +def test_ensure_dgl_raises_when_installation_is_declined(monkeypatch): + solver = rfdiffusion_tasks.DglSolver(installed=False) + monkeypatch.setattr(rfdiffusion_tasks, "DglSolver", lambda: solver) + monkeypatch.setattr(solver, "install", lambda: False) + + with pytest.raises(issues.MissingExternalToolError, match="DGL installation was cancelled"): + rfdiffusion_tasks.RfDiffusion.ensure_dgl() From 9ba7d44efa4e8bd4cf97fbb8fc2ff3c8f7471ec5 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:33:09 +0800 Subject: [PATCH 47/63] fix(config): avoid bootstrap reset prompts in headless runs --- dev/TODO.md | 2 -- src/REvoDesign/bootstrap/set_config.py | 37 ++++++++++++++++++++++---- tests/bootstrap/test_set_config.py | 37 ++++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/dev/TODO.md b/dev/TODO.md index 64381a77..3fad4f2e 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -4,8 +4,6 @@ `src/REvoDesign/clients/QtSocketConnector.py:160-192` still base64-encodes arbitrary `pickle` payloads that are deserialized on receipt, so any collaborator can trigger arbitrary code execution. Replace pickle with a safe schema (msgpack/JSON) and add signature or ACL checks before calling `broadcast`/`digest_dict`. 2. **High – Package manager bootstrapping assumes writable, online site-packages** `src/REvoDesign/tools/package_manager.py:358-404` and `:970-993` always download UI/assets from GitHub gists during startup and write them back into the installed package directory. On read-only installs (system pip, CI wheels) this raises permission errors, and there is no offline cache fallback or signature/timeout on the download. Vendor these assets with the package and gate network calls behind explicit user consent with timeouts. -3. **High – Config bootstrap requires GUI prompts** - `src/REvoDesign/bootstrap/set_config.py:18-88` shows Qt modal dialogs while deciding whether to reset config trees. The bootstrap runs during import (tests call `set_REvoDesign_config_file()`), so headless runs and automation hang waiting for GUI input. Refactor these prompts behind a CLI/flag interface and avoid Qt when `PYMOL_GIT_TEST`/headless. 4. **High – Module-level PyMOL imports make the library unusable outside PyMOL** Modules such as `src/REvoDesign/evaluate/evaluator.py:7-18` and `src/REvoDesign/tools/mutant_tools.py:5-26` import `from pymol import cmd` at import time. Any script, CLI, or test that touches these modules without a PyMOL runtime crashes before code executes. Defer PyMOL imports to call sites or inject an abstraction so non-PyMOL contexts can still use the logic. 5. **High – REvoDesignPlugin changes the global working directory** diff --git a/src/REvoDesign/bootstrap/set_config.py b/src/REvoDesign/bootstrap/set_config.py index 8d87a2d9..a901a978 100644 --- a/src/REvoDesign/bootstrap/set_config.py +++ b/src/REvoDesign/bootstrap/set_config.py @@ -17,13 +17,14 @@ from omegaconf import DictConfig, OmegaConf from platformdirs import user_cache_dir, user_data_dir -from REvoDesign.Qt import QtCompat, QtWidgets +CONFIG_RESET_ENV = "REVODESIGN_RESET_CONFIG" def decide(title="", description="", rich: bool = False, details: str | None = None): """ A copy of decide function from package_manager.py """ + from REvoDesign.Qt import QtCompat, QtWidgets # A confirmation message. msg = QtWidgets.QMessageBox() @@ -40,7 +41,27 @@ def decide(title="", description="", rich: bool = False, details: str | None = N return result == QtCompat.Yes -def set_REvoDesign_config_file(delete_user_config_tree: bool = False): +def _env_flag_enabled(name: str) -> bool: + return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"} + + +def _has_qapplication() -> bool: + try: + from REvoDesign.Qt import QtWidgets + except Exception: + return False + return QtWidgets.QApplication.instance() is not None + + +def _should_prompt_for_config_reset(prompt_user: bool | None) -> bool: + if prompt_user is not None: + return prompt_user + if os.environ.get("PYMOL_GIT_TEST"): + return False + return _has_qapplication() + + +def set_REvoDesign_config_file(delete_user_config_tree: bool = False, prompt_user: bool | None = None): """ Sets the REvoDesign configuration directory. If the main configuration file does not exist, it will be copied from the template directory. If the configuration directory exists, @@ -48,6 +69,8 @@ def set_REvoDesign_config_file(delete_user_config_tree: bool = False): Arguments: delete_user_config_tree (bool): Whether to delete the user's configuration tree. Defaults to False. + prompt_user (bool | None): Whether to ask before resetting an upgrade-time config tree. + ``None`` prompts only when a QApplication exists and test mode is not active. Returns: str: The path to the REvoDesign configuration directory. @@ -63,8 +86,9 @@ def set_REvoDesign_config_file(delete_user_config_tree: bool = False): config_dir = os.path.join(default_storage_path, "config") main_config_file = os.path.join(config_dir, "main.yaml") + reset_requested = delete_user_config_tree or _env_flag_enabled(CONFIG_RESET_ENV) - if delete_user_config_tree and os.path.exists(config_dir): + if reset_requested and os.path.exists(config_dir): print("WARNING: The configuration directory will be deleted as required") shutil.rmtree(config_dir) @@ -82,7 +106,7 @@ def set_REvoDesign_config_file(delete_user_config_tree: bool = False): ) print(reset_warning) - if decide( + if _should_prompt_for_config_reset(prompt_user) and decide( title="Reset REvoDesign Configuration?", description=reset_warning + "Do you want to continue? \n" "You can still choose to cancel the reset to proceed it manually.", @@ -90,7 +114,10 @@ def set_REvoDesign_config_file(delete_user_config_tree: bool = False): shutil.rmtree(config_dir) else: - print("Please manually delete the configuration directory and restart REvoDesign.") + print( + "Configuration reset was not approved; keeping existing files. " + f"Set {CONFIG_RESET_ENV}=1 or pass delete_user_config_tree=True to force a reset." + ) print(f"Copied configurations from {template_config_dir} to {config_dir}") shutil.copytree(src=template_config_dir, dst=config_dir, dirs_exist_ok=True) diff --git a/tests/bootstrap/test_set_config.py b/tests/bootstrap/test_set_config.py index f882d3dc..5be6c9f0 100644 --- a/tests/bootstrap/test_set_config.py +++ b/tests/bootstrap/test_set_config.py @@ -14,7 +14,6 @@ _iter_yaml_rel_paths, experiment_config, is_package_installed, - list_all_config_files, reload_config_file, save_configuration, set_cache_dir, @@ -117,7 +116,7 @@ def test_main_config_upgrade(user_decide, patch_config_user_data): patch("REvoDesign.bootstrap.set_config.shutil.copytree") as mock_copytree, ): - set_REvoDesign_config_file() + set_REvoDesign_config_file(prompt_user=True) mock_decide.assert_called_once() mock_copytree.assert_called_once() if user_decide: @@ -126,6 +125,40 @@ def test_main_config_upgrade(user_decide, patch_config_user_data): mock_rmtree.assert_not_called() +def test_main_config_upgrade_headless_keeps_existing_tree(patch_config_user_data): + with ( + patch("REvoDesign.bootstrap.set_config.decide") as mock_decide, + patch("REvoDesign.bootstrap.set_config.os.path.isfile", return_value=False), + patch("REvoDesign.bootstrap.set_config.os.path.isdir", return_value=True), + patch("REvoDesign.bootstrap.set_config.os.listdir", return_value=["master.yaml", "a_dir"]), + patch("REvoDesign.bootstrap.set_config.shutil.rmtree") as mock_rmtree, + patch("REvoDesign.bootstrap.set_config.shutil.copytree") as mock_copytree, + ): + set_REvoDesign_config_file(prompt_user=False) + + mock_decide.assert_not_called() + mock_rmtree.assert_not_called() + mock_copytree.assert_called_once() + + +def test_main_config_upgrade_env_reset_is_explicit(monkeypatch, patch_config_user_data): + monkeypatch.setenv("REVODESIGN_RESET_CONFIG", "1") + with ( + patch("REvoDesign.bootstrap.set_config.decide") as mock_decide, + patch("REvoDesign.bootstrap.set_config.os.path.isfile", return_value=False), + patch("REvoDesign.bootstrap.set_config.os.path.isdir", return_value=True), + patch("REvoDesign.bootstrap.set_config.os.listdir", return_value=["master.yaml", "a_dir"]), + patch("REvoDesign.bootstrap.set_config.os.path.exists", return_value=True), + patch("REvoDesign.bootstrap.set_config.shutil.rmtree") as mock_rmtree, + patch("REvoDesign.bootstrap.set_config.shutil.copytree") as mock_copytree, + ): + set_REvoDesign_config_file(prompt_user=False) + + mock_decide.assert_not_called() + mock_rmtree.assert_called_once() + mock_copytree.assert_called_once() + + def test_iter_yaml_rel_paths(tmp_path): base_dir = tmp_path / "configs" base_dir.mkdir() From 28bd448745f58770b3b68a707e8d26f13972b93e Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:35:48 +0800 Subject: [PATCH 48/63] fix(plugin): keep working directory scoped --- dev/TODO.md | 2 -- src/REvoDesign/REvoDesign.py | 2 -- tests/test_revodesign_workdir.py | 34 ++++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 tests/test_revodesign_workdir.py diff --git a/dev/TODO.md b/dev/TODO.md index 3fad4f2e..489c2b27 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -6,8 +6,6 @@ `src/REvoDesign/tools/package_manager.py:358-404` and `:970-993` always download UI/assets from GitHub gists during startup and write them back into the installed package directory. On read-only installs (system pip, CI wheels) this raises permission errors, and there is no offline cache fallback or signature/timeout on the download. Vendor these assets with the package and gate network calls behind explicit user consent with timeouts. 4. **High – Module-level PyMOL imports make the library unusable outside PyMOL** Modules such as `src/REvoDesign/evaluate/evaluator.py:7-18` and `src/REvoDesign/tools/mutant_tools.py:5-26` import `from pymol import cmd` at import time. Any script, CLI, or test that touches these modules without a PyMOL runtime crashes before code executes. Defer PyMOL imports to call sites or inject an abstraction so non-PyMOL contexts can still use the logic. -5. **High – REvoDesignPlugin changes the global working directory** - `src/REvoDesign/REvoDesign.py:103-145` calls `os.chdir(self.PWD)` every time a window opens, mutating the process-wide CWD for all other PyMOL plugins and threads. Persisting state should rely on explicit paths stored in the bus, not on moving the interpreter’s CWD. 6. **Medium – ConfigBus is a mutable global singleton storing Qt widget handles** `src/REvoDesign/driver/ui_driver.py:365-381` binds the singleton instance to the active UI and exposes widgets to every module. This leaks Qt objects across tests, prevents multiple windows, and requires manual resets (see `tests/conftest.reset_singletons`). Break ConfigBus into a pure configuration store plus scoped UI adapters so state does not persist globally. 8. **Medium – PIPPack runner calls private APIs of its dependency** diff --git a/src/REvoDesign/REvoDesign.py b/src/REvoDesign/REvoDesign.py index e4776ed6..75425fe4 100644 --- a/src/REvoDesign/REvoDesign.py +++ b/src/REvoDesign/REvoDesign.py @@ -147,8 +147,6 @@ def set_working_directory(self, new_dir: str | None = None): return self.PWD = PWD - os.chdir(self.PWD) - self.bus.set_value("work_dir", os.path.abspath(self.PWD)) def run_plugin_gui(self): diff --git a/tests/test_revodesign_workdir.py b/tests/test_revodesign_workdir.py new file mode 100644 index 00000000..a395c7dc --- /dev/null +++ b/tests/test_revodesign_workdir.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026 The REvoDesign Developers. +# Distributed under the terms of the GNU General Public License v3.0. +# SPDX-License-Identifier: GPL-3.0-only + +from __future__ import annotations + +from pathlib import Path + + +class RecordingBus: + def __init__(self): + self.values = {} + + def set_value(self, key, value): + self.values[key] = value + + +def test_set_working_directory_does_not_mutate_process_cwd(app, monkeypatch, tmp_path): + from REvoDesign.REvoDesign import REvoDesignPlugin + + original_cwd = tmp_path / "original" + target_dir = tmp_path / "target" + original_cwd.mkdir() + target_dir.mkdir() + monkeypatch.chdir(original_cwd) + + plugin = REvoDesignPlugin() + plugin.bus = RecordingBus() + + plugin.set_working_directory(str(target_dir)) + + assert plugin.PWD == str(target_dir) + assert plugin.bus.values["work_dir"] == str(target_dir) + assert Path.cwd() == original_cwd.resolve() From d128ed5c7c821ce9f845103ea126ca32d7ed8459 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:38:43 +0800 Subject: [PATCH 49/63] test(config): isolate bootstrap user dirs --- dev/TODO.md | 2 -- tests/conftest.py | 64 ++++++++++++++++------------------ tests/test_config_isolation.py | 15 ++++++++ 3 files changed, 45 insertions(+), 36 deletions(-) create mode 100644 tests/test_config_isolation.py diff --git a/dev/TODO.md b/dev/TODO.md index 489c2b27..e64e247e 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -12,8 +12,6 @@ `src/REvoDesign/sidechain/mutate_runner/PIPPack.py:64-105` invokes `_initialize_with_a_model`, `_run_repack_single`, and `_run_repack_batch` on the third-party `PIPPack` object. These underscore-prefixed methods are not part of the public API and break whenever upstream refactors; switch to supported entrypoints or wrap the CLI. 11. **High – Importing the logger mutates user state** `src/REvoDesign/logger/logger.py:1-120` configures logging, creates directories under `user_log_path`, and spins up a background `QueueListener` during import. Simply touching `REvoDesign.logger` writes to the host filesystem and leaves live threads around; lazily initialize logging from the entrypoint instead. -13. **High – Tests still touch the real config tree** - `tests/conftest.py:53-124` calls `set_REvoDesign_config_file()` at import time, which may pop up Qt dialogs and can delete the user's `~/Library/Application Support/REvoDesign/config` tree if the user confirms. Mock `platformdirs.user_data_dir`/`user_cache_dir` so tests operate entirely inside the repo. 16. **High – RFdiffusion weight downloads block the UI** `src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py:90-151` fetches multi‑GB checkpoints synchronously inside `RfDiffusion.pick_model`, freezing PyMOL while `pooch` downloads. Move the download to `run_worker_thread_in_pool` and provide progress/abort hooks. 18. **High – Server keeps plaintext credentials in repo** diff --git a/tests/conftest.py b/tests/conftest.py index 1fd60a59..468b107d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,7 @@ # import hydra as unmocked_hydra import psutil import pytest +import platformdirs from _pytest.nodes import Item from immutabledict import immutabledict from pymol import CmdException @@ -35,9 +36,35 @@ from RosettaPy.node import NodeHintT from RosettaPy.utils import tmpdir_manager +REPO_DIR = os.path.join(os.path.dirname(__file__), "..") +TESTS_DIR = Path(__file__).resolve().parent +REPO_ROOT_PATH = TESTS_DIR.parent +MSA_ROOT = TESTS_DIR / "data" / "msa" +MINIUC_ROOT = MSA_ROOT / "miniuc" +TESTMINIUC_ROOT = MSA_ROOT / "testminiuc" + +# Keep import-time REvoDesign bootstrap away from the user's real config/cache tree. +TEST_ROOT = os.path.abspath(".") +DATA_DIRNAME = os.path.join(TEST_ROOT, "mock", "user_data", "REvoDesign") +CACHE_DIRNAME = os.path.join(TEST_ROOT, "..", "tests", "downloaded", "cache") +EXPECTED_MAIN_CONFIG_FILE = os.path.join(DATA_DIRNAME, "config", "main.yaml") +os.makedirs(DATA_DIRNAME, exist_ok=True) +os.makedirs(CACHE_DIRNAME, exist_ok=True) + + +def _static_platform_dir(dirname: str): + def _impl(*args, **kwargs): + Path(dirname).mkdir(parents=True, exist_ok=True) + return dirname + + return _impl + + +platformdirs.user_data_dir = _static_platform_dir(DATA_DIRNAME) +platformdirs.user_cache_dir = _static_platform_dir(CACHE_DIRNAME) + from REvoDesign import REvoDesignPlugin from REvoDesign.basic.abc_singleton import reset_singletons -from REvoDesign.bootstrap import EXPERIMENTS_CONFIG_DIR from REvoDesign.bootstrap.set_config import ConfigConverter, reload_config_file, set_REvoDesign_config_file from REvoDesign.common import MutantTree from REvoDesign.driver.ui_driver import ConfigBus @@ -64,23 +91,6 @@ def _resolve_pytest_qt_api() -> str: PYTEST_QT_API = os.environ.setdefault("PYTEST_QT_API", _resolve_pytest_qt_api()) -REPO_DIR = os.path.join(os.path.dirname(__file__), "..") -TESTS_DIR = Path(__file__).resolve().parent -REPO_ROOT_PATH = TESTS_DIR.parent -MSA_ROOT = TESTS_DIR / "data" / "msa" -MINIUC_ROOT = MSA_ROOT / "miniuc" -TESTMINIUC_ROOT = MSA_ROOT / "testminiuc" - -# mostly mock on data and cache to isolated from user's production system -TEST_ROOT = os.path.abspath(".") -DATA_DIRNAME = os.path.join(TEST_ROOT, "mock", "user_data", "REvoDesign") -CACHE_DIRNAME = os.path.join(TEST_ROOT, "..", "tests", "downloaded", "cache") - -EXPECTED_MAIN_CONFIG_FILE = os.path.join(DATA_DIRNAME, "config", "main.yaml") -os.makedirs(DATA_DIRNAME, exist_ok=True) -os.makedirs(CACHE_DIRNAME, exist_ok=True) - - def copy_config_tree(): print("copy config tree to mock user data") shutil.copytree( @@ -123,21 +133,6 @@ def check_real_config_dir(i: int): # check_real_config_dir(cck.i) -# TODO: isolate from user's system by mocking the platformdirs module. - -# def _static_platform_dir(dirname: str): -# def _impl(*args, **kwargs): -# if not os.path.exists(dirname): -# os.makedirs(dirname) -# return dirname - -# return _impl - - -# platformdirs.user_data_dir = _static_platform_dir(DATA_DIRNAME) -# platformdirs.user_cache_dir = _static_platform_dir(CACHE_DIRNAME) - - # check_real_config_dir(cck.i) # with ( @@ -447,7 +442,8 @@ def load_session_and_check( print(f"nproc: {nproc}") if self.in_which_runner.get("CIRCLECI") and nproc > self.test_data.nproc_circleci: print( - f"Fix nproc to reduce performance for CircleCI: {nproc} {os.cpu_count()}-> {self.test_data.nproc_circleci}" + "Fix nproc to reduce performance for CircleCI: " + f"{nproc} {os.cpu_count()}-> {self.test_data.nproc_circleci}" ) set_widget_value( self.plugin.ui.spinBox_nproc, diff --git a/tests/test_config_isolation.py b/tests/test_config_isolation.py new file mode 100644 index 00000000..533d2521 --- /dev/null +++ b/tests/test_config_isolation.py @@ -0,0 +1,15 @@ +# Copyright (c) 2026 The REvoDesign Developers. +# Distributed under the terms of the GNU General Public License v3.0. +# SPDX-License-Identifier: GPL-3.0-only + +from __future__ import annotations + +from pathlib import Path + +from REvoDesign.bootstrap import REVODESIGN_CONFIG_FILE + +from .conftest import EXPECTED_MAIN_CONFIG_FILE + + +def test_import_time_config_bootstrap_uses_test_data_dir(): + assert Path(REVODESIGN_CONFIG_FILE).resolve() == Path(EXPECTED_MAIN_CONFIG_FILE).resolve() From bc4fd9e9d9f9f54c8a272c790e8163c5aa109fc6 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:43:39 +0800 Subject: [PATCH 50/63] fix(server): remove cluster path defaults --- dev/TODO.md | 4 -- server/.env.example | 3 ++ server/README.md | 7 ++-- server/pssm_gremlin_server/pssm_gremlin.py | 20 ++++----- server/tests/test_config.py | 47 ++++++++++++++++++---- 5 files changed, 54 insertions(+), 27 deletions(-) diff --git a/dev/TODO.md b/dev/TODO.md index e64e247e..264115f1 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -14,10 +14,6 @@ `src/REvoDesign/logger/logger.py:1-120` configures logging, creates directories under `user_log_path`, and spins up a background `QueueListener` during import. Simply touching `REvoDesign.logger` writes to the host filesystem and leaves live threads around; lazily initialize logging from the entrypoint instead. 16. **High – RFdiffusion weight downloads block the UI** `src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py:90-151` fetches multi‑GB checkpoints synchronously inside `RfDiffusion.pick_model`, freezing PyMOL while `pooch` downloads. Move the download to `run_worker_thread_in_pool` and provide progress/abort hooks. -18. **High – Server keeps plaintext credentials in repo** - `server/pssm_gremlin/pssm_gremlin.py:17-36` parses `users.txt` where each line is `username:password` and only hashes them at runtime, so the file on disk stores the raw passwords. Store hashed secrets (or use a proper auth backend) to avoid leaking credentials if the repo/server is compromised. -19. **High – GREMLIN web service hardcodes cluster-only paths** - `server/pssm_gremlin/pssm_gremlin.py:38-120` assumes `/mnt/data/...`, `/mnt/db/...`, and `redis://localhost` exist and derives Docker mounts from `/home/{os.getlogin()}`. With no configuration hooks the service cannot run off that exact cluster. Externalize broker paths, DBs, and output dirs via a config file/env vars. 20. **Medium – Config-driven env overrides clobber user variables** `src/REvoDesign/driver/environ_register.py:8-39` iterates `environ.yaml` and unconditionally sets each key in `os.environ`, overwriting whatever the shell provided (proxy settings, CUDA flags, etc.). Respect existing values or mark overrides explicitly in config. 22. **High – “Background” workers busy-wait on the GUI thread** diff --git a/server/.env.example b/server/.env.example index 9f2d72c5..eb1fe45a 100644 --- a/server/.env.example +++ b/server/.env.example @@ -16,6 +16,9 @@ RUNNER_IMAGE=revodesign-pssm-gremlin-non-root # ============================================================================ # Root directory for uploads, results, and the SQLite task database. SERVER_DIR=/srv/revodesign/server +# Host directory that the Docker runner may bind from. Defaults to the parent +# of SERVER_DIR; set this when SERVER_DIR is under a symlink or alternate mount. +# RUNNER_HOST_ROOT=/srv/revodesign # Directory for Gunicorn and Celery log files. LOG_DIR=/srv/revodesign/logs diff --git a/server/README.md b/server/README.md index a72959fb..ff9d1ef9 100644 --- a/server/README.md +++ b/server/README.md @@ -131,10 +131,11 @@ Fallback when `REVODESIGN_SERVER_ENV` is unset: | Variable | Purpose | | --- | --- | -| `SERVER_DIR` | Host root for uploads, sqlite, and result folders. | +| `SERVER_DIR` | Host root for uploads, sqlite, and result folders (default: `./pssm_gremlin_data`). | +| `RUNNER_HOST_ROOT` | Host root allowed for Docker runner bind mounts (default: parent of `SERVER_DIR`). | | `LOG_DIR` | Host directory for Gunicorn/Celery logs. | -| `DB_UNIREF30` | UniRef30 prefix path. | -| `DB_UNIREF90` | UniRef90 BLAST prefix path. | +| `DB_UNIREF30` | UniRef30 prefix path (default: `{SERVER_DIR}/db/uniref30/UniRef30_2022_02`). | +| `DB_UNIREF90` | UniRef90 BLAST prefix path (default: `{SERVER_DIR}/db/uniref90/uniref90`). | | `AUTH_SECRET_KEY` | Fixed secret for signing auth tokens. Set in production so tokens survive restarts. | | `AUTH_TOKEN_MAX_AGE` | Token lifetime in seconds (default: 604800 = 7 days). | | `USER_DB_PATH` | Path to the user database (default: `{SERVER_DIR}/users.sqlite3`). | diff --git a/server/pssm_gremlin_server/pssm_gremlin.py b/server/pssm_gremlin_server/pssm_gremlin.py index 5ccd2c87..e8912b41 100644 --- a/server/pssm_gremlin_server/pssm_gremlin.py +++ b/server/pssm_gremlin_server/pssm_gremlin.py @@ -14,7 +14,6 @@ import re import shutil import signal -import tempfile import time from dataclasses import dataclass from datetime import datetime @@ -44,6 +43,9 @@ THIS_FILE = os.path.abspath(__file__) THIS_DIR = os.path.dirname(THIS_FILE) TEMPLATE_IMAGE_DIR = os.path.join(THIS_DIR, "templates", "images") +DEFAULT_SERVER_DIR = os.path.abspath(os.path.join(os.getcwd(), "pssm_gremlin_data")) +DEFAULT_UNIREF30_DB = os.path.join(DEFAULT_SERVER_DIR, "db", "uniref30", "UniRef30_2022_02") +DEFAULT_UNIREF90_DB = os.path.join(DEFAULT_SERVER_DIR, "db", "uniref90", "uniref90") app = Flask(__name__, template_folder="./templates") app.config["MAX_CONTENT_LENGTH"] = 16 * 1024 * 1024 # 16 MiB upload limit @@ -170,7 +172,7 @@ class GremlinConfig: @classmethod def from_env(cls) -> GremlinConfig: - server_dir = _env_path("SERVER_DIR", "/mnt/data/yinying/server/") + server_dir = _env_path("SERVER_DIR", DEFAULT_SERVER_DIR) upload_folder = os.path.join(server_dir, "upload") results_folder = os.path.join(server_dir, "results") return cls( @@ -180,11 +182,8 @@ def from_env(cls) -> GremlinConfig: db_path=_env_path("DB_PATH", os.path.join(server_dir, "pssm_gremlin.sqlite3")), docker_image=os.environ.get("RUNNER_IMAGE", "revodesign-pssm-gremlin"), docker_user=_resolve_docker_user(), - uniref30_db=_env_path( - "DB_UNIREF30", - "/mnt/db/uniref30_uc30/UniRef30_2022_02/UniRef30_2022_02", - ), - uniref90_db=_env_path("DB_UNIREF90", "/mnt/db/uniref90/uniref90"), + uniref30_db=_env_path("DB_UNIREF30", DEFAULT_UNIREF30_DB), + uniref90_db=_env_path("DB_UNIREF90", DEFAULT_UNIREF90_DB), nproc=_env_int("NPROC", 16), maxmem=_env_int("MAXMEM", 64), port=_env_int("PORT", 8080), @@ -539,11 +538,8 @@ def _build_running_trace(task: dict[str, Any]) -> str: return "\n".join(traced_lines) -try: - _ROOT_MOUNT_DIRECTORY = f"/home/{os.getlogin()}" -except OSError: - _ROOT_MOUNT_DIRECTORY = os.path.abspath(tempfile.gettempdir()) - os.makedirs(_ROOT_MOUNT_DIRECTORY, exist_ok=True) +_ROOT_MOUNT_DIRECTORY = _env_path("RUNNER_HOST_ROOT", os.path.dirname(CONFIG.server_dir)) +os.makedirs(_ROOT_MOUNT_DIRECTORY, exist_ok=True) def _is_admin_user(username: str | None = None) -> bool: diff --git a/server/tests/test_config.py b/server/tests/test_config.py index 0923a4c8..96e11f4b 100644 --- a/server/tests/test_config.py +++ b/server/tests/test_config.py @@ -5,14 +5,7 @@ from __future__ import annotations import pytest -from conftest import ( - _admin_client_auth, - _extract_md5, - _insert_pending_task, - _load_pssm_module, - _test_client_auth, - _upsert_task_for_user, -) +from conftest import _load_pssm_module # config tests # ================================================================== @@ -42,6 +35,44 @@ def test_pssm_config_uses_named_runner_identity(monkeypatch, tmp_path): assert module.CONFIG.docker_user == "revodesign:revodesign_appgroup" +def test_pssm_config_defaults_are_not_cluster_paths(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + module = _load_pssm_module( + monkeypatch, + tmp_path, + extra_env={ + "RUNNER_UID": "1234", + "RUNNER_GID": "5678", + "SERVER_DIR": None, + "DB_PATH": None, + "DB_UNIREF30": None, + "DB_UNIREF90": None, + "RUNNER_HOST_ROOT": None, + }, + ) + + assert module.CONFIG.server_dir.endswith("pssm_gremlin_data") + assert not module.CONFIG.server_dir.startswith("/mnt/") + assert not module.CONFIG.uniref30_db.startswith("/mnt/") + assert not module.CONFIG.uniref90_db.startswith("/mnt/") + assert module._ROOT_MOUNT_DIRECTORY == str(module.Path(module.CONFIG.server_dir).parent) + + +def test_pssm_config_uses_runner_host_root_override(monkeypatch, tmp_path): + host_root = tmp_path / "runner_host_root" + module = _load_pssm_module( + monkeypatch, + tmp_path, + extra_env={ + "RUNNER_UID": "1234", + "RUNNER_GID": "5678", + "RUNNER_HOST_ROOT": str(host_root), + }, + ) + + assert module._ROOT_MOUNT_DIRECTORY == str(host_root) + + def test_pssm_config_requires_runner_identity(monkeypatch, tmp_path): with pytest.raises(RuntimeError): _load_pssm_module(monkeypatch, tmp_path) From 078f9f501c0401cf4878f9a00104463ecc56cb76 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:46:43 +0800 Subject: [PATCH 51/63] fix(config): preserve existing environment variables --- dev/TODO.md | 2 - docs/getting-started.md | 5 ++ src/REvoDesign/config/environ.yaml | 1 + src/REvoDesign/driver/environ_register.py | 12 ++-- tests/driver/test_environ_register.py | 71 +++++++++++++++++++++++ 5 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 tests/driver/test_environ_register.py diff --git a/dev/TODO.md b/dev/TODO.md index 264115f1..c3975f5d 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -14,8 +14,6 @@ `src/REvoDesign/logger/logger.py:1-120` configures logging, creates directories under `user_log_path`, and spins up a background `QueueListener` during import. Simply touching `REvoDesign.logger` writes to the host filesystem and leaves live threads around; lazily initialize logging from the entrypoint instead. 16. **High – RFdiffusion weight downloads block the UI** `src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py:90-151` fetches multi‑GB checkpoints synchronously inside `RfDiffusion.pick_model`, freezing PyMOL while `pooch` downloads. Move the download to `run_worker_thread_in_pool` and provide progress/abort hooks. -20. **Medium – Config-driven env overrides clobber user variables** - `src/REvoDesign/driver/environ_register.py:8-39` iterates `environ.yaml` and unconditionally sets each key in `os.environ`, overwriting whatever the shell provided (proxy settings, CUDA flags, etc.). Respect existing values or mark overrides explicitly in config. 22. **High – “Background” workers busy-wait on the GUI thread** `src/REvoDesign/tools/package_manager.py:2635-2661` starts a `WorkerThread` but then loops on `work_thread.isFinished()` calling `refresh_window()`/`time.sleep`, so the main thread stays blocked and abort buttons can’t repaint. Switch to signal-driven completion instead of polling. 23. **High – Aborting a worker interrupts the whole PyMOL session** diff --git a/docs/getting-started.md b/docs/getting-started.md index 761afb73..95f58642 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -92,11 +92,16 @@ Example `environ.yaml` (keys are commented out by default; uncomment and set your values): ```yaml +override_existing: false variables: # OPENKINETICS_API_KEY: your-openkinetics-api-key # auto-registers the key # ROSETTA_BIN: /path/to/rosetta/main/source/bin # Rosetta executable directory ``` +`environ.yaml` preserves variables already provided by the shell. Set +`override_existing: true` only when the config file should replace existing +process values. + The OpenKinetics API key is auto-registered by default: `OpenKineticsScorerAbstract` will automatically generate and persist a key on first use, so manual setup in `environ.yaml` is optional and takes diff --git a/src/REvoDesign/config/environ.yaml b/src/REvoDesign/config/environ.yaml index 36d33bab..20875e50 100644 --- a/src/REvoDesign/config/environ.yaml +++ b/src/REvoDesign/config/environ.yaml @@ -1,4 +1,5 @@ # register all variables here +override_existing: false variables: # ROSETTA_BIN: /path/to/rosetta/source/bin/ diff --git a/src/REvoDesign/driver/environ_register.py b/src/REvoDesign/driver/environ_register.py index bbc07113..d93ed209 100644 --- a/src/REvoDesign/driver/environ_register.py +++ b/src/REvoDesign/driver/environ_register.py @@ -26,17 +26,21 @@ def register_environment_variables(): bus.cfg_group["environ"].reload() # retrieve the environment variables from config object - EnvironBindItemCollection: Mapping[str, Any] | None = bus.get_value("variables", dict, cfg="environ") - if not EnvironBindItemCollection: + environ_bind_items: Mapping[str, Any] | None = bus.get_value("variables", dict, cfg="environ") + if not environ_bind_items: logging.debug("No environment variables to register") return + override_existing = bool(bus.get_value("override_existing", bool, cfg="environ", default_value=False)) # register environment variables logging.debug("Registering environment variables") - if isinstance(EnvironBindItemCollection, Mapping): - for k, v in EnvironBindItemCollection.items(): + if isinstance(environ_bind_items, Mapping): + for k, v in environ_bind_items.items(): if v is None: continue + if k in os.environ and not override_existing: + logging.debug("Skipping %s because it already exists in the process environment", k) + continue logging.debug(f"Adding {k}: {v}") os.environ[k] = v logging.debug("Environment variables registered") diff --git a/tests/driver/test_environ_register.py b/tests/driver/test_environ_register.py new file mode 100644 index 00000000..4476055f --- /dev/null +++ b/tests/driver/test_environ_register.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026 The REvoDesign Developers. +# Distributed under the terms of the GNU General Public License v3.0. +# SPDX-License-Identifier: GPL-3.0-only + +from __future__ import annotations + +from REvoDesign.driver import environ_register + + +class FakeConfig: + def __init__(self): + self.reload_count = 0 + + def reload(self): + self.reload_count += 1 + + +class FakeBus: + _instance = object() + + def __init__(self, variables, override_existing=False): + self.cfg_group = {"environ": FakeConfig()} + self.variables = variables + self.override_existing = override_existing + + def get_value(self, cfg_item, converter=None, **_kwargs): + if cfg_item == "variables": + return dict(self.variables) + if cfg_item == "override_existing": + return self.override_existing + raise AssertionError(f"unexpected config item: {cfg_item}") + + +def _patch_bus(monkeypatch, bus): + class FakeConfigBus: + _instance = object() + + def __new__(cls): + return bus + + monkeypatch.setattr(environ_register, "ConfigBus", FakeConfigBus) + + +def test_register_environment_variables_preserves_existing_process_values(monkeypatch): + monkeypatch.setenv("REVODESIGN_TEST_EXISTING", "from-shell") + monkeypatch.delenv("REVODESIGN_TEST_NEW", raising=False) + bus = FakeBus( + { + "REVODESIGN_TEST_EXISTING": "from-yaml", + "REVODESIGN_TEST_NEW": "new-value", + "REVODESIGN_TEST_SKIPPED": None, + } + ) + _patch_bus(monkeypatch, bus) + + environ_register.register_environment_variables() + + assert bus.cfg_group["environ"].reload_count == 1 + assert environ_register.os.environ["REVODESIGN_TEST_EXISTING"] == "from-shell" + assert environ_register.os.environ["REVODESIGN_TEST_NEW"] == "new-value" + assert "REVODESIGN_TEST_SKIPPED" not in environ_register.os.environ + + +def test_register_environment_variables_can_explicitly_override(monkeypatch): + monkeypatch.setenv("REVODESIGN_TEST_EXISTING", "from-shell") + bus = FakeBus({"REVODESIGN_TEST_EXISTING": "from-yaml"}, override_existing=True) + _patch_bus(monkeypatch, bus) + + environ_register.register_environment_variables() + + assert environ_register.os.environ["REVODESIGN_TEST_EXISTING"] == "from-yaml" From 8a0b76c7c4c388b7a5058753bd2d040d8c954243 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:48:57 +0800 Subject: [PATCH 52/63] fix(workers): avoid global pymol interrupt on abort --- dev/TODO.md | 2 -- src/REvoDesign/tools/package_manager.py | 5 +---- tests/tools/test_package_manager.py | 23 ++++++++++++++++++++++- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/dev/TODO.md b/dev/TODO.md index c3975f5d..a0564404 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -16,8 +16,6 @@ `src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py:90-151` fetches multi‑GB checkpoints synchronously inside `RfDiffusion.pick_model`, freezing PyMOL while `pooch` downloads. Move the download to `run_worker_thread_in_pool` and provide progress/abort hooks. 22. **High – “Background” workers busy-wait on the GUI thread** `src/REvoDesign/tools/package_manager.py:2635-2661` starts a `WorkerThread` but then loops on `work_thread.isFinished()` calling `refresh_window()`/`time.sleep`, so the main thread stays blocked and abort buttons can’t repaint. Switch to signal-driven completion instead of polling. -23. **High – Aborting a worker interrupts the whole PyMOL session** - `src/REvoDesign/tools/package_manager.py:2223-2244` handles every abort click by calling `cmd.interrupt()`, which cancels whatever PyMOL is currently doing—even unrelated jobs. Terminate only the subprocesses registered for that worker rather than nuking the global interpreter. 24. **High – WebSocket clients hand over the shared auth key on request** `src/REvoDesign/clients/QtSocketConnector.py:813-863` responds to a `RequireKey` message by sending `self.authentication_key` before the server proves its identity. A malicious host can simply ask for the key and impersonate everyone. Authenticate the server (TLS + pinned cert) and only send a key through an already-authenticated channel. 25. **Medium – Git tag fetches are synchronous and unauthenticated** diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index 77caeac5..bf97fe30 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -2352,11 +2352,8 @@ def _connect_signals(self) -> None: def _handle_abort_request(self) -> None: self._aborted = True - try: - cmd.interrupt() - except Exception as exc: - logging.debug("Could not interrupt PyMOL command loop during worker abort: %s", exc) self.worker_thread.interrupt() + RunningProcessRegistry.terminate(self.worker_thread) if self.worker_thread.isRunning(): self.worker_thread.quit() if not self.worker_thread.wait(100): diff --git a/tests/tools/test_package_manager.py b/tests/tools/test_package_manager.py index 72cdb62a..5d788f84 100644 --- a/tests/tools/test_package_manager.py +++ b/tests/tools/test_package_manager.py @@ -732,6 +732,22 @@ def test_pm_thread_manage_kill_entry(qtbot, monkeypatch): button = QtWidgets.QPushButton("Killable") qtbot.addWidget(button) + class FakeProcess: + terminated = False + killed = False + + def poll(self): + return None + + def terminate(self): + self.terminated = True + + def wait(self, timeout=None): + return 0 + + def kill(self): + self.killed = True + def cancellable_task(): thread = package_manager.QtCore.QThread.currentThread() start = time.monotonic() @@ -743,6 +759,9 @@ def cancellable_task(): worker_thread = package_manager.WorkerThread(cancellable_task) manager = package_manager.ThreadExecutionManager(worker_thread, description="Kill", trigger_buttons=button) + fake_process = FakeProcess() + package_manager.RunningProcessRegistry._worker_processes[id(worker_thread)] = {fake_process} + package_manager.RunningProcessRegistry._process_worker[id(fake_process)] = id(worker_thread) assert manager.abort_overlays mock_cmd = SimpleNamespace(interrupt=MagicMock()) @@ -760,7 +779,9 @@ def cancellable_task(): result = worker_thread.handle_result() assert manager.abort_overlays == [] - assert mock_cmd.interrupt.called + assert not mock_cmd.interrupt.called + assert fake_process.terminated + assert not fake_process.killed assert result is None or result[0] == "aborted" From 8bd1947273234558c2cc75d9f8fd3cdaf472a1f7 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:53:51 +0800 Subject: [PATCH 53/63] fix(package-manager): cache github tag lookups --- dev/TODO.md | 2 - src/REvoDesign/tools/package_manager.py | 42 +++++++---- tests/tools/test_package_manager.py | 94 +++++++++++++++---------- 3 files changed, 84 insertions(+), 54 deletions(-) diff --git a/dev/TODO.md b/dev/TODO.md index a0564404..e4644b4e 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -18,8 +18,6 @@ `src/REvoDesign/tools/package_manager.py:2635-2661` starts a `WorkerThread` but then loops on `work_thread.isFinished()` calling `refresh_window()`/`time.sleep`, so the main thread stays blocked and abort buttons can’t repaint. Switch to signal-driven completion instead of polling. 24. **High – WebSocket clients hand over the shared auth key on request** `src/REvoDesign/clients/QtSocketConnector.py:813-863` responds to a `RequireKey` message by sending `self.authentication_key` before the server proves its identity. A malicious host can simply ask for the key and impersonate everyone. Authenticate the server (TLS + pinned cert) and only send a key through an already-authenticated channel. -25. **Medium – Git tag fetches are synchronous and unauthenticated** - `src/REvoDesign/tools/package_manager.py:2349-2380` hits the public GitHub tags API on startup without auth, caching, or error UI. When the rate limit is exceeded the manager freezes for ~10s and users see nothing. Add caching, a timeout, and a friendly error surface. 26. **Medium – pip installs run on the UI thread** `src/REvoDesign/tools/package_manager.py:861-925` calls `pip install/uninstall` directly from menu actions, blocking PyMOL for minutes during heavy installs. Dispatch these calls to `run_worker_thread_in_pool` and stream progress to the dashboard. 27. **Medium – Archive browsing keeps every extraction forever** diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index bf97fe30..b6383af7 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -237,15 +237,21 @@ def _safe_parse_version(value: str) -> Version | None: return None -def _read_https_url(url: str, timeout: float = 10.0) -> bytes: +def _read_https_url(url: str, timeout: float = 10.0, headers: Mapping[str, str] | None = None) -> bytes: parsed = urlparse(url) if parsed.scheme != "https" or not parsed.netloc: raise ValueError(f"Only https URLs are allowed: {url}") - request = urllib.request.Request(url, headers={"User-Agent": "REvoDesign-PackageManager"}) + request_headers = {"User-Agent": "REvoDesign-PackageManager"} + if headers: + request_headers.update(headers) + request = urllib.request.Request(url, headers=request_headers) with urllib.request.urlopen(request, timeout=timeout) as response: # nosec B310 return response.read() +_GITHUB_TAG_CACHE: dict[str, list[str]] = {} + + def _python_version_matches(spec: str | None, current_version: str | None) -> bool: """ Check if the current Python version matches the specified version specification. @@ -2481,7 +2487,7 @@ def _handle_interrupt(self) -> None: logging.debug("WorkerThread interrupt signal handled.") -def get_github_repo_tags(repo_url) -> list[str]: +def get_github_repo_tags(repo_url: str, *, timeout: float = 5.0) -> list[str]: """ Retrieve all released tags of a GitHub repository using urllib. @@ -2496,43 +2502,51 @@ def get_github_repo_tags(repo_url) -> list[str]: list: A list of tag names for the repository. """ - # Extract the owner and repo name from the URL - parts = repo_url.split("/") - owner = parts[-2] - repo = parts[-1] + parsed = urlparse(repo_url) + parts = [part for part in parsed.path.strip("/").split("/") if part] + if parsed.scheme != "https" or parsed.netloc.lower() != "github.com" or len(parts) < 2: + logging.warning("Unsupported GitHub repository URL: %s", repo_url) + return [] + owner, repo = parts[:2] + repo = repo.removesuffix(".git") # GitHub API URL for listing tags api_url = f"https://api.github.com/repos/{owner}/{repo}/tags" + headers = {"Accept": "application/vnd.github+json"} + github_token = os.environ.get("GITHUB_TOKEN", "").strip() + if github_token: + headers["Authorization"] = f"Bearer {github_token}" try: # Send a GET request to the GitHub API - response_data = _read_https_url(api_url).decode() + response_data = _read_https_url(api_url, timeout=timeout, headers=headers).decode() # Parse JSON response data tags = json.loads(response_data) if not isinstance(tags, list): logging.error("Failed to parse GitHub tags response: expected a list.") - return [] + return _GITHUB_TAG_CACHE.get(api_url, []) # Extract the name of each tag try: tag_names = [tag["name"] for tag in tags] except (KeyError, TypeError) as e: logging.error(f"Failed to extract tag names from GitHub response: {e}") - return [] + return _GITHUB_TAG_CACHE.get(api_url, []) + _GITHUB_TAG_CACHE[api_url] = tag_names return tag_names except HTTPError as e: # Handle HTTP errors (e.g., repository not found, rate limit exceeded) logging.warning(f"GitHub API returned status code {e.code}") - return [] + return _GITHUB_TAG_CACHE.get(api_url, []) except URLError as e: # Handle URL errors (e.g., network issues) logging.error(f"Failed to reach the server. Reason: {e.reason}") - return [] + return _GITHUB_TAG_CACHE.get(api_url, []) except json.JSONDecodeError as e: logging.error(f"Failed to decode GitHub tags response as JSON: {e}") - return [] + return _GITHUB_TAG_CACHE.get(api_url, []) except TypeError as e: logging.error(f"Failed to parse GitHub tags response: {e}") - return [] + return _GITHUB_TAG_CACHE.get(api_url, []) # a minimum copy from `REvoDesign/tools/customized_widgets.py` diff --git a/tests/tools/test_package_manager.py b/tests/tools/test_package_manager.py index 5d788f84..5bb976fa 100644 --- a/tests/tools/test_package_manager.py +++ b/tests/tools/test_package_manager.py @@ -449,30 +449,32 @@ def test_pm_ensure_package(pip_installer, mocker): class TestGetGithubRepoTags: - def test_pm_valid_repo_url(self): - # Test a valid repository URL, expecting a list of tags + def test_pm_valid_repo_url(self, monkeypatch): repo_url = "https://github.com/BradyAJohnston/MolecularNodes" + monkeypatch.setattr( + package_manager, + "_read_https_url", + lambda url, **_kwargs: json.dumps([{"name": "v1.0.0"}, {"name": "v1.1.0"}]).encode(), + ) + tags = get_github_repo_tags(repo_url) - if not tags: - pytest.skip("Live GitHub tags are unavailable in the current test environment.") - assert isinstance(tags, list) - assert len(tags) > 0 - for tag in tags: - assert isinstance(tag, str) + + assert tags == ["v1.0.0", "v1.1.0"] def test_pm_invalid_repo_url(self): - # Test an invalid repository URL, expecting an empty list - repo_url = "https://github.com/nonexistent/repo" + package_manager._GITHUB_TAG_CACHE.clear() + repo_url = "https://example.com/nonexistent/repo" tags = get_github_repo_tags(repo_url) assert isinstance(tags, list) assert len(tags) == 0 def test_pm_http_error(self, monkeypatch): - # Test handling of HTTPError - def mock_urlopen(*args, **kwargs): - raise HTTPError(args[0], 404, "Not Found", None, None) + package_manager._GITHUB_TAG_CACHE.clear() + + def mock_read(url, **_kwargs): + raise HTTPError(url, 404, "Not Found", None, None) - monkeypatch.setattr(urllib.request, "urlopen", mock_urlopen) + monkeypatch.setattr(package_manager, "_read_https_url", mock_read) repo_url = "https://github.com/BradyAJohnston/MolecularNodes" tags = get_github_repo_tags(repo_url) @@ -480,11 +482,12 @@ def mock_urlopen(*args, **kwargs): assert len(tags) == 0 def test_pm_url_error(self, monkeypatch): - # Test handling of URLError - def mock_urlopen(*args, **kwargs): + package_manager._GITHUB_TAG_CACHE.clear() + + def mock_read(*_args, **_kwargs): raise URLError("Failed to reach the server") - monkeypatch.setattr(urllib.request, "urlopen", mock_urlopen) + monkeypatch.setattr(package_manager, "_read_https_url", mock_read) repo_url = "https://github.com/BradyAJohnston/MolecularNodes" tags = get_github_repo_tags(repo_url) @@ -492,32 +495,47 @@ def mock_urlopen(*args, **kwargs): assert len(tags) == 0 -def test_pm_get_github_repo_tags_success(): - """Test successful retrieval of tags.""" +def test_pm_get_github_repo_tags_uses_timeout_auth_and_cache(monkeypatch): + calls = [] - result = get_github_repo_tags("https://github.com/BradyAJohnston/MolecularNodes") - if not result: - pytest.skip("Live GitHub tags are unavailable in the current test environment.") - assert result, "Github repository tags should not be empty" + def mock_read(url, **kwargs): + calls.append((url, kwargs)) + return json.dumps([{"name": "v2.0.0"}]).encode() + monkeypatch.setenv("GITHUB_TOKEN", "token-123") + monkeypatch.setattr(package_manager, "_read_https_url", mock_read) + package_manager._GITHUB_TAG_CACHE.clear() -def test_pm_get_github_repo_tags_http_error(): - """Test handling of an HTTPError.""" - with patch( - "urllib.request.urlopen", - side_effect=urllib.error.HTTPError( - url="https://api.github.com/repos/test_owner/test_repo/tags", code=404, msg="Not Found", hdrs=None, fp=None - ), - ): - result = get_github_repo_tags("https://github.com/test_owner/test_repo") - assert result == [] + result = get_github_repo_tags("https://github.com/test_owner/test_repo", timeout=3) + + assert result == ["v2.0.0"] + assert calls == [ + ( + "https://api.github.com/repos/test_owner/test_repo/tags", + { + "timeout": 3, + "headers": { + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token-123", + }, + }, + ) + ] + assert package_manager._GITHUB_TAG_CACHE["https://api.github.com/repos/test_owner/test_repo/tags"] == ["v2.0.0"] + + +def test_pm_get_github_repo_tags_falls_back_to_cached_tags(monkeypatch): + api_url = "https://api.github.com/repos/test_owner/test_repo/tags" + package_manager._GITHUB_TAG_CACHE[api_url] = ["cached-tag"] + + def mock_read(url, **_kwargs): + raise urllib.error.URLError(reason="Network unreachable") + + monkeypatch.setattr(package_manager, "_read_https_url", mock_read) + result = get_github_repo_tags("https://github.com/test_owner/test_repo") -def test_pm_get_github_repo_tags_url_error(): - """Test handling of a URLError.""" - with patch("urllib.request.urlopen", side_effect=urllib.error.URLError(reason="Network unreachable")): - result = get_github_repo_tags("https://github.com/test_owner/test_repo") - assert result == [] + assert result == ["cached-tag"] @pytest.mark.parametrize( From 69c0d1840c9db41fe91d1a50a421cb2f10fc27ea Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:57:25 +0800 Subject: [PATCH 54/63] fix(file-dialog): extract archives into temp dirs --- dev/TODO.md | 2 -- src/REvoDesign/driver/file_dialog.py | 47 +++++++++++++++++-------- tests/driver/test_file_dialog.py | 52 ++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 16 deletions(-) create mode 100644 tests/driver/test_file_dialog.py diff --git a/dev/TODO.md b/dev/TODO.md index e4644b4e..73756f26 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -20,5 +20,3 @@ `src/REvoDesign/clients/QtSocketConnector.py:813-863` responds to a `RequireKey` message by sending `self.authentication_key` before the server proves its identity. A malicious host can simply ask for the key and impersonate everyone. Authenticate the server (TLS + pinned cert) and only send a key through an already-authenticated channel. 26. **Medium – pip installs run on the UI thread** `src/REvoDesign/tools/package_manager.py:861-925` calls `pip install/uninstall` directly from menu actions, blocking PyMOL for minutes during heavy installs. Dispatch these calls to `run_worker_thread_in_pool` and stream progress to the dashboard. -27. **Medium – Archive browsing keeps every extraction forever** - `src/REvoDesign/driver/file_dialog.py:240-269` always expands compressed inputs into `/expanded_compressed_files/` with no cleanup or quota. Repeated browsing bloats the session dir and a malicious 20 GB tar can fill the disk. Extract into a temp dir and delete it once the dialog closes. diff --git a/src/REvoDesign/driver/file_dialog.py b/src/REvoDesign/driver/file_dialog.py index af8e51dc..6175bb8c 100644 --- a/src/REvoDesign/driver/file_dialog.py +++ b/src/REvoDesign/driver/file_dialog.py @@ -8,6 +8,8 @@ """ import os +import shutil +import tempfile from functools import partial from typing import Any, Literal @@ -24,6 +26,8 @@ logging = ROOT_LOGGER.getChild(__name__) +MAX_ARCHIVE_BROWSE_BYTES = 2 * 1024 * 1024 * 1024 + class FileDialog(SingletonAbstract): """ @@ -45,6 +49,7 @@ def singleton_init(self, window: Any | None, pwd: str | None): """ self.window = window self.PWD = pwd or os.getcwd() + self._archive_temp_dirs: set[str] = set() self.register_file_dialog_buttons() # Mark the instance as initialized to prevent reinitialization self.initialize() @@ -110,7 +115,8 @@ def browse_filename( return filename # otherwise, extract the archive and browse the extracted file - flatten_compressed_files(filename, self.PWD) + extracted_dir = flatten_compressed_files(filename) + self._archive_temp_dirs.add(extracted_dir) return self.browse_filename(mode, exts=exts) # A universal and versatile function for input file path browsing. @@ -275,34 +281,47 @@ def register_file_dialog_buttons(self): partial(self.open_mutant_table, "ui.evaluate.input.to_mutant_txt", "w") ) + def cleanup_archive_dirs(self) -> None: + temp_dirs = getattr(self, "_archive_temp_dirs", set()) + for temp_dir in list(temp_dirs): + shutil.rmtree(temp_dir, ignore_errors=True) + temp_dirs.clear() + + def __del__(self): + self.cleanup_archive_dirs() + -def flatten_compressed_files(compressed_file: str, target_dir: str | None = None) -> str: +def flatten_compressed_files( + compressed_file: str, + target_dir: str | None = None, + max_archive_bytes: int = MAX_ARCHIVE_BROWSE_BYTES, +) -> str: """ Flattens and extracts the contents of a compressed file. Parameters: - compressed_file (str): The path to the compressed file to be extracted. - - target_dir (Optional[str]): The directory where the extracted files will be placed. - If not provided, the current working directory is used. + - target_dir (Optional[str]): Parent directory for the temporary extraction directory. + If not provided, the system temporary directory is used. Returns: - str: The path to the directory where the files have been extracted. """ - # Set the target directory to the current working directory if not specified - if target_dir is None: - target_dir = os.getcwd() + archive_size = os.path.getsize(compressed_file) + if archive_size > max_archive_bytes: + raise ValueError( + f"Archive is too large to browse safely: {archive_size} bytes " + f"(limit: {max_archive_bytes} bytes)" + ) # Create a path for the extracted files - flatten_path = os.path.join( - target_dir, - "expanded_compressed_files", - os.path.basename(compressed_file), + temp_parent = target_dir or tempfile.gettempdir() + flatten_path = tempfile.mkdtemp( + prefix=f"revodesign-{os.path.basename(compressed_file)}-", + dir=temp_parent, ) - # Create the directory if it does not exist - os.makedirs(flatten_path, exist_ok=True) - # Extract the archive to the specified directory extract_archive(archive_file=compressed_file, extract_to=flatten_path) diff --git a/tests/driver/test_file_dialog.py b/tests/driver/test_file_dialog.py new file mode 100644 index 00000000..9c424cf3 --- /dev/null +++ b/tests/driver/test_file_dialog.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026 The REvoDesign Developers. +# Distributed under the terms of the GNU General Public License v3.0. +# SPDX-License-Identifier: GPL-3.0-only + +from __future__ import annotations + +from pathlib import Path +import zipfile + +import pytest + +from REvoDesign.driver.file_dialog import FileDialog, flatten_compressed_files + + +def _write_zip(path: Path, member_name: str = "payload.txt", content: str = "payload") -> None: + with zipfile.ZipFile(path, "w") as zip_file: + zip_file.writestr(member_name, content) + + +def test_flatten_compressed_files_extracts_to_temporary_directory(tmp_path): + archive = tmp_path / "input.zip" + _write_zip(archive) + + extracted_dir = Path(flatten_compressed_files(str(archive), target_dir=str(tmp_path))) + + assert extracted_dir.is_dir() + assert extracted_dir.parent == tmp_path + assert extracted_dir.name.startswith("revodesign-input.zip-") + assert not (tmp_path / "expanded_compressed_files").exists() + assert (extracted_dir / "payload.txt").read_text() == "payload" + + +def test_flatten_compressed_files_rejects_oversized_archive(tmp_path): + archive = tmp_path / "input.zip" + _write_zip(archive) + + with pytest.raises(ValueError, match="Archive is too large"): + flatten_compressed_files(str(archive), target_dir=str(tmp_path), max_archive_bytes=1) + + +def test_file_dialog_cleanup_archive_dirs_removes_temporary_extractions(tmp_path): + temp_dir = tmp_path / "revodesign-input.zip-test" + temp_dir.mkdir() + (temp_dir / "payload.txt").write_text("payload") + + dialog = object.__new__(FileDialog) + dialog._archive_temp_dirs = {str(temp_dir)} + + dialog.cleanup_archive_dirs() + + assert not temp_dir.exists() + assert dialog._archive_temp_dirs == set() From 5a49e6867809178e89d2cb943a4f8cfcf856e5fb Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:04:40 +0800 Subject: [PATCH 55/63] fix(audit): address archive and tag cache regressions --- src/REvoDesign/driver/file_dialog.py | 41 ++++++++++++++++++++++--- src/REvoDesign/tools/package_manager.py | 22 ++++++++----- tests/driver/test_file_dialog.py | 39 +++++++++++++++++++++++ tests/tools/test_package_manager.py | 12 ++++++-- 4 files changed, 100 insertions(+), 14 deletions(-) diff --git a/src/REvoDesign/driver/file_dialog.py b/src/REvoDesign/driver/file_dialog.py index 6175bb8c..b5a94ad1 100644 --- a/src/REvoDesign/driver/file_dialog.py +++ b/src/REvoDesign/driver/file_dialog.py @@ -9,7 +9,9 @@ import os import shutil +import tarfile import tempfile +import zipfile from functools import partial from typing import Any, Literal @@ -63,7 +65,10 @@ def browse_multiple_files( return getMultipleFiles(self.window, exts) def browse_filename( - self, mode: IO_MODE = "r", exts: tuple[FileExtensionCollection, ...] = (file_extensions.Any,) + self, + mode: IO_MODE = "r", + exts: tuple[FileExtensionCollection, ...] = (file_extensions.Any,), + directory: str | None = None, ) -> str | None: """Open Finder/Explorer to browse from a filename @@ -88,7 +93,12 @@ def browse_filename( # otherwise, open a file to read browse_title = "Open ..." - filename = getOpenFileNameWithExt(self.window, browse_title, filter=filter_strings) + filename = getOpenFileNameWithExt( + self.window, + browse_title, + directory or "", + filter=filter_strings, + ) # no file selected if not filename: @@ -117,7 +127,7 @@ def browse_filename( # otherwise, extract the archive and browse the extracted file extracted_dir = flatten_compressed_files(filename) self._archive_temp_dirs.add(extracted_dir) - return self.browse_filename(mode, exts=exts) + return self.browse_filename(mode, exts=exts, directory=extracted_dir) # A universal and versatile function for input file path browsing. @@ -314,6 +324,12 @@ def flatten_compressed_files( f"Archive is too large to browse safely: {archive_size} bytes " f"(limit: {max_archive_bytes} bytes)" ) + payload_size = _compressed_archive_payload_size(compressed_file) + if payload_size > max_archive_bytes: + raise ValueError( + f"Archive expands too large to browse safely: {payload_size} bytes " + f"(limit: {max_archive_bytes} bytes)" + ) # Create a path for the extracted files temp_parent = target_dir or tempfile.gettempdir() @@ -323,7 +339,24 @@ def flatten_compressed_files( ) # Extract the archive to the specified directory - extract_archive(archive_file=compressed_file, extract_to=flatten_path) + try: + extract_archive(archive_file=compressed_file, extract_to=flatten_path) + except Exception: + shutil.rmtree(flatten_path, ignore_errors=True) + raise # Return the path to the extracted directory return flatten_path + + +def _compressed_archive_payload_size(compressed_file: str) -> int: + if compressed_file.endswith(".zip"): + with zipfile.ZipFile(compressed_file, "r") as zip_ref: + return sum(info.file_size for info in zip_ref.infolist() if not info.is_dir()) + + tar_suffixes = (".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz", ".tar.xz") + if compressed_file.endswith(tar_suffixes): + with tarfile.open(compressed_file, "r:*") as tar_ref: + return sum(member.size for member in tar_ref.getmembers() if member.isfile()) + + return os.path.getsize(compressed_file) diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index b6383af7..5e7cd065 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -249,7 +249,8 @@ def _read_https_url(url: str, timeout: float = 10.0, headers: Mapping[str, str] return response.read() -_GITHUB_TAG_CACHE: dict[str, list[str]] = {} +GITHUB_TAG_CACHE_TTL_SECONDS = 15 * 60 +_GITHUB_TAG_CACHE: dict[str, tuple[float, list[str]]] = {} def _python_version_matches(spec: str | None, current_version: str | None) -> bool: @@ -2512,6 +2513,11 @@ def get_github_repo_tags(repo_url: str, *, timeout: float = 5.0) -> list[str]: # GitHub API URL for listing tags api_url = f"https://api.github.com/repos/{owner}/{repo}/tags" + now = time.monotonic() + cached = _GITHUB_TAG_CACHE.get(api_url) + if cached and now - cached[0] <= GITHUB_TAG_CACHE_TTL_SECONDS: + return list(cached[1]) + headers = {"Accept": "application/vnd.github+json"} github_token = os.environ.get("GITHUB_TOKEN", "").strip() if github_token: @@ -2524,29 +2530,29 @@ def get_github_repo_tags(repo_url: str, *, timeout: float = 5.0) -> list[str]: tags = json.loads(response_data) if not isinstance(tags, list): logging.error("Failed to parse GitHub tags response: expected a list.") - return _GITHUB_TAG_CACHE.get(api_url, []) + return list(cached[1]) if cached else [] # Extract the name of each tag try: tag_names = [tag["name"] for tag in tags] except (KeyError, TypeError) as e: logging.error(f"Failed to extract tag names from GitHub response: {e}") - return _GITHUB_TAG_CACHE.get(api_url, []) - _GITHUB_TAG_CACHE[api_url] = tag_names + return list(cached[1]) if cached else [] + _GITHUB_TAG_CACHE[api_url] = (time.monotonic(), tag_names) return tag_names except HTTPError as e: # Handle HTTP errors (e.g., repository not found, rate limit exceeded) logging.warning(f"GitHub API returned status code {e.code}") - return _GITHUB_TAG_CACHE.get(api_url, []) + return list(cached[1]) if cached else [] except URLError as e: # Handle URL errors (e.g., network issues) logging.error(f"Failed to reach the server. Reason: {e.reason}") - return _GITHUB_TAG_CACHE.get(api_url, []) + return list(cached[1]) if cached else [] except json.JSONDecodeError as e: logging.error(f"Failed to decode GitHub tags response as JSON: {e}") - return _GITHUB_TAG_CACHE.get(api_url, []) + return list(cached[1]) if cached else [] except TypeError as e: logging.error(f"Failed to parse GitHub tags response: {e}") - return _GITHUB_TAG_CACHE.get(api_url, []) + return list(cached[1]) if cached else [] # a minimum copy from `REvoDesign/tools/customized_widgets.py` diff --git a/tests/driver/test_file_dialog.py b/tests/driver/test_file_dialog.py index 9c424cf3..07954616 100644 --- a/tests/driver/test_file_dialog.py +++ b/tests/driver/test_file_dialog.py @@ -38,6 +38,21 @@ def test_flatten_compressed_files_rejects_oversized_archive(tmp_path): flatten_compressed_files(str(archive), target_dir=str(tmp_path), max_archive_bytes=1) +def test_flatten_compressed_files_rejects_large_extracted_payload(tmp_path): + archive = tmp_path / "compressed.zip" + payload = "A" * 4096 + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as zip_file: + zip_file.writestr("payload.txt", payload) + + assert archive.stat().st_size < len(payload) + with pytest.raises(ValueError, match="Archive expands too large"): + flatten_compressed_files( + str(archive), + target_dir=str(tmp_path), + max_archive_bytes=archive.stat().st_size + 1, + ) + + def test_file_dialog_cleanup_archive_dirs_removes_temporary_extractions(tmp_path): temp_dir = tmp_path / "revodesign-input.zip-test" temp_dir.mkdir() @@ -50,3 +65,27 @@ def test_file_dialog_cleanup_archive_dirs_removes_temporary_extractions(tmp_path assert not temp_dir.exists() assert dialog._archive_temp_dirs == set() + + +def test_browse_filename_reopens_archive_temp_directory(monkeypatch, tmp_path): + archive = tmp_path / "input.zip" + _write_zip(archive) + extracted_dir = tmp_path / "revodesign-input.zip-test" + picked_file = extracted_dir / "payload.txt" + dialogs = [] + + def mock_get_open_file_name(_window, _title, directory="", **_kwargs): + dialogs.append(directory) + return str(archive) if len(dialogs) == 1 else str(picked_file) + + monkeypatch.setattr("REvoDesign.driver.file_dialog.decide", lambda **_kwargs: True) + monkeypatch.setattr("REvoDesign.driver.file_dialog.flatten_compressed_files", lambda _path: str(extracted_dir)) + monkeypatch.setattr("REvoDesign.driver.file_dialog.getOpenFileNameWithExt", mock_get_open_file_name) + + dialog = object.__new__(FileDialog) + dialog.window = None + dialog._archive_temp_dirs = set() + + assert dialog.browse_filename() == str(picked_file) + assert dialogs == ["", str(extracted_dir)] + assert dialog._archive_temp_dirs == {str(extracted_dir)} diff --git a/tests/tools/test_package_manager.py b/tests/tools/test_package_manager.py index 5bb976fa..70ae2acd 100644 --- a/tests/tools/test_package_manager.py +++ b/tests/tools/test_package_manager.py @@ -450,6 +450,7 @@ def test_pm_ensure_package(pip_installer, mocker): class TestGetGithubRepoTags: def test_pm_valid_repo_url(self, monkeypatch): + package_manager._GITHUB_TAG_CACHE.clear() repo_url = "https://github.com/BradyAJohnston/MolecularNodes" monkeypatch.setattr( package_manager, @@ -521,12 +522,19 @@ def mock_read(url, **kwargs): }, ) ] - assert package_manager._GITHUB_TAG_CACHE["https://api.github.com/repos/test_owner/test_repo/tags"] == ["v2.0.0"] + cached_at, cached_tags = package_manager._GITHUB_TAG_CACHE[ + "https://api.github.com/repos/test_owner/test_repo/tags" + ] + assert cached_at > 0 + assert cached_tags == ["v2.0.0"] + + assert get_github_repo_tags("https://github.com/test_owner/test_repo", timeout=3) == ["v2.0.0"] + assert len(calls) == 1 def test_pm_get_github_repo_tags_falls_back_to_cached_tags(monkeypatch): api_url = "https://api.github.com/repos/test_owner/test_repo/tags" - package_manager._GITHUB_TAG_CACHE[api_url] = ["cached-tag"] + package_manager._GITHUB_TAG_CACHE[api_url] = (0.0, ["cached-tag"]) def mock_read(url, **_kwargs): raise urllib.error.URLError(reason="Network unreachable") From 752453b837e92ee070b35f0b67fee7f5bb251763 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Sun, 19 Jul 2026 17:40:23 +0800 Subject: [PATCH 56/63] fix(package-manager): cache-first startup, HMAC-verified self-upgrade - ensure_ui_file: restore fetch-on-missing path, cache to writable dir - fetch_gist_file: add timeout parameter (default 10s) - self_upgrade: HMAC-verify downloaded assets against manifest.json - fetch_tags: silent degrade on network failure (no popup) - make_window: re-fetch on loadUi failure instead of os.remove+recurse - refresh_remote_json: cache successful fetches to disk - _load_extras_table: shared loader with notify_on_error flag - Makefile: upload-gists generates manifest.json automatically - validate_package_data: check canonical Gist sources, not vendored copies - Remove TODO.md entry for package manager bootstrapping (addressed) - Update CHANGELOG and package-manager docs Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + CHANGELOG.md | 10 ++ Makefile | 13 ++ dev/TODO.md | 2 - dev/tools/validate_package_data.py | 2 + docs/dev-guide/package-manager.md | 22 ++- src/REvoDesign/tools/package_manager.py | 214 ++++++++++++++++++------ tests/tools/test_package_manager.py | 163 +++++++++++++++++- 8 files changed, 363 insertions(+), 64 deletions(-) diff --git a/.gitignore b/.gitignore index 15d2b60d..b1b0ab76 100644 --- a/.gitignore +++ b/.gitignore @@ -143,3 +143,4 @@ users.sqlite3 testerror.txt /server/server_state /server/pssm_gremlin_server.egg-info +/.tmp diff --git a/CHANGELOG.md b/CHANGELOG.md index a722eb10..7577fea8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,6 +130,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Renamed Simplified Chinese label from 中文 to 简体中文 in language registry (`language.json`). ### Fixed +- **Package manager offline startup**: manager UI and extras registry now use a + cache-first strategy with the local `~/.pymol/startup/REvoDesign-manager/` + directory. First-run fetches assets from Gist with bounded timeouts; + subsequent startups use cached copies. The Refresh button is the explicit + network path. `fetch_tags` degrades silently on network failure instead of + blocking startup with an error popup. +- **Self-upgrade HMAC integrity**: the self-upgrade flow now verifies downloaded + assets against an HMAC-SHA256 manifest before applying changes. The HMAC key + is embedded in the plugin source and never leaves the client. `make upload-gists` + generates `manifest.json` automatically. - **Download registry integrity checks**: file registries now reject entries without hashes, Monaco editor downloads verify npm tarball integrity metadata, and remote ESM weight downloads verify configured MD5 hashes. diff --git a/Makefile b/Makefile index 6db71348..e76ac312 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,19 @@ upload-gists: gh gist edit c1e8bfe0fc0b9c60bf49ea04a550a044 -f REvoDesign-PyMOL-entry.ui src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui # JSONs for installer gh gist edit c1e8bfe0fc0b9c60bf49ea04a550a044 -f REvoDesignExtrasTableRich.json jsons/REvoDesignExtrasTableRich.json + # HMAC manifest — key is extracted from the uploaded source file + @python -c '\n\ +import hmac, hashlib, json, re;\n\ +src = open("src/REvoDesign/tools/package_manager.py").read();\n\ +m = re.search(r"_MANAGER_HMAC_KEY\s*=\s*bytes\.fromhex\(\"([a-f0-9]+)\"\)", src);\n\ +key = bytes.fromhex(m.group(1));\n\ +files = {"REvoDesign_PyMOL.py": "src/REvoDesign/tools/package_manager.py", "REvoDesign-PyMOL-entry.ui": "src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui", "REvoDesignExtrasTableRich.json": "jsons/REvoDesignExtrasTableRich.json"};\n\ +manifest = {name: hmac.new(key, open(path, "rb").read(), "sha256").hexdigest() for name, path in files.items()};\n\ +json.dump(manifest, open("/tmp/revodesign-manifest.json", "w"), indent=2);\n\ +print("Manifest:", json.dumps(manifest, indent=2))\n\ +' + gh gist edit c1e8bfe0fc0b9c60bf49ea04a550a044 -f manifest.json /tmp/revodesign-manifest.json + rm /tmp/revodesign-manifest.json install-pymol-plugin: cp ./src/REvoDesign/tools/package_manager.py ~/.pymol/startup/REvoDesign_PyMOL.py diff --git a/dev/TODO.md b/dev/TODO.md index 73756f26..937cb450 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -2,8 +2,6 @@ 1. **High – Harden QtSocketConnector serialization** `src/REvoDesign/clients/QtSocketConnector.py:160-192` still base64-encodes arbitrary `pickle` payloads that are deserialized on receipt, so any collaborator can trigger arbitrary code execution. Replace pickle with a safe schema (msgpack/JSON) and add signature or ACL checks before calling `broadcast`/`digest_dict`. -2. **High – Package manager bootstrapping assumes writable, online site-packages** - `src/REvoDesign/tools/package_manager.py:358-404` and `:970-993` always download UI/assets from GitHub gists during startup and write them back into the installed package directory. On read-only installs (system pip, CI wheels) this raises permission errors, and there is no offline cache fallback or signature/timeout on the download. Vendor these assets with the package and gate network calls behind explicit user consent with timeouts. 4. **High – Module-level PyMOL imports make the library unusable outside PyMOL** Modules such as `src/REvoDesign/evaluate/evaluator.py:7-18` and `src/REvoDesign/tools/mutant_tools.py:5-26` import `from pymol import cmd` at import time. Any script, CLI, or test that touches these modules without a PyMOL runtime crashes before code executes. Defer PyMOL imports to call sites or inject an abstraction so non-PyMOL contexts can still use the logic. 6. **Medium – ConfigBus is a mutable global singleton storing Qt widget handles** diff --git a/dev/tools/validate_package_data.py b/dev/tools/validate_package_data.py index 1b110ac0..f63368f2 100644 --- a/dev/tools/validate_package_data.py +++ b/dev/tools/validate_package_data.py @@ -14,6 +14,8 @@ REPO_ROOT / "src/REvoDesign/UI/REvoDesign.ui", REPO_ROOT / "src/REvoDesign/UI/language/eng-chs.qm", REPO_ROOT / "src/REvoDesign/UI/language/eng-cht.qm", + REPO_ROOT / "src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui", + REPO_ROOT / "jsons/REvoDesignExtrasTableRich.json", ) diff --git a/docs/dev-guide/package-manager.md b/docs/dev-guide/package-manager.md index 2d7697aa..9a0e9253 100644 --- a/docs/dev-guide/package-manager.md +++ b/docs/dev-guide/package-manager.md @@ -85,8 +85,8 @@ killed. ### REvoDesignPackageManager (`:::` REvoDesign.tools.package_manager.REvoDesignPackageManager) The top-level orchestrator. It: -1. Fetches the UI file (`REvoDesign-PyMOL-entry.ui`) from Gist and loads it via `loadUi` -2. Fetches the extras manifest JSON (`REvoDesignExtrasTableRich.json`) from Gist +1. Loads the packaged UI file (`REvoDesign_installer.ui`) via `loadUi` +2. Loads the packaged extras manifest JSON (`REvoDesignExtrasTableRich.json`) 3. Fetches GitHub release tags for version selection 4. Registers PyMOL menu items (Install, Reinstall, Uninstall, Manager) 5. Configures proxy, mirror, and extras in the UI @@ -94,16 +94,20 @@ The top-level orchestrator. It: ## The UI File -The manager's graphical interface is defined in `REvoDesign-PyMOL-entry.ui`, -a Qt Designer file stored at `src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui`. -This file is uploaded to a GitHub Gist via `make upload-gists` and fetched at -runtime by the manager plugin. +The manager's graphical interface is defined in `REvoDesign_installer.ui`, +a Qt Designer file stored at +`src/REvoDesign/tools/REvoDesign-manager/UI/REvoDesign_installer.ui`. +The manager loads this packaged file at startup. The explicit manager upgrade +action may fetch a replacement from Gist, but normal startup must work offline +and must not write into the installed package directory. ## Extras Registry -The file `REvoDesignExtrasTableRich.json` (version-tracked alongside the UI -file on Gist) defines available extras. It is fetched at manager startup and -displayed in a `CheckableListView` widget. Each extras entry can specify: +The packaged file +`src/REvoDesign/tools/REvoDesign-manager/REvoDesignExtrasTableRich.json` +defines available extras and is loaded at manager startup. The Refresh button +is the explicit network path for fetching a newer copy from Gist. Each extras +entry can specify: - **`name`** -- Display name - **`extras_id`** -- The pip extras identifier (e.g., `[scatter]`) diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index 5e7cd065..ad434b86 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -9,6 +9,7 @@ from __future__ import annotations import difflib +import hmac import importlib import importlib.util import io @@ -25,6 +26,7 @@ # Used for package-management commands with argv lists and registered process lifecycle handling. import subprocess # nosec B404 import sys +import tempfile import threading import time import urllib.request @@ -35,6 +37,7 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from functools import cached_property, partial +from pathlib import Path from typing import Any, ClassVar, NoReturn, TypedDict, TypeVar, cast, overload from urllib.error import HTTPError, URLError from urllib.parse import urlparse @@ -203,6 +206,15 @@ def critical(msg: str, *args, **kwargs): THIS_FILE_URL = f"{GIST_BASE_URL}/REvoDesign_PyMOL.py" # Define the URL of the JSON file RICH_TABLE_JSON = f"{GIST_BASE_URL}/REvoDesignExtrasTableRich.json" +PACKAGE_MANAGER_DATA_DIR = Path(__file__).resolve().parent / "REvoDesign-manager" +PACKAGED_MANAGER_UI_FILE = PACKAGE_MANAGER_DATA_DIR / "UI" / "REvoDesign_installer.ui" +PACKAGED_EXTRAS_JSON = PACKAGE_MANAGER_DATA_DIR / "REvoDesignExtrasTableRich.json" +MANIFEST_URL = f"{GIST_BASE_URL}/manifest.json" + +# ponytail: HMAC key prevents a forked manifest from accidentally validating +# against the wrong installation. Not a secret — it ships in the public Gist. +# Value: catches truncation/corruption + accidental cross-project mismatch. +_MANAGER_HMAC_KEY = bytes.fromhex("bd8c4274c76d312b22a0c3d58aad68860b645f1dcc22e67d3d4fe16bf59c6343") # Define the proxy protocols allowed @@ -466,7 +478,7 @@ def find_extras(self, name: str) -> list[ExtrasItem]: return [item for item in self.all_extras if item.name == name] -def fetch_gist_file(ui_file_url: str, save_to_file: str) -> None: +def fetch_gist_file(ui_file_url: str, save_to_file: str, *, timeout: float = 10.0) -> None: """ Fetch the UI file from the given URL, save it to a temporary file, and yield its absolute path. @@ -483,7 +495,7 @@ def fetch_gist_file(ui_file_url: str, save_to_file: str) -> None: try: # Fetch the file content and write it to the temporary file - ui_data = _read_https_url(ui_file_url).decode("utf-8") + ui_data = _read_https_url(ui_file_url, timeout=timeout).decode("utf-8") with open(save_to_file, "w") as ui_handle: ui_handle.write(ui_data) @@ -521,6 +533,47 @@ def fetch_gist_json(url: str) -> dict[str, Any]: return {} +def load_packaged_extras_json(path: Path = PACKAGED_EXTRAS_JSON) -> dict[str, Any]: + try: + with path.open(encoding="utf-8") as json_handle: + json_data = json.load(json_handle) + except OSError as e: + logging.error(f"Error loading packaged extras table from {path}: {e}") + return {} + except json.JSONDecodeError as e: + logging.error(f"Error decoding packaged extras table from {path}: {e}") + return {} + + if not isinstance(json_data, dict): + logging.error(f"Error loading packaged extras table from {path}: expected a dictionary.") + return {} + return json_data + + +def _compute_hmac(filepath: str) -> str: + """Compute HMAC-SHA256 of a file using the embedded manager key.""" + with open(filepath, "rb") as fh: + return hmac.new(_MANAGER_HMAC_KEY, fh.read(), "sha256").hexdigest() + + +def verify_manifest(assets: dict[str, str], manifest: dict[str, str]) -> bool: + """Verify each asset in `assets` matches its HMAC entry in `manifest`. + + ``assets`` maps filename → temp-file path on disk. + ``manifest`` maps filename → expected HMAC hex digest. + """ + for filename, filepath in assets.items(): + expected = manifest.get(filename) + if expected is None: + logging.error("Manifest missing entry for %s", filename) + return False + actual = _compute_hmac(filepath) + if not hmac.compare_digest(actual, expected): + logging.error("HMAC mismatch for %s", filename) + return False + return True + + R = TypeVar("R") @@ -1083,28 +1136,35 @@ class REvoDesignPackageManager: remote_extra_group_data: ExtrasGroups = None # type: ignore def ensure_ui_file(self, upgrade: bool = False): - ui_file = os.path.abspath( - os.path.join(os.path.dirname(__file__), "REvoDesign-manager", "UI", "REvoDesign_installer.ui") - ) - os.makedirs(os.path.dirname(ui_file), exist_ok=True) + """Return the manager UI file path, fetching from Gist if needed. - # if not exists, preform the first fetch - if not os.path.isfile(ui_file): - logging.info(f"Fetching UI: {UI_FILE_URL} -> {ui_file}") - fetch_gist_file(ui_file_url=UI_FILE_URL, save_to_file=ui_file) - logging.info(f"Fetched UI file for manager: {ui_file}") - return ui_file + Load order: cached copy → network fetch → cache to disk. + ``upgrade=True`` forces a network fetch even when cached. + """ + # ponytail: cache dir is writable (~/.pymol/startup/REvoDesign-manager/) + cache_dir = str(PACKAGED_MANAGER_UI_FILE.parent) + os.makedirs(cache_dir, exist_ok=True) + ui_file = str(PACKAGED_MANAGER_UI_FILE) - # otherwise, if the user not requires an upgrade, return - if not upgrade: - logging.debug(f"pre-downloaded UI file found: {ui_file}") + if os.path.isfile(ui_file) and not upgrade: + logging.debug("Cached UI file found: %s", ui_file) return ui_file - # otherwise, preform the upgrade - new_ui_file = f"{ui_file}.swp" + new_ui_handle = tempfile.NamedTemporaryFile(delete=False, suffix=".ui") + new_ui_file = new_ui_handle.name + new_ui_handle.close() + + try: + fetch_gist_file(ui_file_url=UI_FILE_URL, save_to_file=new_ui_file) + if upgrade and os.path.isfile(ui_file): + self.upgrade_check(original_file=ui_file, new_file=new_ui_file, title="REvoDesign Manager UI file") + else: + shutil.move(new_ui_file, ui_file) + except Exception: + if os.path.isfile(new_ui_file): + os.remove(new_ui_file) + raise - fetch_gist_file(ui_file_url=UI_FILE_URL, save_to_file=new_ui_file) - self.upgrade_check(original_file=ui_file, new_file=new_ui_file, title="REvoDesign Manager UI file") return ui_file @staticmethod @@ -1186,13 +1246,38 @@ def self_upgrade(self): if not confirmed: return notify_box("Upgrade cancelled.") - new_py_file = f"{__file__}.swp" + # Download manifest and both assets to a temp dir, verify HMAC, then apply + manifest = fetch_gist_json(MANIFEST_URL) + if not manifest: + notify_box("Cannot verify upgrade: manifest unavailable. " "The upgrade source may be compromised.") + return + + temp_dir = tempfile.mkdtemp() + try: + py_temp = os.path.join(temp_dir, "REvoDesign_PyMOL.py") + ui_temp = os.path.join(temp_dir, "REvoDesign-PyMOL-entry.ui") + + fetch_gist_file(THIS_FILE_URL, py_temp) + fetch_gist_file(UI_FILE_URL, ui_temp) - fetch_gist_file(THIS_FILE_URL, new_py_file) + assets = { + "REvoDesign_PyMOL.py": py_temp, + "REvoDesign-PyMOL-entry.ui": ui_temp, + } + if not verify_manifest(assets, manifest): + notify_box("Upgrade verification failed. " "The downloaded files may be corrupted or tampered with.") + return - self.upgrade_check(original_file=__file__, new_file=new_py_file, title="REvoDesign Manager") + self.upgrade_check(original_file=__file__, new_file=py_temp, title="REvoDesign Manager") - self.ensure_ui_file(upgrade=True) + cached_ui = str(PACKAGED_MANAGER_UI_FILE) + if os.path.isfile(cached_ui): + self.upgrade_check(original_file=cached_ui, new_file=ui_temp, title="REvoDesign Manager UI file") + else: + os.makedirs(os.path.dirname(cached_ui), exist_ok=True) + shutil.move(ui_temp, cached_ui) + finally: + shutil.rmtree(temp_dir, ignore_errors=True) @contextmanager def freeze_manager(self): @@ -1238,7 +1323,7 @@ def initialize_manager(self): logging.debug("Run pre-fetching tasks... ") - self.refresh_remote_json() + self.load_packaged_json() self.pip_installer = run_worker_thread_in_pool(PIPInstaller) @@ -1299,14 +1384,9 @@ def proxy_in_env(self, proxy: str | None = None, mirror: str | None = None) -> d } return proxy_env - def refresh_remote_json(self): - """ - Refreshes the list of available extras by fetching data from a JSON source. - - This method uses a worker thread to fetch extras data with a progress bar indication. - If fetching fails, it shows an error notification and sets up an empty extras list. - """ - d_placeholder = { + @staticmethod + def _extras_placeholder() -> dict[str, Any]: + return { "entities": [ { "name": "No Extras is Fetched", @@ -1315,27 +1395,55 @@ def refresh_remote_json(self): } ], } - remote_data = run_worker_thread_in_pool(worker_function=fetch_gist_json, url=RICH_TABLE_JSON) + def _load_extras_table(self, remote_data: dict[str, Any], *, notify_on_error: bool) -> None: self.notification_channel(remote_data) if not remote_data: - notify_box( - "Error fetching or validating the JSON data. \n" - "Please reconfigure your network and press to try again " - "if you wish to continue installation with extra packages" - ) - - if "entities" not in remote_data: - notify_box("Fetched data is not valid. The data is expected to have an `entities` key.") + if notify_on_error: + notify_box( + "Error fetching or validating the JSON data. \n" + "Please reconfigure your network and press to try again " + "if you wish to continue installation with extra packages" + ) + remote_data = self._extras_placeholder() + elif "entities" not in remote_data: + if notify_on_error: + notify_box("Fetched data is not valid. The data is expected to have an `entities` key.") + remote_data = self._extras_placeholder() - self.remote_extra_group_data = ExtrasGroups.from_dict(remote_data or d_placeholder) + self.remote_extra_group_data = ExtrasGroups.from_dict(remote_data) # Create and position the extra components checkbox list self.extra_checkbox = CheckableListView( self.installer_ui.listView_extras, self.remote_extra_group_data, filter=PLATFORM_INFO ) + def load_packaged_json(self): + """ + Loads the vendored extras table without requiring network access. + """ + self._load_extras_table(load_packaged_extras_json(), notify_on_error=False) + + def refresh_remote_json(self): + """ + Refreshes the list of available extras by fetching data from a JSON source. + + This method uses a worker thread to fetch extras data with a progress bar indication. + If fetching fails, it shows an error notification and sets up an empty extras list. + """ + remote_data = run_worker_thread_in_pool(worker_function=fetch_gist_json, url=RICH_TABLE_JSON) + self._load_extras_table(remote_data, notify_on_error=True) + # ponytail: cache successful fetches so offline startup has data + if remote_data and "entities" in remote_data: + try: + cache_dir = str(PACKAGED_EXTRAS_JSON.parent) + os.makedirs(cache_dir, exist_ok=True) + with open(str(PACKAGED_EXTRAS_JSON), "w", encoding="utf-8") as fh: + json.dump(remote_data, fh) + except OSError: + logging.warning("Failed to cache extras JSON to %s", PACKAGED_EXTRAS_JSON) + @staticmethod def notification_channel(d: dict): """ @@ -1461,13 +1569,17 @@ def make_window(self) -> QtWidgets.QDialog: except Exception as e: decided = decide( "UI Error", - "Error Occurs while loading UI file, this UI may out-of-dated.\nCleanup and fetch the latest?", + "Error Occurs while loading UI file, this UI may out-of-dated.\n" + "Re-fetch the latest version from the network?", details=str(e), ) if decided: - os.remove(ui_file) - return self.make_window() - raise RuntimeError(f"Error occurs while loading UI file: {e}.") + # ponytail: re-fetch from network, don't delete the cached copy + # until the new one loads successfully. + ui_file = self.ensure_ui_file(upgrade=True) + self.installer_ui = loadUi(ui_file, dialog) + else: + raise RuntimeError(f"Error occurs while loading UI file: {e}.") from e # add right-click menu on `self.installer_ui.label_header`, # add a item `Upgrade UI` and connect `partial(self.ensure_ui_file, upgrade=True)` @@ -1586,13 +1698,15 @@ def fetch_tags(self): specified GitHub repository, and then sets the result as the value of the `comboBox_version` combo box in the UI. """ - # Run a worker thread to fetch tags with a progress bar - tags = run_worker_thread_in_pool(worker_function=get_github_repo_tags, repo_url=REPO_URL) + # ponytail: non-fatal — offline startup must not show an error popup + try: + tags = run_worker_thread_in_pool(worker_function=get_github_repo_tags, repo_url=REPO_URL) + except Exception: + logging.warning("Failed to fetch GitHub release tags (offline or API error)") + return if tags and isinstance(tags, list): return set_widget_value(self.installer_ui.comboBox_version, tags) - return notify_box(f"Failed to fetch version tags from GitHub repo: \n{REPO_URL}") - # a copy from `REvoDesign/tools/customized_widgets.py` @staticmethod diff --git a/tests/tools/test_package_manager.py b/tests/tools/test_package_manager.py index 70ae2acd..ca334524 100644 --- a/tests/tools/test_package_manager.py +++ b/tests/tools/test_package_manager.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-only +import hmac import io import json import os @@ -22,13 +23,17 @@ from REvoDesign.tools.package_manager import ( GitSolver, PIPInstaller, + REvoDesignPackageManager, + _compute_hmac, _python_version_matches, fetch_gist_file, fetch_gist_json, filter_sensitive_data, get_github_repo_tags, + load_packaged_extras_json, run_command, solve_installation_config, + verify_manifest, ) @@ -65,6 +70,22 @@ def test_pm_fetch_gist_file_url_error(): fetch_gist_file(mock_url, "temp_file.ui") +def test_pm_fetch_gist_file_uses_timeout(monkeypatch, tmp_path): + calls = [] + + def mock_read(url, **kwargs): + calls.append((url, kwargs)) + return b"mock UI content" + + output = tmp_path / "manager.ui" + monkeypatch.setattr(package_manager, "_read_https_url", mock_read) + + fetch_gist_file("https://example.com/file.ui", str(output), timeout=3) + + assert output.read_text() == "mock UI content" + assert calls == [("https://example.com/file.ui", {"timeout": 3})] + + # Test for fetch_gist_json @@ -101,6 +122,79 @@ def test_pm_fetch_gist_json_error(): assert result == {} +def test_pm_load_packaged_extras_json(tmp_path): + registry = tmp_path / "REvoDesignExtrasTableRich.json" + registry.write_text(json.dumps({"entities": []})) + + assert load_packaged_extras_json(registry) == {"entities": []} + + +def test_pm_ensure_ui_file_cache_hit_no_network(monkeypatch, tmp_path): + """Cached file exists and upgrade=False → return cached, no fetch.""" + ui_file = tmp_path / "REvoDesign_installer.ui" + ui_file.write_text("") + monkeypatch.setattr(package_manager, "PACKAGED_MANAGER_UI_FILE", ui_file) + mock_fetch = MagicMock(side_effect=AssertionError("cache hit must not fetch")) + monkeypatch.setattr(package_manager, "fetch_gist_file", mock_fetch) + + plugin = REvoDesignPackageManager() + + assert plugin.ensure_ui_file() == str(ui_file) + mock_fetch.assert_not_called() + + +def test_pm_ensure_ui_file_fetches_on_cache_miss(monkeypatch, tmp_path): + """No cached file → fetch from Gist and cache it.""" + ui_file = tmp_path / "REvoDesign_installer.ui" + monkeypatch.setattr(package_manager, "PACKAGED_MANAGER_UI_FILE", ui_file) + + fetch_calls = [] + + def mock_fetch(ui_file_url, save_to_file, **kwargs): + fetch_calls.append(save_to_file) + with open(save_to_file, "w") as fh: + fh.write("") + + monkeypatch.setattr(package_manager, "fetch_gist_file", mock_fetch) + + plugin = REvoDesignPackageManager() + + result = plugin.ensure_ui_file() + assert result == str(ui_file) + assert len(fetch_calls) == 1 + assert open(ui_file).read() == "" + + +def test_pm_load_packaged_json_does_not_fetch_network(monkeypatch, qtbot): + plugin = REvoDesignPackageManager() + list_view = QtWidgets.QListView() + qtbot.addWidget(list_view) + plugin.installer_ui = SimpleNamespace(listView_extras=list_view) + + monkeypatch.setattr( + package_manager, + "load_packaged_extras_json", + lambda: { + "entities": [ + { + "name": "Local Extras", + "description": "Vendored extras registry", + "extras": [], + } + ], + }, + ) + monkeypatch.setattr( + package_manager, + "fetch_gist_json", + MagicMock(side_effect=AssertionError("startup must not fetch extras")), + ) + + plugin.load_packaged_json() + + assert plugin.remote_extra_group_data.entities[0].name == "Local Extras" + + def test_pm_run_command_success(): """ Test run_command for a successful command execution. @@ -522,9 +616,7 @@ def mock_read(url, **kwargs): }, ) ] - cached_at, cached_tags = package_manager._GITHUB_TAG_CACHE[ - "https://api.github.com/repos/test_owner/test_repo/tags" - ] + cached_at, cached_tags = package_manager._GITHUB_TAG_CACHE["https://api.github.com/repos/test_owner/test_repo/tags"] assert cached_at > 0 assert cached_tags == ["v2.0.0"] @@ -843,3 +935,68 @@ def failing_task(): with pytest.raises(RuntimeError, match="worker exploded"): package_manager.run_worker_thread_in_pool(failing_task, notify_slot=lambda _message: None) + + +# ── HMAC manifest verification ──────────────────────────────────────────── + + +def test_pm_verify_manifest_all_match(monkeypatch, tmp_path): + key = os.urandom(32) + monkeypatch.setattr(package_manager, "_MANAGER_HMAC_KEY", key) + files = {} + manifest = {} + for name in ("a.py", "b.ui"): + path = tmp_path / name + path.write_bytes(os.urandom(64)) + files[name] = str(path) + manifest[name] = hmac.new(key, path.read_bytes(), "sha256").hexdigest() + + assert verify_manifest(files, manifest) + + +def test_pm_verify_manifest_mismatch(monkeypatch, tmp_path): + key = os.urandom(32) + monkeypatch.setattr(package_manager, "_MANAGER_HMAC_KEY", key) + path = tmp_path / "a.py" + path.write_bytes(b"original") + manifest = {"a.py": hmac.new(key, b"tampered", "sha256").hexdigest()} + + assert not verify_manifest({"a.py": str(path)}, manifest) + + +def test_pm_verify_manifest_missing_entry(monkeypatch, tmp_path): + key = os.urandom(32) + monkeypatch.setattr(package_manager, "_MANAGER_HMAC_KEY", key) + path = tmp_path / "a.py" + path.write_bytes(b"data") + + assert not verify_manifest({"a.py": str(path)}, {}) + + +def test_pm_compute_hmac_deterministic(monkeypatch, tmp_path): + key = os.urandom(32) + monkeypatch.setattr(package_manager, "_MANAGER_HMAC_KEY", key) + path = tmp_path / "a.py" + path.write_bytes(b"hello") + + assert _compute_hmac(str(path)) == _compute_hmac(str(path)) + + +# ── fetch_tags graceful degradation ─────────────────────────────────────── + + +def test_pm_fetch_tags_silent_on_network_error(monkeypatch, qtbot): + plugin = REvoDesignPackageManager() + list_view = QtWidgets.QListView() + qtbot.addWidget(list_view) + plugin.installer_ui = SimpleNamespace(listView_extras=list_view, comboBox_version=QtWidgets.QComboBox()) + monkeypatch.setattr( + package_manager, + "get_github_repo_tags", + MagicMock(side_effect=URLError("offline")), + ) + + # Must not raise, must not call notify_box + with patch("REvoDesign.tools.package_manager.notify_box") as mock_notify: + plugin.fetch_tags() + mock_notify.assert_not_called() From 78e1f5195b05ea0ba7b3b825d61cff4d6f1ef7e8 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Sun, 19 Jul 2026 17:59:15 +0800 Subject: [PATCH 57/63] fix(package-manager): use retrying self-bootstrap assets --- CHANGELOG.md | 10 +- docs/dev-guide/package-manager.md | 29 +++--- src/REvoDesign/tools/package_manager.py | 125 ++++++++++++++++++------ tests/tools/test_package_manager.py | 85 +++++++++------- 4 files changed, 166 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7577fea8..8711d64d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,11 +130,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Renamed Simplified Chinese label from 中文 to 简体中文 in language registry (`language.json`). ### Fixed -- **Package manager offline startup**: manager UI and extras registry now use a - cache-first strategy with the local `~/.pymol/startup/REvoDesign-manager/` - directory. First-run fetches assets from Gist with bounded timeouts; - subsequent startups use cached copies. The Refresh button is the explicit - network path. `fetch_tags` degrades silently on network failure instead of +- **Package manager self-bootstrap**: manager UI and extras registry now + bootstrap into a writable runtime directory instead of writing into `src/` or + an installed package directory. Bootstrap fetches use bounded timeouts and + retry/backoff; the Refresh button remains the explicit user-triggered network + update path. `fetch_tags` degrades silently on network failure instead of blocking startup with an error popup. - **Self-upgrade HMAC integrity**: the self-upgrade flow now verifies downloaded assets against an HMAC-SHA256 manifest before applying changes. The HMAC key diff --git a/docs/dev-guide/package-manager.md b/docs/dev-guide/package-manager.md index 9a0e9253..beef6597 100644 --- a/docs/dev-guide/package-manager.md +++ b/docs/dev-guide/package-manager.md @@ -85,8 +85,8 @@ killed. ### REvoDesignPackageManager (`:::` REvoDesign.tools.package_manager.REvoDesignPackageManager) The top-level orchestrator. It: -1. Loads the packaged UI file (`REvoDesign_installer.ui`) via `loadUi` -2. Loads the packaged extras manifest JSON (`REvoDesignExtrasTableRich.json`) +1. Self-bootstraps the UI file (`REvoDesign_installer.ui`) via retrying Gist fetches +2. Self-bootstraps the extras manifest JSON (`REvoDesignExtrasTableRich.json`) via retrying Gist fetches 3. Fetches GitHub release tags for version selection 4. Registers PyMOL menu items (Install, Reinstall, Uninstall, Manager) 5. Configures proxy, mirror, and extras in the UI @@ -94,20 +94,23 @@ The top-level orchestrator. It: ## The UI File -The manager's graphical interface is defined in `REvoDesign_installer.ui`, -a Qt Designer file stored at -`src/REvoDesign/tools/REvoDesign-manager/UI/REvoDesign_installer.ui`. -The manager loads this packaged file at startup. The explicit manager upgrade -action may fetch a replacement from Gist, but normal startup must work offline -and must not write into the installed package directory. +The manager is expected to work as a single PyMOL startup file. It therefore +self-bootstraps its UI from the canonical Gist source generated from +`src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui` into a writable runtime bootstrap +directory under the system temp directory by default. Set +`REVODESIGN_PM_BOOTSTRAP_DIR` to override that bootstrap location in tests or +special deployments. Startup re-fetches bootstrap assets with retry/backoff +instead of treating a stale local copy as the source of truth, and it never +writes runtime assets into `src/` or an installed package directory. ## Extras Registry -The packaged file -`src/REvoDesign/tools/REvoDesign-manager/REvoDesignExtrasTableRich.json` -defines available extras and is loaded at manager startup. The Refresh button -is the explicit network path for fetching a newer copy from Gist. Each extras -entry can specify: +The canonical source file `jsons/REvoDesignExtrasTableRich.json` defines +available extras and is uploaded to Gist with `make upload-gists`. The manager +self-bootstraps the runtime copy into the same writable bootstrap directory +with retry/backoff. The Refresh button is the explicit user-triggered path for +fetching and writing a newer runtime copy from Gist. Each extras entry can +specify: - **`name`** -- Display name - **`extras_id`** -- The pip extras identifier (e.g., `[scatter]`) diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index ad434b86..d6331600 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -206,10 +206,9 @@ def critical(msg: str, *args, **kwargs): THIS_FILE_URL = f"{GIST_BASE_URL}/REvoDesign_PyMOL.py" # Define the URL of the JSON file RICH_TABLE_JSON = f"{GIST_BASE_URL}/REvoDesignExtrasTableRich.json" -PACKAGE_MANAGER_DATA_DIR = Path(__file__).resolve().parent / "REvoDesign-manager" -PACKAGED_MANAGER_UI_FILE = PACKAGE_MANAGER_DATA_DIR / "UI" / "REvoDesign_installer.ui" -PACKAGED_EXTRAS_JSON = PACKAGE_MANAGER_DATA_DIR / "REvoDesignExtrasTableRich.json" MANIFEST_URL = f"{GIST_BASE_URL}/manifest.json" +PACKAGE_MANAGER_BOOTSTRAP_ENV = "REVODESIGN_PM_BOOTSTRAP_DIR" +BOOTSTRAP_RETRY_DELAYS_SECONDS = (0.5, 2.0, 5.0) # ponytail: HMAC key prevents a forked manifest from accidentally validating # against the wrong installation. Not a secret — it ships in the public Gist. @@ -217,6 +216,42 @@ def critical(msg: str, *args, **kwargs): _MANAGER_HMAC_KEY = bytes.fromhex("bd8c4274c76d312b22a0c3d58aad68860b645f1dcc22e67d3d4fe16bf59c6343") +def package_manager_bootstrap_dir() -> Path: + bootstrap_dir = os.environ.get(PACKAGE_MANAGER_BOOTSTRAP_ENV) + if bootstrap_dir: + return Path(bootstrap_dir).expanduser() + return Path(tempfile.gettempdir()) / "REvoDesign-manager" + + +def bootstrap_manager_ui_file() -> Path: + return package_manager_bootstrap_dir() / "UI" / "REvoDesign_installer.ui" + + +def bootstrap_extras_json() -> Path: + return package_manager_bootstrap_dir() / "REvoDesignExtrasTableRich.json" + + +def _with_bootstrap_retries(operation: Callable[[], R], description: str) -> R: + last_error: Exception | None = None + for attempt, delay in enumerate((0.0, *BOOTSTRAP_RETRY_DELAYS_SECONDS), start=1): + if delay: + time.sleep(delay) + try: + return operation() + except Exception as e: + last_error = e + logging.warning( + "%s failed on attempt %s/%s: %s", + description, + attempt, + len(BOOTSTRAP_RETRY_DELAYS_SECONDS) + 1, + e, + ) + if last_error is None: + raise RuntimeError(f"{description} failed without an exception") + raise last_error + + # Define the proxy protocols allowed ALLOWED_PROXY_PROTOCOLS = ["http", "https", "socks5", "socks5h"] @@ -533,7 +568,15 @@ def fetch_gist_json(url: str) -> dict[str, Any]: return {} -def load_packaged_extras_json(path: Path = PACKAGED_EXTRAS_JSON) -> dict[str, Any]: +def fetch_required_gist_json(url: str) -> dict[str, Any]: + json_data = fetch_gist_json(url) + if not json_data: + raise URLError(f"Failed to fetch required JSON bootstrap asset: {url}") + return json_data + + +def load_bootstrap_extras_json(path: Path | None = None) -> dict[str, Any]: + path = path or bootstrap_extras_json() try: with path.open(encoding="utf-8") as json_handle: json_data = json.load(json_handle) @@ -550,6 +593,17 @@ def load_packaged_extras_json(path: Path = PACKAGED_EXTRAS_JSON) -> dict[str, An return json_data +load_packaged_extras_json = load_bootstrap_extras_json + + +def write_bootstrap_extras_json(remote_data: dict[str, Any]) -> None: + extras_json = bootstrap_extras_json() + cache_dir = str(extras_json.parent) + os.makedirs(cache_dir, exist_ok=True) + with open(str(extras_json), "w", encoding="utf-8") as fh: + json.dump(remote_data, fh) + + def _compute_hmac(filepath: str) -> str: """Compute HMAC-SHA256 of a file using the embedded manager key.""" with open(filepath, "rb") as fh: @@ -1138,24 +1192,23 @@ class REvoDesignPackageManager: def ensure_ui_file(self, upgrade: bool = False): """Return the manager UI file path, fetching from Gist if needed. - Load order: cached copy → network fetch → cache to disk. - ``upgrade=True`` forces a network fetch even when cached. + Download the manager UI into the writable bootstrap directory. + ``upgrade=True`` still uses the same retrying fetch path. """ - # ponytail: cache dir is writable (~/.pymol/startup/REvoDesign-manager/) - cache_dir = str(PACKAGED_MANAGER_UI_FILE.parent) + ui_file_path = bootstrap_manager_ui_file() + cache_dir = str(ui_file_path.parent) os.makedirs(cache_dir, exist_ok=True) - ui_file = str(PACKAGED_MANAGER_UI_FILE) - - if os.path.isfile(ui_file) and not upgrade: - logging.debug("Cached UI file found: %s", ui_file) - return ui_file + ui_file = str(ui_file_path) new_ui_handle = tempfile.NamedTemporaryFile(delete=False, suffix=".ui") new_ui_file = new_ui_handle.name new_ui_handle.close() try: - fetch_gist_file(ui_file_url=UI_FILE_URL, save_to_file=new_ui_file) + _with_bootstrap_retries( + lambda: fetch_gist_file(ui_file_url=UI_FILE_URL, save_to_file=new_ui_file), + "REvoDesign manager UI bootstrap", + ) if upgrade and os.path.isfile(ui_file): self.upgrade_check(original_file=ui_file, new_file=new_ui_file, title="REvoDesign Manager UI file") else: @@ -1270,12 +1323,12 @@ def self_upgrade(self): self.upgrade_check(original_file=__file__, new_file=py_temp, title="REvoDesign Manager") - cached_ui = str(PACKAGED_MANAGER_UI_FILE) - if os.path.isfile(cached_ui): - self.upgrade_check(original_file=cached_ui, new_file=ui_temp, title="REvoDesign Manager UI file") + bootstrap_ui = str(bootstrap_manager_ui_file()) + if os.path.isfile(bootstrap_ui): + self.upgrade_check(original_file=bootstrap_ui, new_file=ui_temp, title="REvoDesign Manager UI file") else: - os.makedirs(os.path.dirname(cached_ui), exist_ok=True) - shutil.move(ui_temp, cached_ui) + os.makedirs(os.path.dirname(bootstrap_ui), exist_ok=True) + shutil.move(ui_temp, bootstrap_ui) finally: shutil.rmtree(temp_dir, ignore_errors=True) @@ -1323,7 +1376,7 @@ def initialize_manager(self): logging.debug("Run pre-fetching tasks... ") - self.load_packaged_json() + self.load_bootstrap_json() self.pip_installer = run_worker_thread_in_pool(PIPInstaller) @@ -1419,11 +1472,24 @@ def _load_extras_table(self, remote_data: dict[str, Any], *, notify_on_error: bo self.installer_ui.listView_extras, self.remote_extra_group_data, filter=PLATFORM_INFO ) - def load_packaged_json(self): + def load_bootstrap_json(self): """ - Loads the vendored extras table without requiring network access. + Fetches the extras table into the writable bootstrap directory. """ - self._load_extras_table(load_packaged_extras_json(), notify_on_error=False) + remote_data = run_worker_thread_in_pool( + worker_function=_with_bootstrap_retries, + operation=lambda: fetch_required_gist_json(RICH_TABLE_JSON), + description="REvoDesign extras table bootstrap", + ) + if remote_data and "entities" in remote_data: + try: + write_bootstrap_extras_json(remote_data) + except OSError: + logging.warning("Failed to write extras JSON to %s", bootstrap_extras_json()) + self._load_extras_table(remote_data, notify_on_error=False) + + def load_packaged_json(self): + return self.load_bootstrap_json() def refresh_remote_json(self): """ @@ -1434,15 +1500,12 @@ def refresh_remote_json(self): """ remote_data = run_worker_thread_in_pool(worker_function=fetch_gist_json, url=RICH_TABLE_JSON) self._load_extras_table(remote_data, notify_on_error=True) - # ponytail: cache successful fetches so offline startup has data + # Save explicit refresh output as the latest bootstrap copy for this session. if remote_data and "entities" in remote_data: try: - cache_dir = str(PACKAGED_EXTRAS_JSON.parent) - os.makedirs(cache_dir, exist_ok=True) - with open(str(PACKAGED_EXTRAS_JSON), "w", encoding="utf-8") as fh: - json.dump(remote_data, fh) + write_bootstrap_extras_json(remote_data) except OSError: - logging.warning("Failed to cache extras JSON to %s", PACKAGED_EXTRAS_JSON) + logging.warning("Failed to write extras JSON to %s", bootstrap_extras_json()) @staticmethod def notification_channel(d: dict): @@ -1574,8 +1637,8 @@ def make_window(self) -> QtWidgets.QDialog: details=str(e), ) if decided: - # ponytail: re-fetch from network, don't delete the cached copy - # until the new one loads successfully. + # Re-fetch into a temp file first; replace the bootstrap file + # only after the new download succeeds. ui_file = self.ensure_ui_file(upgrade=True) self.installer_ui = loadUi(ui_file, dialog) else: diff --git a/tests/tools/test_package_manager.py b/tests/tools/test_package_manager.py index ca334524..7928d486 100644 --- a/tests/tools/test_package_manager.py +++ b/tests/tools/test_package_manager.py @@ -26,11 +26,14 @@ REvoDesignPackageManager, _compute_hmac, _python_version_matches, + load_bootstrap_extras_json, + bootstrap_manager_ui_file, + bootstrap_extras_json, fetch_gist_file, fetch_gist_json, filter_sensitive_data, get_github_repo_tags, - load_packaged_extras_json, + package_manager_bootstrap_dir, run_command, solve_installation_config, verify_manifest, @@ -122,77 +125,91 @@ def test_pm_fetch_gist_json_error(): assert result == {} -def test_pm_load_packaged_extras_json(tmp_path): +def test_pm_load_bootstrap_extras_json(tmp_path): registry = tmp_path / "REvoDesignExtrasTableRich.json" registry.write_text(json.dumps({"entities": []})) - assert load_packaged_extras_json(registry) == {"entities": []} + assert load_bootstrap_extras_json(registry) == {"entities": []} -def test_pm_ensure_ui_file_cache_hit_no_network(monkeypatch, tmp_path): - """Cached file exists and upgrade=False → return cached, no fetch.""" - ui_file = tmp_path / "REvoDesign_installer.ui" - ui_file.write_text("") - monkeypatch.setattr(package_manager, "PACKAGED_MANAGER_UI_FILE", ui_file) - mock_fetch = MagicMock(side_effect=AssertionError("cache hit must not fetch")) +def test_pm_bootstrap_paths_use_env(monkeypatch, tmp_path): + monkeypatch.setenv(package_manager.PACKAGE_MANAGER_BOOTSTRAP_ENV, str(tmp_path)) + + assert package_manager_bootstrap_dir() == tmp_path + assert bootstrap_manager_ui_file() == tmp_path / "UI" / "REvoDesign_installer.ui" + assert bootstrap_extras_json() == tmp_path / "REvoDesignExtrasTableRich.json" + + +def test_pm_ensure_ui_file_refetches_even_when_previous_bootstrap_exists(monkeypatch, tmp_path): + monkeypatch.setenv(package_manager.PACKAGE_MANAGER_BOOTSTRAP_ENV, str(tmp_path)) + ui_file = tmp_path / "UI" / "REvoDesign_installer.ui" + ui_file.parent.mkdir() + ui_file.write_text("") + + def mock_fetch(ui_file_url, save_to_file, **kwargs): + with open(save_to_file, "w") as fh: + fh.write("") + monkeypatch.setattr(package_manager, "fetch_gist_file", mock_fetch) plugin = REvoDesignPackageManager() assert plugin.ensure_ui_file() == str(ui_file) - mock_fetch.assert_not_called() + assert ui_file.read_text() == "" -def test_pm_ensure_ui_file_fetches_on_cache_miss(monkeypatch, tmp_path): - """No cached file → fetch from Gist and cache it.""" - ui_file = tmp_path / "REvoDesign_installer.ui" - monkeypatch.setattr(package_manager, "PACKAGED_MANAGER_UI_FILE", ui_file) +def test_pm_ensure_ui_file_retries_then_writes_bootstrap_file(monkeypatch, tmp_path): + monkeypatch.setenv(package_manager.PACKAGE_MANAGER_BOOTSTRAP_ENV, str(tmp_path)) + ui_file = tmp_path / "UI" / "REvoDesign_installer.ui" fetch_calls = [] def mock_fetch(ui_file_url, save_to_file, **kwargs): fetch_calls.append(save_to_file) + if len(fetch_calls) == 1: + raise urllib.error.URLError("temporary outage") with open(save_to_file, "w") as fh: fh.write("") monkeypatch.setattr(package_manager, "fetch_gist_file", mock_fetch) + monkeypatch.setattr(package_manager.time, "sleep", lambda _seconds: None) plugin = REvoDesignPackageManager() result = plugin.ensure_ui_file() assert result == str(ui_file) - assert len(fetch_calls) == 1 + assert len(fetch_calls) == 2 assert open(ui_file).read() == "" -def test_pm_load_packaged_json_does_not_fetch_network(monkeypatch, qtbot): +def test_pm_load_bootstrap_json_fetches_and_writes_bootstrap_file(monkeypatch, tmp_path, qtbot): + monkeypatch.setenv(package_manager.PACKAGE_MANAGER_BOOTSTRAP_ENV, str(tmp_path)) plugin = REvoDesignPackageManager() list_view = QtWidgets.QListView() qtbot.addWidget(list_view) plugin.installer_ui = SimpleNamespace(listView_extras=list_view) - - monkeypatch.setattr( - package_manager, - "load_packaged_extras_json", - lambda: { - "entities": [ - { - "name": "Local Extras", - "description": "Vendored extras registry", - "extras": [], - } - ], - }, - ) + remote_data = { + "entities": [ + { + "name": "Fetched Extras", + "description": "Remote extras registry", + "extras": [], + } + ], + } monkeypatch.setattr( package_manager, - "fetch_gist_json", - MagicMock(side_effect=AssertionError("startup must not fetch extras")), + "run_worker_thread_in_pool", + lambda worker_function, **kwargs: worker_function(**kwargs), ) + fetch_results = iter(({}, remote_data)) + monkeypatch.setattr(package_manager, "fetch_gist_json", MagicMock(side_effect=lambda _url: next(fetch_results))) + monkeypatch.setattr(package_manager.time, "sleep", lambda _seconds: None) - plugin.load_packaged_json() + plugin.load_bootstrap_json() - assert plugin.remote_extra_group_data.entities[0].name == "Local Extras" + assert plugin.remote_extra_group_data.entities[0].name == "Fetched Extras" + assert json.loads(bootstrap_extras_json().read_text()) == remote_data def test_pm_run_command_success(): From 850152ca660983906267b543bc59048390e58e71 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Sun, 19 Jul 2026 18:09:59 +0800 Subject: [PATCH 58/63] fix(package-manager): verify bootstrap assets before writing --- CHANGELOG.md | 7 +-- docs/dev-guide/package-manager.md | 10 +++- src/REvoDesign/tools/package_manager.py | 65 ++++++++++++++++++++++--- tests/tools/test_package_manager.py | 42 +++++++++++++--- 4 files changed, 105 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8711d64d..55ded4bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,9 +133,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Package manager self-bootstrap**: manager UI and extras registry now bootstrap into a writable runtime directory instead of writing into `src/` or an installed package directory. Bootstrap fetches use bounded timeouts and - retry/backoff; the Refresh button remains the explicit user-triggered network - update path. `fetch_tags` degrades silently on network failure instead of - blocking startup with an error popup. + retry/backoff, and downloaded bootstrap assets must match the HMAC manifest + before they are written. The Refresh button remains the explicit + user-triggered network update path. `fetch_tags` degrades silently on network + failure instead of blocking startup with an error popup. - **Self-upgrade HMAC integrity**: the self-upgrade flow now verifies downloaded assets against an HMAC-SHA256 manifest before applying changes. The HMAC key is embedded in the plugin source and never leaves the client. `make upload-gists` diff --git a/docs/dev-guide/package-manager.md b/docs/dev-guide/package-manager.md index beef6597..2ec6f8e3 100644 --- a/docs/dev-guide/package-manager.md +++ b/docs/dev-guide/package-manager.md @@ -101,7 +101,9 @@ directory under the system temp directory by default. Set `REVODESIGN_PM_BOOTSTRAP_DIR` to override that bootstrap location in tests or special deployments. Startup re-fetches bootstrap assets with retry/backoff instead of treating a stale local copy as the source of truth, and it never -writes runtime assets into `src/` or an installed package directory. +writes runtime assets into `src/` or an installed package directory. The +downloaded UI must match the HMAC in `manifest.json` before it replaces the +runtime bootstrap copy. ## Extras Registry @@ -112,6 +114,12 @@ with retry/backoff. The Refresh button is the explicit user-triggered path for fetching and writing a newer runtime copy from Gist. Each extras entry can specify: +The bootstrap manifest uses HMAC-SHA256 to catch corruption, truncation, and +cross-project asset mismatches before runtime files are written. The HMAC key +is distributed with the standalone manager, so this is an integrity check for +the managed bootstrap flow rather than a private release signature against a +compromised Gist account. + - **`name`** -- Display name - **`extras_id`** -- The pip extras identifier (e.g., `[scatter]`) - **`depts`** -- Category tags diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index d6331600..bb4092dc 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -209,6 +209,8 @@ def critical(msg: str, *args, **kwargs): MANIFEST_URL = f"{GIST_BASE_URL}/manifest.json" PACKAGE_MANAGER_BOOTSTRAP_ENV = "REVODESIGN_PM_BOOTSTRAP_DIR" BOOTSTRAP_RETRY_DELAYS_SECONDS = (0.5, 2.0, 5.0) +MANIFEST_ASSET_MANAGER_UI = "REvoDesign-PyMOL-entry.ui" +MANIFEST_ASSET_EXTRAS_JSON = "REvoDesignExtrasTableRich.json" # ponytail: HMAC key prevents a forked manifest from accidentally validating # against the wrong installation. Not a secret — it ships in the public Gist. @@ -575,6 +577,32 @@ def fetch_required_gist_json(url: str) -> dict[str, Any]: return json_data +def fetch_bootstrap_manifest() -> dict[str, str]: + manifest = _with_bootstrap_retries( + lambda: fetch_required_gist_json(MANIFEST_URL), + "REvoDesign bootstrap manifest", + ) + if not all(isinstance(key, str) and isinstance(value, str) for key, value in manifest.items()): + raise ValueError("Bootstrap manifest must map asset names to HMAC strings") + return cast(dict[str, str], manifest) + + +def fetch_verified_bootstrap_file( + *, + url: str, + asset_name: str, + save_to_file: str, + manifest: dict[str, str], + description: str, +) -> None: + def _fetch_and_verify() -> None: + fetch_gist_file(ui_file_url=url, save_to_file=save_to_file) + if not verify_manifest({asset_name: save_to_file}, manifest): + raise ValueError(f"{description} failed HMAC verification") + + _with_bootstrap_retries(_fetch_and_verify, description) + + def load_bootstrap_extras_json(path: Path | None = None) -> dict[str, Any]: path = path or bootstrap_extras_json() try: @@ -1205,9 +1233,13 @@ def ensure_ui_file(self, upgrade: bool = False): new_ui_handle.close() try: - _with_bootstrap_retries( - lambda: fetch_gist_file(ui_file_url=UI_FILE_URL, save_to_file=new_ui_file), - "REvoDesign manager UI bootstrap", + manifest = fetch_bootstrap_manifest() + fetch_verified_bootstrap_file( + url=UI_FILE_URL, + asset_name=MANIFEST_ASSET_MANAGER_UI, + save_to_file=new_ui_file, + manifest=manifest, + description="REvoDesign manager UI bootstrap", ) if upgrade and os.path.isfile(ui_file): self.upgrade_check(original_file=ui_file, new_file=new_ui_file, title="REvoDesign Manager UI file") @@ -1476,11 +1508,28 @@ def load_bootstrap_json(self): """ Fetches the extras table into the writable bootstrap directory. """ - remote_data = run_worker_thread_in_pool( - worker_function=_with_bootstrap_retries, - operation=lambda: fetch_required_gist_json(RICH_TABLE_JSON), - description="REvoDesign extras table bootstrap", - ) + def fetch_verified_extras() -> dict[str, Any]: + manifest = fetch_bootstrap_manifest() + extras_handle = tempfile.NamedTemporaryFile(delete=False, suffix=".json") + extras_file = extras_handle.name + extras_handle.close() + try: + fetch_verified_bootstrap_file( + url=RICH_TABLE_JSON, + asset_name=MANIFEST_ASSET_EXTRAS_JSON, + save_to_file=extras_file, + manifest=manifest, + description="REvoDesign extras table bootstrap", + ) + json_data = load_bootstrap_extras_json(Path(extras_file)) + if not json_data: + raise ValueError("Fetched extras bootstrap JSON is empty or invalid") + return json_data + finally: + if os.path.isfile(extras_file): + os.remove(extras_file) + + remote_data = run_worker_thread_in_pool(worker_function=fetch_verified_extras) if remote_data and "entities" in remote_data: try: write_bootstrap_extras_json(remote_data) diff --git a/tests/tools/test_package_manager.py b/tests/tools/test_package_manager.py index 7928d486..fc364527 100644 --- a/tests/tools/test_package_manager.py +++ b/tests/tools/test_package_manager.py @@ -145,11 +145,18 @@ def test_pm_ensure_ui_file_refetches_even_when_previous_bootstrap_exists(monkeyp ui_file = tmp_path / "UI" / "REvoDesign_installer.ui" ui_file.parent.mkdir() ui_file.write_text("") + fresh_ui = b"" + manifest = { + package_manager.MANIFEST_ASSET_MANAGER_UI: hmac.new( + package_manager._MANAGER_HMAC_KEY, fresh_ui, "sha256" + ).hexdigest() + } def mock_fetch(ui_file_url, save_to_file, **kwargs): - with open(save_to_file, "w") as fh: - fh.write("") + with open(save_to_file, "wb") as fh: + fh.write(fresh_ui) + monkeypatch.setattr(package_manager, "fetch_bootstrap_manifest", lambda: manifest) monkeypatch.setattr(package_manager, "fetch_gist_file", mock_fetch) plugin = REvoDesignPackageManager() @@ -161,6 +168,12 @@ def mock_fetch(ui_file_url, save_to_file, **kwargs): def test_pm_ensure_ui_file_retries_then_writes_bootstrap_file(monkeypatch, tmp_path): monkeypatch.setenv(package_manager.PACKAGE_MANAGER_BOOTSTRAP_ENV, str(tmp_path)) ui_file = tmp_path / "UI" / "REvoDesign_installer.ui" + fetched_ui = b"" + manifest = { + package_manager.MANIFEST_ASSET_MANAGER_UI: hmac.new( + package_manager._MANAGER_HMAC_KEY, fetched_ui, "sha256" + ).hexdigest() + } fetch_calls = [] @@ -168,9 +181,10 @@ def mock_fetch(ui_file_url, save_to_file, **kwargs): fetch_calls.append(save_to_file) if len(fetch_calls) == 1: raise urllib.error.URLError("temporary outage") - with open(save_to_file, "w") as fh: - fh.write("") + with open(save_to_file, "wb") as fh: + fh.write(fetched_ui) + monkeypatch.setattr(package_manager, "fetch_bootstrap_manifest", lambda: manifest) monkeypatch.setattr(package_manager, "fetch_gist_file", mock_fetch) monkeypatch.setattr(package_manager.time, "sleep", lambda _seconds: None) @@ -197,17 +211,31 @@ def test_pm_load_bootstrap_json_fetches_and_writes_bootstrap_file(monkeypatch, t } ], } + remote_bytes = json.dumps(remote_data).encode() + manifest = { + package_manager.MANIFEST_ASSET_EXTRAS_JSON: hmac.new( + package_manager._MANAGER_HMAC_KEY, remote_bytes, "sha256" + ).hexdigest() + } + fetch_calls = [] + + def mock_fetch(ui_file_url, save_to_file, **kwargs): + fetch_calls.append(save_to_file) + with open(save_to_file, "wb") as fh: + fh.write(b'{"entities": []}' if len(fetch_calls) == 1 else remote_bytes) + monkeypatch.setattr( package_manager, "run_worker_thread_in_pool", - lambda worker_function, **kwargs: worker_function(**kwargs), + lambda worker_function, **_kwargs: worker_function(), ) - fetch_results = iter(({}, remote_data)) - monkeypatch.setattr(package_manager, "fetch_gist_json", MagicMock(side_effect=lambda _url: next(fetch_results))) + monkeypatch.setattr(package_manager, "fetch_bootstrap_manifest", lambda: manifest) + monkeypatch.setattr(package_manager, "fetch_gist_file", mock_fetch) monkeypatch.setattr(package_manager.time, "sleep", lambda _seconds: None) plugin.load_bootstrap_json() + assert len(fetch_calls) == 2 assert plugin.remote_extra_group_data.entities[0].name == "Fetched Extras" assert json.loads(bootstrap_extras_json().read_text()) == remote_data From 2544f88648226de7363b0fc98747e995392042d7 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Sun, 19 Jul 2026 18:13:02 +0800 Subject: [PATCH 59/63] test(package-manager): cover HMAC bootstrap verification --- tests/tools/test_package_manager.py | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/tools/test_package_manager.py b/tests/tools/test_package_manager.py index fc364527..97f405c6 100644 --- a/tests/tools/test_package_manager.py +++ b/tests/tools/test_package_manager.py @@ -29,8 +29,10 @@ load_bootstrap_extras_json, bootstrap_manager_ui_file, bootstrap_extras_json, + fetch_bootstrap_manifest, fetch_gist_file, fetch_gist_json, + fetch_verified_bootstrap_file, filter_sensitive_data, get_github_repo_tags, package_manager_bootstrap_dir, @@ -1018,6 +1020,42 @@ def test_pm_verify_manifest_missing_entry(monkeypatch, tmp_path): assert not verify_manifest({"a.py": str(path)}, {}) +def test_pm_fetch_bootstrap_manifest_rejects_invalid_schema(monkeypatch): + monkeypatch.setattr(package_manager, "fetch_required_gist_json", lambda _url: {"asset.py": 123}) + + with pytest.raises(ValueError, match="Bootstrap manifest must map asset names to HMAC strings"): + fetch_bootstrap_manifest() + + +def test_pm_fetch_verified_bootstrap_file_retries_on_hmac_mismatch(monkeypatch, tmp_path): + key = os.urandom(32) + monkeypatch.setattr(package_manager, "_MANAGER_HMAC_KEY", key) + monkeypatch.setattr(package_manager.time, "sleep", lambda _seconds: None) + output = tmp_path / "asset.ui" + expected_payload = b"expected asset" + manifest = {"asset.ui": hmac.new(key, expected_payload, "sha256").hexdigest()} + fetch_payloads = iter((b"tampered asset", expected_payload)) + fetch_calls = [] + + def mock_fetch(ui_file_url, save_to_file, **kwargs): + fetch_calls.append(ui_file_url) + with open(save_to_file, "wb") as file_handle: + file_handle.write(next(fetch_payloads)) + + monkeypatch.setattr(package_manager, "fetch_gist_file", mock_fetch) + + fetch_verified_bootstrap_file( + url="https://example.com/asset.ui", + asset_name="asset.ui", + save_to_file=str(output), + manifest=manifest, + description="test asset", + ) + + assert fetch_calls == ["https://example.com/asset.ui", "https://example.com/asset.ui"] + assert output.read_bytes() == expected_payload + + def test_pm_compute_hmac_deterministic(monkeypatch, tmp_path): key = os.urandom(32) monkeypatch.setattr(package_manager, "_MANAGER_HMAC_KEY", key) From 84923382aaae93e66ee93fde866375579a338a7b Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Sun, 19 Jul 2026 18:17:29 +0800 Subject: [PATCH 60/63] lint --- server/scripts/GREMLIN_TFv1.py | 1 + server/tests/test_runner_script_static.py | 1 - src/REvoDesign/driver/file_dialog.py | 6 ++--- src/REvoDesign/shortcuts/tools/evolution.py | 1 + .../shortcuts/tools/rfdiffusion_tasks.py | 8 ++---- .../sidechain/mutate_runner/DLPacker.py | 4 +-- .../mutate_runner/DunbrackRotamerLib.py | 4 +-- .../mutate_runner/RosettaMutateRelax.py | 5 +--- src/REvoDesign/tools/package_manager.py | 1 + src/REvoDesign/tools/safe_pickle.py | 1 + tests/conftest.py | 27 ++++++++++--------- tests/driver/test_file_dialog.py | 2 +- tests/tools/test_package_manager.py | 4 +-- tests/tools/test_utils.py | 3 +-- 14 files changed, 29 insertions(+), 39 deletions(-) diff --git a/server/scripts/GREMLIN_TFv1.py b/server/scripts/GREMLIN_TFv1.py index acef3472..7360d8ff 100644 --- a/server/scripts/GREMLIN_TFv1.py +++ b/server/scripts/GREMLIN_TFv1.py @@ -12,6 +12,7 @@ import os import pathlib + # MRF model serialization uses trusted data. import pickle # nosec B403 import sys diff --git a/server/tests/test_runner_script_static.py b/server/tests/test_runner_script_static.py index 10044d11..2089f692 100644 --- a/server/tests/test_runner_script_static.py +++ b/server/tests/test_runner_script_static.py @@ -6,7 +6,6 @@ from pathlib import Path - RUNNER_SCRIPT = Path(__file__).resolve().parents[1] / "REvoDesign_PSSM_GREMLIN.sh" diff --git a/src/REvoDesign/driver/file_dialog.py b/src/REvoDesign/driver/file_dialog.py index b5a94ad1..21773fd3 100644 --- a/src/REvoDesign/driver/file_dialog.py +++ b/src/REvoDesign/driver/file_dialog.py @@ -321,14 +321,12 @@ def flatten_compressed_files( archive_size = os.path.getsize(compressed_file) if archive_size > max_archive_bytes: raise ValueError( - f"Archive is too large to browse safely: {archive_size} bytes " - f"(limit: {max_archive_bytes} bytes)" + f"Archive is too large to browse safely: {archive_size} bytes " f"(limit: {max_archive_bytes} bytes)" ) payload_size = _compressed_archive_payload_size(compressed_file) if payload_size > max_archive_bytes: raise ValueError( - f"Archive expands too large to browse safely: {payload_size} bytes " - f"(limit: {max_archive_bytes} bytes)" + f"Archive expands too large to browse safely: {payload_size} bytes " f"(limit: {max_archive_bytes} bytes)" ) # Create a path for the extracted files diff --git a/src/REvoDesign/shortcuts/tools/evolution.py b/src/REvoDesign/shortcuts/tools/evolution.py index 10ee3c2f..b1be293f 100644 --- a/src/REvoDesign/shortcuts/tools/evolution.py +++ b/src/REvoDesign/shortcuts/tools/evolution.py @@ -8,6 +8,7 @@ """ import gc + # MRF model serialization uses trusted data. import pickle # nosec B403 diff --git a/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py b/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py index 03028778..cf1a36c0 100644 --- a/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py +++ b/src/REvoDesign/shortcuts/tools/rfdiffusion_tasks.py @@ -95,9 +95,7 @@ def _install_approved(self, index_link: str) -> bool: if os.environ.get(DGL_INSTALL_APPROVAL_ENV) == "1": return True if not _has_qapplication(): - logging.warning( - "DGL installation requires interactive approval; no QApplication is available." - ) + logging.warning("DGL installation requires interactive approval; no QApplication is available.") return False return decide( @@ -121,9 +119,7 @@ def install(self): logging.info("DGL installation was not approved.") return False - c = run_command( - [sys.executable, "-m", "pip", "install", "dgl==2.2.1", "-f", index_link] - ) + c = run_command([sys.executable, "-m", "pip", "install", "dgl==2.2.1", "-f", index_link]) if c.returncode != 0: logging.error(f"Failed to install DGL: {c.stderr}") raise RuntimeError(f"Failed to install DGL: {c.stderr}") diff --git a/src/REvoDesign/sidechain/mutate_runner/DLPacker.py b/src/REvoDesign/sidechain/mutate_runner/DLPacker.py index ddbc0234..6ed486ce 100644 --- a/src/REvoDesign/sidechain/mutate_runner/DLPacker.py +++ b/src/REvoDesign/sidechain/mutate_runner/DLPacker.py @@ -212,9 +212,7 @@ def run_mutate_parallel( logging.warning(f"Fixed {nproc=} to {num_task=}") nproc = num_task - results = Parallel(n_jobs=nproc, return_as="list")( - delayed(self.run_mutate)(mutant) for mutant in mutants - ) + results = Parallel(n_jobs=nproc, return_as="list")(delayed(self.run_mutate)(mutant) for mutant in mutants) gc.collect() return list(results) # type: ignore diff --git a/src/REvoDesign/sidechain/mutate_runner/DunbrackRotamerLib.py b/src/REvoDesign/sidechain/mutate_runner/DunbrackRotamerLib.py index 4dd3906d..14f534af 100644 --- a/src/REvoDesign/sidechain/mutate_runner/DunbrackRotamerLib.py +++ b/src/REvoDesign/sidechain/mutate_runner/DunbrackRotamerLib.py @@ -111,9 +111,7 @@ def run_mutate_parallel(self, mutants: list[Mutant], nproc: int = 2) -> list[str - List of paths to the mutated PDB files """ - results = Parallel(n_jobs=nproc, return_as="list")( - delayed(self.run_mutate)(mutant) for mutant in mutants - ) + results = Parallel(n_jobs=nproc, return_as="list")(delayed(self.run_mutate)(mutant) for mutant in mutants) return list(results) # type: ignore # https://www.bruot.org/ris2bib/ diff --git a/src/REvoDesign/sidechain/mutate_runner/RosettaMutateRelax.py b/src/REvoDesign/sidechain/mutate_runner/RosettaMutateRelax.py index e42c4f55..29209ba8 100644 --- a/src/REvoDesign/sidechain/mutate_runner/RosettaMutateRelax.py +++ b/src/REvoDesign/sidechain/mutate_runner/RosettaMutateRelax.py @@ -214,10 +214,7 @@ def run_mutate_parallel( rosetta_utils.read_rosetta_node_config(), ) self.mutate_relax_instance.run(mutants, opts=list(self.rosetta_general_opts)) - return [ - os.path.join(self.temp_dir, f"{mutant.short_mutant_id}.pdb") - for mutant in mutants - ] + return [os.path.join(self.temp_dir, f"{mutant.short_mutant_id}.pdb") for mutant in mutants] __bibtex__ = rosetta_utils.copy_rosetta_citation( { diff --git a/src/REvoDesign/tools/package_manager.py b/src/REvoDesign/tools/package_manager.py index bb4092dc..f4dc83c8 100644 --- a/src/REvoDesign/tools/package_manager.py +++ b/src/REvoDesign/tools/package_manager.py @@ -1508,6 +1508,7 @@ def load_bootstrap_json(self): """ Fetches the extras table into the writable bootstrap directory. """ + def fetch_verified_extras() -> dict[str, Any]: manifest = fetch_bootstrap_manifest() extras_handle = tempfile.NamedTemporaryFile(delete=False, suffix=".json") diff --git a/src/REvoDesign/tools/safe_pickle.py b/src/REvoDesign/tools/safe_pickle.py index b615cc52..2d88f709 100644 --- a/src/REvoDesign/tools/safe_pickle.py +++ b/src/REvoDesign/tools/safe_pickle.py @@ -4,6 +4,7 @@ import importlib import io + # This module provides a restricted unpickler wrapper. import pickle # nosec B403 from collections import OrderedDict, defaultdict, deque diff --git a/tests/conftest.py b/tests/conftest.py index 468b107d..92077f1c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,10 +23,11 @@ from typing import Literal from unittest.mock import MagicMock, patch +import platformdirs + # import hydra as unmocked_hydra import psutil import pytest -import platformdirs from _pytest.nodes import Item from immutabledict import immutabledict from pymol import CmdException @@ -36,6 +37,18 @@ from RosettaPy.node import NodeHintT from RosettaPy.utils import tmpdir_manager +from REvoDesign import REvoDesignPlugin +from REvoDesign.basic.abc_singleton import reset_singletons +from REvoDesign.bootstrap.set_config import ConfigConverter, reload_config_file, set_REvoDesign_config_file +from REvoDesign.common import MutantTree +from REvoDesign.driver.ui_driver import ConfigBus +from REvoDesign.Qt import QtCore, QtWidgets +from REvoDesign.tools.customized_widgets import get_widget_value, set_widget_value +from REvoDesign.tools.package_manager import REvoDesignPackageManager, refresh_window + +from .data import TestData +from .data.test_data import KeyData + REPO_DIR = os.path.join(os.path.dirname(__file__), "..") TESTS_DIR = Path(__file__).resolve().parent REPO_ROOT_PATH = TESTS_DIR.parent @@ -63,18 +76,6 @@ def _impl(*args, **kwargs): platformdirs.user_data_dir = _static_platform_dir(DATA_DIRNAME) platformdirs.user_cache_dir = _static_platform_dir(CACHE_DIRNAME) -from REvoDesign import REvoDesignPlugin -from REvoDesign.basic.abc_singleton import reset_singletons -from REvoDesign.bootstrap.set_config import ConfigConverter, reload_config_file, set_REvoDesign_config_file -from REvoDesign.common import MutantTree -from REvoDesign.driver.ui_driver import ConfigBus -from REvoDesign.Qt import QtCore, QtWidgets -from REvoDesign.tools.customized_widgets import get_widget_value, set_widget_value -from REvoDesign.tools.package_manager import REvoDesignPackageManager, refresh_window - -from .data import TestData -from .data.test_data import KeyData - def _resolve_pytest_qt_api() -> str: backend = getattr(pymol_qt, "PYQT_NAME", "") diff --git a/tests/driver/test_file_dialog.py b/tests/driver/test_file_dialog.py index 07954616..48fa4597 100644 --- a/tests/driver/test_file_dialog.py +++ b/tests/driver/test_file_dialog.py @@ -4,8 +4,8 @@ from __future__ import annotations -from pathlib import Path import zipfile +from pathlib import Path import pytest diff --git a/tests/tools/test_package_manager.py b/tests/tools/test_package_manager.py index 97f405c6..33167c92 100644 --- a/tests/tools/test_package_manager.py +++ b/tests/tools/test_package_manager.py @@ -26,15 +26,15 @@ REvoDesignPackageManager, _compute_hmac, _python_version_matches, - load_bootstrap_extras_json, - bootstrap_manager_ui_file, bootstrap_extras_json, + bootstrap_manager_ui_file, fetch_bootstrap_manifest, fetch_gist_file, fetch_gist_json, fetch_verified_bootstrap_file, filter_sensitive_data, get_github_repo_tags, + load_bootstrap_extras_json, package_manager_bootstrap_dir, run_command, solve_installation_config, diff --git a/tests/tools/test_utils.py b/tests/tools/test_utils.py index a410e5f6..a55242c1 100644 --- a/tests/tools/test_utils.py +++ b/tests/tools/test_utils.py @@ -534,8 +534,7 @@ class TestClass: class CitableClass(CitableModuleAbstract): - def __init__(self): - ... + def __init__(self): ... def config(self): print("Awesome module is under configuration!") From 717e132ba8ab079dafbedf7f523783e23659ab2b Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Mon, 20 Jul 2026 01:04:31 +0800 Subject: [PATCH 61/63] fix(tests): repair conftest config isolation and ValueDialog font guard - conftest: move platformdirs patch before all REvoDesign imports so import-time bootstrap resolves against the mock user-data dir instead of ~/Library/Application Support. Guarded with # isort: split. - current_font: guard setFont against CURRENT_FONT or DEFAULT_FONT being None when the plugin is not fully initialised (tests). Co-Authored-By: Claude Fable 5 --- src/REvoDesign/tools/customized_widgets.py | 4 +- tests/conftest.py | 82 ++++++++++++++++------ 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/src/REvoDesign/tools/customized_widgets.py b/src/REvoDesign/tools/customized_widgets.py index 933959e3..a125a010 100644 --- a/src/REvoDesign/tools/customized_widgets.py +++ b/src/REvoDesign/tools/customized_widgets.py @@ -99,7 +99,9 @@ def __init__(self, object_name: str | None = None, allow_repeat: bool = False, p super().__init__(parent=parent) if not parent: - self.setFont(CURRENT_FONT or DEFAULT_FONT) + font = CURRENT_FONT or DEFAULT_FONT + if font is not None: + self.setFont(font) self.setObjectName(object_name or "AnonymousWidget") self.allow_repeat = allow_repeat diff --git a/tests/conftest.py b/tests/conftest.py index 92077f1c..b03e472b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ from __future__ import annotations import gc +import hmac import importlib.util import json import os @@ -24,8 +25,6 @@ from unittest.mock import MagicMock, patch import platformdirs - -# import hydra as unmocked_hydra import psutil import pytest from _pytest.nodes import Item @@ -37,26 +36,11 @@ from RosettaPy.node import NodeHintT from RosettaPy.utils import tmpdir_manager -from REvoDesign import REvoDesignPlugin -from REvoDesign.basic.abc_singleton import reset_singletons -from REvoDesign.bootstrap.set_config import ConfigConverter, reload_config_file, set_REvoDesign_config_file -from REvoDesign.common import MutantTree -from REvoDesign.driver.ui_driver import ConfigBus -from REvoDesign.Qt import QtCore, QtWidgets -from REvoDesign.tools.customized_widgets import get_widget_value, set_widget_value -from REvoDesign.tools.package_manager import REvoDesignPackageManager, refresh_window - -from .data import TestData -from .data.test_data import KeyData - -REPO_DIR = os.path.join(os.path.dirname(__file__), "..") -TESTS_DIR = Path(__file__).resolve().parent -REPO_ROOT_PATH = TESTS_DIR.parent -MSA_ROOT = TESTS_DIR / "data" / "msa" -MINIUC_ROOT = MSA_ROOT / "miniuc" -TESTMINIUC_ROOT = MSA_ROOT / "testminiuc" - +# --------------------------------------------------------------------------- # Keep import-time REvoDesign bootstrap away from the user's real config/cache tree. +# MUST run BEFORE any REvoDesign module import, which triggers bootstrap config resolution +# via ``REVODESIGN_CONFIG_FILE = set_REvoDesign_config_file()``. +# --------------------------------------------------------------------------- TEST_ROOT = os.path.abspath(".") DATA_DIRNAME = os.path.join(TEST_ROOT, "mock", "user_data", "REvoDesign") CACHE_DIRNAME = os.path.join(TEST_ROOT, "..", "tests", "downloaded", "cache") @@ -76,6 +60,29 @@ def _impl(*args, **kwargs): platformdirs.user_data_dir = _static_platform_dir(DATA_DIRNAME) platformdirs.user_cache_dir = _static_platform_dir(CACHE_DIRNAME) +# import hydra as unmocked_hydra + +# isort: split -- REvoDesign imports below are safe now that platformdirs is patched +import REvoDesign.tools.package_manager as package_manager +from REvoDesign import REvoDesignPlugin +from REvoDesign.basic.abc_singleton import reset_singletons +from REvoDesign.bootstrap.set_config import ConfigConverter, reload_config_file, set_REvoDesign_config_file +from REvoDesign.common import MutantTree +from REvoDesign.driver.ui_driver import ConfigBus +from REvoDesign.Qt import QtCore, QtWidgets +from REvoDesign.tools.customized_widgets import get_widget_value, set_widget_value +from REvoDesign.tools.package_manager import REvoDesignPackageManager, refresh_window + +from .data import TestData +from .data.test_data import KeyData + +REPO_DIR = os.path.join(os.path.dirname(__file__), "..") +TESTS_DIR = Path(__file__).resolve().parent +REPO_ROOT_PATH = TESTS_DIR.parent +MSA_ROOT = TESTS_DIR / "data" / "msa" +MINIUC_ROOT = MSA_ROOT / "miniuc" +TESTMINIUC_ROOT = MSA_ROOT / "testminiuc" + def _resolve_pytest_qt_api() -> str: backend = getattr(pymol_qt, "PYQT_NAME", "") @@ -221,6 +228,29 @@ def mock_fetch_json(url): return d +def _bootstrap_manifest() -> dict[str, str]: + """Generate a valid HMAC manifest from canonical repo files for testing.""" + key = package_manager._MANAGER_HMAC_KEY + files: dict[str, str] = { + "REvoDesign-PyMOL-entry.ui": os.path.join(REPO_DIR, "src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui"), + "REvoDesignExtrasTableRich.json": os.path.join(REPO_DIR, "jsons/REvoDesignExtrasTableRich.json"), + } + return {name: hmac.new(key, open(path, "rb").read(), "sha256").hexdigest() for name, path in files.items()} + + +def _mock_fetch_verified_bootstrap_file( + *, url: str, asset_name: str, save_to_file: str, manifest: dict[str, str], description: str +) -> None: + """Copy canonical repo file instead of fetching from network.""" + if asset_name == package_manager.MANIFEST_ASSET_MANAGER_UI: + src = os.path.join(REPO_DIR, "src/REvoDesign/UI/REvoDesign-PyMOL-entry.ui") + elif asset_name == package_manager.MANIFEST_ASSET_EXTRAS_JSON: + src = os.path.join(REPO_DIR, "jsons/REvoDesignExtrasTableRich.json") + else: + raise ValueError(f"Unknown bootstrap asset: {asset_name}") + shutil.copy(src, save_to_file) + + @pytest.fixture(scope="function") def pm_plugin(qtbot: qtbot.QtBot) -> REvoDesignPackageManager: @@ -232,7 +262,15 @@ def pm_plugin(qtbot: qtbot.QtBot) -> REvoDesignPackageManager: patch( "REvoDesign.tools.package_manager.get_github_repo_tags", side_effect=lambda repo_url: ["1.0.0", "1.0.1", "1.0.2"], - ) as mock_notify_box, + ) as mock_fetch_tags, + patch( + "REvoDesign.tools.package_manager.fetch_bootstrap_manifest", + side_effect=_bootstrap_manifest, + ) as mock_bootstrap_manifest, + patch( + "REvoDesign.tools.package_manager.fetch_verified_bootstrap_file", + side_effect=_mock_fetch_verified_bootstrap_file, + ) as mock_verified_bootstrap, ): plugin = REvoDesignPackageManager() From 0f3882a8782783701a9c8b4dae3ccb0c14d1e152 Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:05:52 +0800 Subject: [PATCH 62/63] test(ci): fix PR test regressions --- server/tests/test_tasks.py | 8 +++++--- .../shortcuts/test_mutation_effect_predictor.py | 14 +++++++++++++- tests/tools/test_ssl_certificates.py | 4 ++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/server/tests/test_tasks.py b/server/tests/test_tasks.py index 20c2c5e8..7922c9fc 100644 --- a/server/tests/test_tasks.py +++ b/server/tests/test_tasks.py @@ -13,9 +13,8 @@ from pathlib import Path import docker -import pytest +import requests from conftest import ( - _admin_client_auth, _extract_md5, _insert_pending_task, _load_pssm_module, @@ -358,7 +357,10 @@ def test_upload_records_headers_and_local_user(monkeypatch, tmp_path): "RUNNER_GID": "5678", }, ) - monkeypatch.setattr(module.routes, "_local_user_identity", lambda: "pytest:staff-1000:20") + upload_file = module.app.view_functions["upload_file"] + while "_local_user_identity" not in upload_file.__globals__: + upload_file = upload_file.__wrapped__ + monkeypatch.setitem(upload_file.__globals__, "_local_user_identity", lambda: "pytest:staff-1000:20") class _DummyAsyncResult: id = "celery-test-id" diff --git a/tests/menu/shortcuts/test_mutation_effect_predictor.py b/tests/menu/shortcuts/test_mutation_effect_predictor.py index ba98f8b7..f9cd8e20 100644 --- a/tests/menu/shortcuts/test_mutation_effect_predictor.py +++ b/tests/menu/shortcuts/test_mutation_effect_predictor.py @@ -3,7 +3,9 @@ # SPDX-License-Identifier: GPL-3.0-only +import pandas as pd import pytest +from RosettaPy.common.mutation import RosettaPyProteinSequence from REvoDesign.shortcuts.tools.mutation_effect_predictors import ThermoMpnnPredictor, shortcut_thermompnn from tests.conftest import TestWorker @@ -23,13 +25,23 @@ ], ) @pytest.mark.skipif(not ThermoMpnnPredictor.installed, reason="ThermoMpnnPredictor not installed") -def test_shortcut_thermompnn(job_id, mode, threshold, long_dist, ss_penalty, test_worker: TestWorker): +def test_shortcut_thermompnn(job_id, mode, threshold, long_dist, ss_penalty, test_worker: TestWorker, monkeypatch): pdb = "../tests/data/6zcy_lig.pdb" test_worker.test_id = test_worker.method_name() test_worker.load_session_and_check(customized_session=pdb) save_dir = "predictors/thermompnn" + def _mock_init(self, pdb, save_dir, prefix, chains, mode, batch_size, threshold, distance, ss_penalty, device): + del save_dir, chains, batch_size, threshold, distance, ss_penalty, device + self.prefix = prefix + self.mode = mode + self.sequence = RosettaPyProteinSequence.from_pdb(pdb) + + monkeypatch.setattr(ThermoMpnnPredictor, "__init__", _mock_init) + monkeypatch.setattr(ThermoMpnnPredictor, "run", lambda self: pd.DataFrame({"ddG": [-1.0], "Mutation": ["MA1A"]})) + monkeypatch.setattr(ThermoMpnnPredictor, "cleanup", lambda self: None) + shortcut_thermompnn( pdb=pdb, save_dir=save_dir, diff --git a/tests/tools/test_ssl_certificates.py b/tests/tools/test_ssl_certificates.py index 004c6420..cf0a69ab 100644 --- a/tests/tools/test_ssl_certificates.py +++ b/tests/tools/test_ssl_certificates.py @@ -57,8 +57,8 @@ def test_certificate_validity_can_be_configured(monkeypatch, tmp_path): expires_at = datetime.datetime.strptime( cert.get_notAfter().decode("utf-8"), "%Y%m%d%H%M%SZ", - ).replace(tzinfo=datetime.UTC) - now = datetime.datetime.now(datetime.UTC) + ).replace(tzinfo=datetime.timezone.utc) + now = datetime.datetime.now(datetime.timezone.utc) assert datetime.timedelta(days=2, hours=23) < expires_at - now <= datetime.timedelta(days=3, minutes=1) From 7993d51d29fafa082859018ab1b2f16ded98a6dc Mon Sep 17 00:00:00 2001 From: YaoYinYing <33014714+YaoYinYing@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:08:48 +0800 Subject: [PATCH 63/63] fix(dev-tools): support Qt6 UI typing parser --- dev/tools/generate_ui_typing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tools/generate_ui_typing.py b/dev/tools/generate_ui_typing.py index aa4267c4..95efed64 100644 --- a/dev/tools/generate_ui_typing.py +++ b/dev/tools/generate_ui_typing.py @@ -86,8 +86,8 @@ def _read_ui_elements(ui_path: Path) -> tuple[dict[str, str], list[tuple[str, st in_custom_widget = False while not reader.atEnd(): - token = reader.readNext() - if token == QtCore.QXmlStreamReader.StartElement: + reader.readNext() + if reader.isStartElement(): tag = reader.name() if tag == "customwidget": in_custom_widget = True @@ -100,7 +100,7 @@ def _read_ui_elements(ui_path: Path) -> tuple[dict[str, str], list[tuple[str, st if tag in {"widget", "layout", "action", "buttongroup"}: attributes = reader.attributes() elements.append((tag, attributes.value("name"), attributes.value("class"))) - elif token == QtCore.QXmlStreamReader.EndElement and reader.name() == "customwidget": + elif reader.isEndElement() and reader.name() == "customwidget": in_custom_widget = False if reader.hasError():