Skip to content

Add cross-process cache for Python dependency checks. #438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
TinyuZhao wants to merge 3 commits into from
Closed

Add cross-process cache for Python dependency checks. #438

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a24324d
Add a log of the time taken for each phase.
TinyuZhao Mar 20, 2026
6675b0d
Add cross-process cache for Python dependency checks.
TinyuZhao Mar 20, 2026
6e161da
Avoid false cache hit after penv recreation.
TinyuZhao Mar 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
177 changes: 170 additions & 7 deletions builder/penv_setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import socket
import subprocess
import sys
import time
import hashlib
from pathlib import Path
from urllib.parse import urlparse

Expand All @@ -41,6 +43,9 @@
sys.exit(1)

github_actions = bool(os.getenv("GITHUB_ACTIONS"))
_python_deps_checked = False
DEPS_STATE_FILE_NAME = "penv_python_deps_state.json"
DEPS_LOCK_FILE_NAME = "penv_python_deps.lock"

PLATFORMIO_URL_VERSION_RE = re.compile(
r'/v?(\d+\.\d+\.\d+(?:[.-](?:alpha|beta|rc|dev|post|pre)\d*)?(?:\.\d+)?)(?:\.(?:zip|tar\.gz|tar\.bz2))?$',
Expand Down Expand Up @@ -69,6 +74,111 @@
}


def _deps_state_paths(platformio_dir):
cache_dir = Path(platformio_dir) / ".cache"
cache_dir.mkdir(parents=True, exist_ok=True)
return cache_dir / DEPS_STATE_FILE_NAME, cache_dir / DEPS_LOCK_FILE_NAME


def _get_penv_dir(penv_python):
return Path(penv_python).parent.parent


def _deps_fingerprint(penv_python):
normalized_deps = json.dumps(python_deps, sort_keys=True, separators=(",", ":"))
penv_dir = _get_penv_dir(penv_python)
pyvenv_cfg = penv_dir / "pyvenv.cfg"
try:
pyvenv_cfg_stat = pyvenv_cfg.stat()
penv_marker = f"{pyvenv_cfg_stat.st_size}:{pyvenv_cfg_stat.st_mtime_ns}"
except OSError:
penv_marker = "missing"
payload = "|".join([
normalized_deps,
f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
str(penv_dir.resolve()),
penv_marker,
])
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
Comment thread
TinyuZhao marked this conversation as resolved.


def _read_deps_state(state_file):
try:
if state_file.is_file():
with state_file.open("r", encoding="utf-8") as fp:
return json.load(fp)
except Exception:
return None
return None


def _write_deps_state(state_file, fingerprint):
payload = {
"fingerprint": fingerprint,
"updated_at": int(time.time()),
}
tmp_file = state_file.with_suffix(".tmp")
with tmp_file.open("w", encoding="utf-8") as fp:
json.dump(payload, fp)
os.replace(tmp_file, state_file)


def _acquire_file_lock(lock_file, timeout_sec=60):
started = time.monotonic()
while True:
try:
fd = os.open(str(lock_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
os.write(fd, str(os.getpid()).encode("ascii"))
return fd
except FileExistsError:
if time.monotonic() - started >= timeout_sec:
raise TimeoutError(f"Timeout waiting lock: {lock_file}")
time.sleep(0.1)
Comment thread
TinyuZhao marked this conversation as resolved.


def _release_file_lock(lock_file, fd):
try:
os.close(fd)
finally:
try:
os.unlink(str(lock_file))
except FileNotFoundError:
pass


def _deps_state_matches(state_file, fingerprint):
state = _read_deps_state(state_file)
return bool(state and state.get("fingerprint") == fingerprint)


def _is_penv_dependency_cache_valid(state_file, fingerprint, penv_python):
if not _deps_state_matches(state_file, fingerprint):
return False

penv_dir = _get_penv_dir(penv_python)
if not Path(get_executable_path(str(penv_dir), "uv")).is_file():
print("[penv] Dependency cache invalidated: uv is missing from penv")
return False

try:
subprocess.run(
[
penv_python,
"-c",
"import certifi, platformio, rich, yaml",
],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=10,
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
print("[penv] Dependency cache invalidated: required Python packages are missing from penv")
return False

Comment on lines +163 to +178
Copy link
Copy Markdown

@coderabbitai coderabbitai Bot Mar 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Cache validation only verifies a subset of dependencies.

The import check validates certifi, platformio, rich, yaml, but python_deps includes many more critical packages (cryptography, intelhex, pyelftools, etc.). If one of the unchecked packages is missing or corrupted, the cache would still be considered valid, potentially causing build failures later.

Consider either expanding the import check to include more critical packages, or documenting why this subset is sufficient.

💡 Suggested improvement 
         subprocess.run(
             [
                 penv_python,
                 "-c",
-                "import certifi, platformio, rich, yaml",
+                "import certifi, platformio, rich, yaml, cryptography, intelhex",
             ],
             check=True,
             stdout=subprocess.DEVNULL,
             stderr=subprocess.DEVNULL,
             timeout=10,
         )
🧰 Tools
🪛 Ruff (0.15.6)

[error] 164-164: subprocess call: check for execution of untrusted input

(S603)

🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@builder/penv_setup.py` around lines 163 - 178, The import check currently
only validates a small hardcoded set of packages in the subprocess.run block and
can miss missing deps; update the cache validation to programmatically import
all packages from the python_deps list (or at least include the missing critical
packages like "cryptography", "intelhex", "pyelftools") instead of the fixed
"certifi, platformio, rich, yaml" string so the subprocess invocation using
penv_python verifies every dependency; modify the code that builds the "-c"
import string (or loop over python_deps) and keep the same exception handling
around subprocess.run to return False on failure.

return True


def has_internet_connection(timeout=5):
"""
Checks practical internet reachability for dependency installation.
Expand All @@ -85,8 +195,9 @@ def has_internet_connection(timeout=5):
# Check if offline mode is forced via environment variable
if os.getenv("PLATFORMIO_OFFLINE", "").strip().lower() in ("1", "true", "yes"):
return False

# 1) Test TCP connectivity to the proxy endpoint.
probe_started_at = time.monotonic()
proxy = os.getenv("HTTPS_PROXY") or os.getenv("https_proxy") or os.getenv("HTTP_PROXY") or os.getenv("http_proxy")
if proxy:
try:
Expand All @@ -95,6 +206,7 @@ def has_internet_connection(timeout=5):
port = u.port or (443 if u.scheme == "https" else 80)
if host and port:
socket.create_connection((host, port), timeout=timeout).close()
print(f"[penv] Internet reachable via proxy in {time.monotonic() - probe_started_at:.2f}s")
return True
except Exception:
# If proxy connection fails, fall back to direct connection test
Expand All @@ -105,16 +217,22 @@ def has_internet_connection(timeout=5):
for host in https_hosts:
try:
socket.create_connection((host, 443), timeout=timeout).close()
print(f"[penv] Internet reachable via {host}:443 in {time.monotonic() - probe_started_at:.2f}s")
return True
except Exception:
continue

print(f"[penv] No internet connection detected after {time.monotonic() - probe_started_at:.2f}s")

# Direct DNS:53 connection is abolished due to many false positives on enterprise networks
# (add it at the end if necessary)
return False


has_network = has_internet_connection() or github_actions
connectivity_started_at = time.monotonic()
online = has_internet_connection()
print(f"[penv] Internet connectivity phase took {time.monotonic() - connectivity_started_at:.2f}s")
has_network = online or github_actions


def get_executable_path(penv_dir, executable_name):
Expand Down Expand Up @@ -245,6 +363,8 @@ def install_python_deps(python_exe, external_uv_executable, uv_cache_dir=None):
Returns:
bool: True if successful, False otherwise
"""
install_started_at = time.monotonic()
print("[penv] Checking Python dependencies...")
# Get the penv directory to locate uv within it
penv_dir = os.path.dirname(os.path.dirname(python_exe))
penv_uv_executable = get_executable_path(penv_dir, "uv")
Expand All @@ -258,13 +378,15 @@ def install_python_deps(python_exe, external_uv_executable, uv_cache_dir=None):
# Check if uv is available in the penv
uv_in_penv_available = False
try:
uv_check_started_at = time.monotonic()
result = subprocess.run(
[penv_uv_executable, "--version"],
capture_output=True,
text=True,
timeout=10
)
uv_in_penv_available = result.returncode == 0
print(f"[penv] uv availability check took {time.monotonic() - uv_check_started_at:.2f}s")
except (FileNotFoundError, subprocess.TimeoutExpired):
uv_in_penv_available = False

Expand All @@ -273,6 +395,7 @@ def install_python_deps(python_exe, external_uv_executable, uv_cache_dir=None):
if external_uv_executable:
# Try external uv first to install uv into the penv
try:
uv_install_started_at = time.monotonic()
subprocess.check_call(
[external_uv_executable, "pip", "install", "uv>=0.1.0", f"--python={python_exe}", "--quiet"],
stdout=subprocess.DEVNULL,
Expand All @@ -281,18 +404,21 @@ def install_python_deps(python_exe, external_uv_executable, uv_cache_dir=None):
env=uv_env
)
uv_in_penv_available = True
print(f"[penv] Installed uv via external uv in {time.monotonic() - uv_install_started_at:.2f}s")
except Exception:
print("Warning: uv installation via external uv failed, falling back to pip")

if not uv_in_penv_available:
# Fallback to pip to install uv into penv
try:
pip_uv_started_at = time.monotonic()
subprocess.check_call(
[python_exe, "-m", "pip", "install", "uv>=0.1.0", "--quiet", "--no-cache-dir"],
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
timeout=300
)
print(f"[penv] Installed uv via pip in {time.monotonic() - pip_uv_started_at:.2f}s")
except subprocess.CalledProcessError as e:
print(f"Error: uv installation via pip failed with exit code {e.returncode}")
return False
Expand All @@ -317,6 +443,7 @@ def _get_installed_uv_packages():
result = {}
try:
cmd = [penv_uv_executable, "pip", "list", f"--python={python_exe}", "--format=json"]
list_started_at = time.monotonic()
result_obj = subprocess.run(
cmd,
capture_output=True,
Expand All @@ -325,6 +452,7 @@ def _get_installed_uv_packages():
timeout=300,
env=uv_env
)
print(f"[penv] uv pip list took {time.monotonic() - list_started_at:.2f}s")

if result_obj.returncode == 0:
content = result_obj.stdout.strip()
Expand All @@ -350,6 +478,7 @@ def _get_installed_uv_packages():

installed_packages = _get_installed_uv_packages()
packages_to_install = list(get_packages_to_install(python_deps, installed_packages))
print(f"[penv] python packages needing install: {len(packages_to_install)}")

if packages_to_install:
packages_list = []
Expand All @@ -367,13 +496,15 @@ def _get_installed_uv_packages():
] + packages_list

try:
deps_install_started_at = time.monotonic()
subprocess.check_call(
cmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
timeout=300,
env=uv_env
)
print(f"[penv] uv pip install finished in {time.monotonic() - deps_install_started_at:.2f}s")

except subprocess.CalledProcessError as e:
print(f"Error: Failed to install Python dependencies (exit code: {e.returncode})")
Expand All @@ -388,6 +519,7 @@ def _get_installed_uv_packages():
print(f"Error installing Python dependencies: {e}")
return False

print(f"[penv] Python dependency check/install total: {time.monotonic() - install_started_at:.2f}s")
return True


Expand Down Expand Up @@ -523,12 +655,43 @@ def _setup_python_environment_core(env, platform, platformio_dir, should_install
uv_executable = get_executable_path(penv_dir, "uv")

# Install required Python dependencies for ESP32 platform
if has_network:
if not install_python_deps(penv_python, used_uv_executable, uv_cache_dir):
sys.stderr.write("Error: Failed to install Python dependencies into penv\n")
sys.exit(1)
global _python_deps_checked
if _python_deps_checked:
print("[penv] Python dependency check already completed in this process, skipping")
else:
print("Warning: No internet connection detected, Python dependency check will be skipped.")
state_file, lock_file = _deps_state_paths(platformio_dir)
fingerprint = _deps_fingerprint(penv_python)
try:
lock_wait_started_at = time.monotonic()
lock_fd = _acquire_file_lock(lock_file)
print(f"[penv] Dependency lock acquired in {time.monotonic() - lock_wait_started_at:.2f}s")
except TimeoutError as e:
lock_fd = None
if _is_penv_dependency_cache_valid(state_file, fingerprint, penv_python):
print(f"[penv] {e}; dependency state already satisfied by another process")
else:
sys.stderr.write(
f"Error: {e}. Refusing to install Python dependencies without synchronization.\n"
)
sys.exit(1)

try:
if _is_penv_dependency_cache_valid(state_file, fingerprint, penv_python):
print("[penv] Cross-process dependency cache hit, skipping dependency check")
else:
if has_network:
if not install_python_deps(penv_python, used_uv_executable, uv_cache_dir):
sys.stderr.write("Error: Failed to install Python dependencies into penv\n")
sys.exit(1)
if lock_fd is not None:
_write_deps_state(state_file, fingerprint)
print("[penv] Dependency cache state updated")
else:
print("Warning: No internet connection detected, Python dependency check will be skipped.")
_python_deps_checked = True
finally:
if lock_fd is not None:
_release_file_lock(lock_file, lock_fd)

# Install esptool package if required
if should_install_esptool:
Expand Down