diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 3b4641540..f5f5722c6 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -20,6 +20,8 @@ import socket import subprocess import sys +import time +import hashlib from pathlib import Path from urllib.parse import urlparse @@ -41,6 +43,9 @@ sys.exit(1) github_actions = bool(os.getenv("GITHUB_ACTIONS")) +_python_deps_checked = False +DEPS_STATE_FILE_NAME = "penv_python_deps_state.json" +DEPS_LOCK_FILE_NAME = "penv_python_deps.lock" PLATFORMIO_URL_VERSION_RE = re.compile( r'/v?(\d+\.\d+\.\d+(?:[.-](?:alpha|beta|rc|dev|post|pre)\d*)?(?:\.\d+)?)(?:\.(?:zip|tar\.gz|tar\.bz2))?$', @@ -69,6 +74,111 @@ } +def _deps_state_paths(platformio_dir): + cache_dir = Path(platformio_dir) / ".cache" + cache_dir.mkdir(parents=True, exist_ok=True) + return cache_dir / DEPS_STATE_FILE_NAME, cache_dir / DEPS_LOCK_FILE_NAME + + +def _get_penv_dir(penv_python): + return Path(penv_python).parent.parent + + +def _deps_fingerprint(penv_python): + normalized_deps = json.dumps(python_deps, sort_keys=True, separators=(",", ":")) + penv_dir = _get_penv_dir(penv_python) + pyvenv_cfg = penv_dir / "pyvenv.cfg" + try: + pyvenv_cfg_stat = pyvenv_cfg.stat() + penv_marker = f"{pyvenv_cfg_stat.st_size}:{pyvenv_cfg_stat.st_mtime_ns}" + except OSError: + penv_marker = "missing" + payload = "|".join([ + normalized_deps, + f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", + str(penv_dir.resolve()), + penv_marker, + ]) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _read_deps_state(state_file): + try: + if state_file.is_file(): + with state_file.open("r", encoding="utf-8") as fp: + return json.load(fp) + except Exception: + return None + return None + + +def _write_deps_state(state_file, fingerprint): + payload = { + "fingerprint": fingerprint, + "updated_at": int(time.time()), + } + tmp_file = state_file.with_suffix(".tmp") + with tmp_file.open("w", encoding="utf-8") as fp: + json.dump(payload, fp) + os.replace(tmp_file, state_file) + + +def _acquire_file_lock(lock_file, timeout_sec=60): + started = time.monotonic() + while True: + try: + fd = os.open(str(lock_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY) + os.write(fd, str(os.getpid()).encode("ascii")) + return fd + except FileExistsError: + if time.monotonic() - started >= timeout_sec: + raise TimeoutError(f"Timeout waiting lock: {lock_file}") + time.sleep(0.1) + + +def _release_file_lock(lock_file, fd): + try: + os.close(fd) + finally: + try: + os.unlink(str(lock_file)) + except FileNotFoundError: + pass + + +def _deps_state_matches(state_file, fingerprint): + state = _read_deps_state(state_file) + return bool(state and state.get("fingerprint") == fingerprint) + + +def _is_penv_dependency_cache_valid(state_file, fingerprint, penv_python): + if not _deps_state_matches(state_file, fingerprint): + return False + + penv_dir = _get_penv_dir(penv_python) + if not Path(get_executable_path(str(penv_dir), "uv")).is_file(): + print("[penv] Dependency cache invalidated: uv is missing from penv") + return False + + try: + subprocess.run( + [ + penv_python, + "-c", + "import certifi, platformio, rich, yaml", + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError): + print("[penv] Dependency cache invalidated: required Python packages are missing from penv") + return False + + return True + + def has_internet_connection(timeout=5): """ Checks practical internet reachability for dependency installation. @@ -85,8 +195,9 @@ def has_internet_connection(timeout=5): # Check if offline mode is forced via environment variable if os.getenv("PLATFORMIO_OFFLINE", "").strip().lower() in ("1", "true", "yes"): return False - + # 1) Test TCP connectivity to the proxy endpoint. + probe_started_at = time.monotonic() proxy = os.getenv("HTTPS_PROXY") or os.getenv("https_proxy") or os.getenv("HTTP_PROXY") or os.getenv("http_proxy") if proxy: try: @@ -95,6 +206,7 @@ def has_internet_connection(timeout=5): port = u.port or (443 if u.scheme == "https" else 80) if host and port: socket.create_connection((host, port), timeout=timeout).close() + print(f"[penv] Internet reachable via proxy in {time.monotonic() - probe_started_at:.2f}s") return True except Exception: # If proxy connection fails, fall back to direct connection test @@ -105,16 +217,22 @@ def has_internet_connection(timeout=5): for host in https_hosts: try: socket.create_connection((host, 443), timeout=timeout).close() + print(f"[penv] Internet reachable via {host}:443 in {time.monotonic() - probe_started_at:.2f}s") return True except Exception: continue + print(f"[penv] No internet connection detected after {time.monotonic() - probe_started_at:.2f}s") + # Direct DNS:53 connection is abolished due to many false positives on enterprise networks # (add it at the end if necessary) return False -has_network = has_internet_connection() or github_actions +connectivity_started_at = time.monotonic() +online = has_internet_connection() +print(f"[penv] Internet connectivity phase took {time.monotonic() - connectivity_started_at:.2f}s") +has_network = online or github_actions def get_executable_path(penv_dir, executable_name): @@ -245,6 +363,8 @@ def install_python_deps(python_exe, external_uv_executable, uv_cache_dir=None): Returns: bool: True if successful, False otherwise """ + install_started_at = time.monotonic() + print("[penv] Checking Python dependencies...") # Get the penv directory to locate uv within it penv_dir = os.path.dirname(os.path.dirname(python_exe)) penv_uv_executable = get_executable_path(penv_dir, "uv") @@ -258,6 +378,7 @@ def install_python_deps(python_exe, external_uv_executable, uv_cache_dir=None): # Check if uv is available in the penv uv_in_penv_available = False try: + uv_check_started_at = time.monotonic() result = subprocess.run( [penv_uv_executable, "--version"], capture_output=True, @@ -265,6 +386,7 @@ def install_python_deps(python_exe, external_uv_executable, uv_cache_dir=None): timeout=10 ) uv_in_penv_available = result.returncode == 0 + print(f"[penv] uv availability check took {time.monotonic() - uv_check_started_at:.2f}s") except (FileNotFoundError, subprocess.TimeoutExpired): uv_in_penv_available = False @@ -273,6 +395,7 @@ def install_python_deps(python_exe, external_uv_executable, uv_cache_dir=None): if external_uv_executable: # Try external uv first to install uv into the penv try: + uv_install_started_at = time.monotonic() subprocess.check_call( [external_uv_executable, "pip", "install", "uv>=0.1.0", f"--python={python_exe}", "--quiet"], stdout=subprocess.DEVNULL, @@ -281,18 +404,21 @@ def install_python_deps(python_exe, external_uv_executable, uv_cache_dir=None): env=uv_env ) uv_in_penv_available = True + print(f"[penv] Installed uv via external uv in {time.monotonic() - uv_install_started_at:.2f}s") except Exception: print("Warning: uv installation via external uv failed, falling back to pip") if not uv_in_penv_available: # Fallback to pip to install uv into penv try: + pip_uv_started_at = time.monotonic() subprocess.check_call( [python_exe, "-m", "pip", "install", "uv>=0.1.0", "--quiet", "--no-cache-dir"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, timeout=300 ) + print(f"[penv] Installed uv via pip in {time.monotonic() - pip_uv_started_at:.2f}s") except subprocess.CalledProcessError as e: print(f"Error: uv installation via pip failed with exit code {e.returncode}") return False @@ -317,6 +443,7 @@ def _get_installed_uv_packages(): result = {} try: cmd = [penv_uv_executable, "pip", "list", f"--python={python_exe}", "--format=json"] + list_started_at = time.monotonic() result_obj = subprocess.run( cmd, capture_output=True, @@ -325,6 +452,7 @@ def _get_installed_uv_packages(): timeout=300, env=uv_env ) + print(f"[penv] uv pip list took {time.monotonic() - list_started_at:.2f}s") if result_obj.returncode == 0: content = result_obj.stdout.strip() @@ -350,6 +478,7 @@ def _get_installed_uv_packages(): installed_packages = _get_installed_uv_packages() packages_to_install = list(get_packages_to_install(python_deps, installed_packages)) + print(f"[penv] python packages needing install: {len(packages_to_install)}") if packages_to_install: packages_list = [] @@ -367,6 +496,7 @@ def _get_installed_uv_packages(): ] + packages_list try: + deps_install_started_at = time.monotonic() subprocess.check_call( cmd, stdout=subprocess.DEVNULL, @@ -374,6 +504,7 @@ def _get_installed_uv_packages(): timeout=300, env=uv_env ) + print(f"[penv] uv pip install finished in {time.monotonic() - deps_install_started_at:.2f}s") except subprocess.CalledProcessError as e: print(f"Error: Failed to install Python dependencies (exit code: {e.returncode})") @@ -388,6 +519,7 @@ def _get_installed_uv_packages(): print(f"Error installing Python dependencies: {e}") return False + print(f"[penv] Python dependency check/install total: {time.monotonic() - install_started_at:.2f}s") return True @@ -523,12 +655,43 @@ def _setup_python_environment_core(env, platform, platformio_dir, should_install uv_executable = get_executable_path(penv_dir, "uv") # Install required Python dependencies for ESP32 platform - if has_network: - if not install_python_deps(penv_python, used_uv_executable, uv_cache_dir): - sys.stderr.write("Error: Failed to install Python dependencies into penv\n") - sys.exit(1) + global _python_deps_checked + if _python_deps_checked: + print("[penv] Python dependency check already completed in this process, skipping") else: - print("Warning: No internet connection detected, Python dependency check will be skipped.") + state_file, lock_file = _deps_state_paths(platformio_dir) + fingerprint = _deps_fingerprint(penv_python) + try: + lock_wait_started_at = time.monotonic() + lock_fd = _acquire_file_lock(lock_file) + print(f"[penv] Dependency lock acquired in {time.monotonic() - lock_wait_started_at:.2f}s") + except TimeoutError as e: + lock_fd = None + if _is_penv_dependency_cache_valid(state_file, fingerprint, penv_python): + print(f"[penv] {e}; dependency state already satisfied by another process") + else: + sys.stderr.write( + f"Error: {e}. Refusing to install Python dependencies without synchronization.\n" + ) + sys.exit(1) + + try: + if _is_penv_dependency_cache_valid(state_file, fingerprint, penv_python): + print("[penv] Cross-process dependency cache hit, skipping dependency check") + else: + if has_network: + if not install_python_deps(penv_python, used_uv_executable, uv_cache_dir): + sys.stderr.write("Error: Failed to install Python dependencies into penv\n") + sys.exit(1) + if lock_fd is not None: + _write_deps_state(state_file, fingerprint) + print("[penv] Dependency cache state updated") + else: + print("Warning: No internet connection detected, Python dependency check will be skipped.") + _python_deps_checked = True + finally: + if lock_fd is not None: + _release_file_lock(lock_file, lock_fd) # Install esptool package if required if should_install_esptool: