diff --git a/README.md b/README.md index ed4f3b8d6..7776f26bb 100644 --- a/README.md +++ b/README.md @@ -56,10 +56,6 @@ pioarduino provides native support for multiple filesystem options, allowing you - **SPIFFS** - Simple legacy filesystem. While still functional, LittleFS is recommended for new projects due to better wear-leveling and reliability. - **FatFS** - Industry-standard FAT filesystem with broad compatibility across platforms and operating systems. -### FatFS Integration - -FatFS support has been fully integrated as a Python module, providing the same seamless experience as LittleFS. Configuration is straightforward - simply specify your preferred filesystem in your project settings: See [FATFS_INTEGRATION.md](FATFS_INTEGRATION.md) for detailed documentation. - **Quick Start:** ```ini diff --git a/builder/main.py b/builder/main.py index eccee2077..441651188 100644 --- a/builder/main.py +++ b/builder/main.py @@ -73,6 +73,27 @@ # Import GDB_TOOL_PACKAGES from penv_setup (already loaded into sys.modules by platform.py) from penv_setup import GDB_TOOL_PACKAGES +# Automatically register pio-lock targets if custom_pio_lock is enabled +env_name = env.subst("$PIOENV") +if projectconfig.get(f"env:{env_name}", "custom_pio_lock", default="false").lower() in ("true", "yes", "1"): + try: + # Try to import pio_lock module from penv + import pio_lock + # Register custom targets with SCons + pio_lock.register_pio_targets(env) + except ImportError as exc: + sys.stderr.write( + f"Warning: custom_pio_lock=true but pio_lock could not be imported " + f"({exc}). Lock targets (lock-capture/lock-restore/lock-check) " + f"will not be available.\n" + ) + except AttributeError as exc: + sys.stderr.write( + f"Warning: pio_lock is installed but does not expose " + f"register_pio_targets ({exc}). Update pio-lock to a compatible " + f"version.\n" + ) + # Load board configuration and determine MCU architecture board = env.BoardConfig() board_id = env.subst("$BOARD") diff --git a/builder/penv_setup.py b/builder/penv_setup.py index e396756a9..da85a5f59 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -329,11 +329,12 @@ def get_packages_to_install(deps, installed_packages): """ Generator for Python packages that need to be installed. Compares package names case-insensitively. - + Handles both semantic version specs and direct URLs (git+, http, etc.). + Args: deps (dict): Dictionary of package names and version specifications installed_packages (dict): Dictionary of currently installed packages (keys should be lowercase) - + Yields: str: Package name that needs to be installed """ @@ -341,21 +342,27 @@ def get_packages_to_install(deps, installed_packages): name = package.lower() if name not in installed_packages: yield package + elif spec.startswith(('http://', 'https://', 'git+', 'file://')): + # URL/git/file specs cannot be parsed by semantic_version.SimpleSpec. + # Treat the pinned URL as already satisfied if present in the env; + # use `uv pip install --upgrade` separately to refresh on demand. + continue else: version_spec = semantic_version.SimpleSpec(spec) if not version_spec.match(installed_packages[name]): yield package -def install_python_deps(python_exe, external_uv_executable, uv_cache_dir=None): +def install_python_deps(python_exe, external_uv_executable, uv_cache_dir=None, additional_deps=None): """ Ensure uv package manager is available in penv and install required Python dependencies. - + Args: python_exe: Path to Python executable in the penv external_uv_executable: Path to external uv executable used to create the penv (can be None) uv_cache_dir: Optional path to uv cache directory - + additional_deps: Optional dictionary of additional package names and version specs to install + Returns: bool: True if successful, False otherwise """ @@ -471,23 +478,29 @@ def _get_installed_uv_packages(): return result installed_packages = _get_installed_uv_packages() - packages_to_install = list(get_packages_to_install(python_deps, installed_packages)) - + + # Combine core and additional dependencies + all_deps = dict(python_deps) + if additional_deps: + all_deps.update(additional_deps) + + packages_to_install = list(get_packages_to_install(all_deps, installed_packages)) + if packages_to_install: packages_list = [] for p in packages_to_install: - spec = python_deps[p] + spec = all_deps[p] if spec.startswith(('http://', 'https://', 'git+', 'file://')): packages_list.append(spec) else: packages_list.append(f"{p}{spec}") - + cmd = [ penv_uv_executable, "pip", "install", f"--python={python_exe}", "--quiet", "--upgrade" ] + packages_list - + try: subprocess.check_call( cmd, @@ -496,7 +509,7 @@ def _get_installed_uv_packages(): timeout=300, env=uv_env ) - + except subprocess.CalledProcessError as e: print(f"Error: Failed to install Python dependencies (exit code: {e.returncode})") return False @@ -509,7 +522,7 @@ def _get_installed_uv_packages(): except Exception as e: print(f"Error installing Python dependencies: {e}") return False - + return True @@ -806,6 +819,34 @@ def _install_esptool_from_tl_install(platform, python_exe, uv_executable, uv_cac # Don't exit - esptool installation is not critical for penv setup +def install_pio_lock(platform, uv_executable, penv_executable, uv_cache_dir=None): + """ + Install pio-lock into the platform's Python virtual environment. + + pio-lock provides dependency lockfile functionality for PlatformIO, + enabling reproducible builds for embedded projects. + + Args: + platform: PlatformIO platform object + uv_executable (str): Path to uv executable + penv_executable (str): Path to penv Python executable + uv_cache_dir: Optional path to uv cache directory + """ + if not has_network: + return + + # Define pio-lock as additional dependency + # todo: Replace with official pio-lock package when available + # For now, use the git source from m-mcgowan without version and install check + pio_lock_dep = { + "pio-lock": "git+https://github.com/m-mcgowan/pio-lock.git@v0.2.0" + } + + # Use the centralized installer + if not install_python_deps(penv_executable, uv_executable, uv_cache_dir, pio_lock_dep): + print("Warning: Failed to install pio-lock") + + def install_freertos_gdb(platform, uv_executable, penv_executable, uv_cache_dir=None): """ Install freertos-gdb into each GDB tool's embedded Python site (share/gdb/python/). diff --git a/ARDUINO_RELINKER_INTEGRATION.md b/docs/ARDUINO_RELINKER_INTEGRATION.md similarity index 100% rename from ARDUINO_RELINKER_INTEGRATION.md rename to docs/ARDUINO_RELINKER_INTEGRATION.md diff --git a/FATFS_INTEGRATION.md b/docs/FATFS_INTEGRATION.md similarity index 100% rename from FATFS_INTEGRATION.md rename to docs/FATFS_INTEGRATION.md diff --git a/docs/PIO_LOCK_INTEGRATION.md b/docs/PIO_LOCK_INTEGRATION.md new file mode 100644 index 000000000..b7a5b2cd8 --- /dev/null +++ b/docs/PIO_LOCK_INTEGRATION.md @@ -0,0 +1,251 @@ +# pio-lock Integration Guide + +The `pio-lock` module provides dependency lockfile functionality for pioarduino, enabling **reproducible builds** for embedded projects. When enabled, it captures the exact versions of all installed dependencies and allows restoring those exact versions later. + +## Overview + +pio-lock scans your installed dependencies and records their exact versions in a `pio.lock.json` file that you commit alongside `platformio.ini`. Later, `restore` reinstalls those exact versions, and `check` verifies nothing has drifted. + +**Captured dependencies:** +- Registry libraries — exact resolved version from `.piopm` metadata +- Git libraries — exact commit SHA from the installed repo +- Local libraries — recorded for completeness, skipped during restore +- Global packages — framework, toolchain, and tool versions from `~/.platformio/packages/` +- Platform URL — the resolved platform specification + +## Enabling pio-lock + +Add the following to your `platformio.ini`: + +```ini +[env:myenv] +platform = https://github.com/pioarduino/platform-espressif32/releases/download/stable/platform-espressif32.zip +board = esp32dev +framework = arduino + +custom_pio_lock = true +``` + +When `custom_pio_lock = true` is set: +1. pio-lock is automatically installed into the platform's Python virtual environment (`penv`) +2. Custom targets are automatically registered + +## Available Targets + +After enabling `custom_pio_lock`, the following targets become available: + +### Lockfile Management + +| Target | Description | +|--------|-------------| +| `lock-capture` | Capture current dependency versions into `pio.lock.json` | +| `lock-check` | Verify installed dependencies match `pio.lock.json` | +| `lock-restore` | Install exact versions from `pio.lock.json` | + +### Build Snapshot Management + +| Target | Description | +|--------|-------------| +| `snapshot-capture` | Save current Git HEAD to `build_snapshot.json` | +| `snapshot-check` | Verify `build_snapshot.json` matches current HEAD | +| `snapshot-clear` | Delete `build_snapshot.json` | +| `snapshot-print` | Display `build_snapshot.json` contents | + +## Usage Examples + +### Initial Setup and Capture + +```bash +# Capture the resolved dependency state +pio run -t lock-capture -e myenv + +# Commit the lockfile to version control +git add pio.lock.json +git commit -m "Add dependency lockfile" +``` + +### Restoring Dependencies (CI/Another Machine) + +```bash +# Clone the repository (with lockfile) +git clone https://github.com/user/project.git +cd project + +# Restore exact dependency versions +pio run -t lock-restore -e myenv + +# Build as usual +pio run -e myenv +``` + +### Checking for Drift + +```bash +# Verify dependencies match the lockfile (useful in CI) +pio run -t lock-check -e myenv +# Exit code: 0 = match, 1 = drift detected +``` + +### Working with Snapshots + +```bash +# Capture current Git state +pio run -t snapshot-capture -e myenv + +# Check if working directory matches snapshot +pio run -t snapshot-check -e myenv + +# View snapshot contents +pio run -t snapshot-print -e myenv + +# Remove snapshot +pio run -t snapshot-clear -e myenv +``` + +## Complete Workflow Example + +### Development Machine + +```bash +# Capture dependencies before committing +pio run -t lock-capture -e esp32dev + +# Also capture Git state for traceability +pio run -t snapshot-capture -e esp32dev + +# Commit everything +git add pio.lock.json build_snapshot.json platformio.ini +git commit -m "Feature: Add sensor driver with locked dependencies" +``` + +### CI/CD Pipeline + +```yaml +# Example GitHub Actions workflow +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install pioarduino + run: pip install pioarduino + + - name: Restore exact dependencies + run: pio run -t lock-restore -e esp32dev + + - name: Verify lockfile is up to date + run: pio run -t lock-check -e esp32dev + + - name: Build firmware + run: pio run -e esp32dev + + - name: Verify Git state (optional) + run: pio run -t snapshot-check -e esp32dev +``` + +## Best Practices + +### 1. Always Commit the Lockfile + +The `pio.lock.json` file should be treated as source code and committed to version control. This ensures all team members and CI systems use identical dependencies. + +```bash +git add pio.lock.json +git commit -m "Update dependencies" +``` + +### 2. Update Lockfile After Dependency Changes + +Whenever you modify `lib_deps` in `platformio.ini` or run `pio pkg install/update`, recapture the lockfile: + +```bash +pio run -t lock-capture -e myenv +``` + +### 3. Use Lockfile in CI + +Always restore from lockfile in CI to ensure reproducible builds: + +```bash +pio run -t lock-restore -e myenv +pio run -e myenv +``` + +### 4. Check for Drift in CI + +Add a check step to detect if `platformio.ini` and `pio.lock.json` are out of sync: + +```bash +pio run -t lock-check -e myenv || exit 1 +``` + +### 5. Combine with Build Snapshots + +For complete traceability, use both lockfiles and build snapshots: + +```bash +# Before release builds +pio run -t lock-capture -e myenv +pio run -t snapshot-capture -e myenv +``` + +This creates a complete record of: +- What code was built (Git commit) +- What dependencies were used (exact versions) + +## Lockfile Format + +The `pio.lock.json` file contains structured dependency information: + +```json +{ + "environments": { + "esp32dev": { + "libraries": [ + { + "name": "ArduinoJson", + "version": "6.21.3", + "source": "registry" + }, + { + "name": "CustomLib", + "version": "abc1234", + "source": "git", + "repository": "https://github.com/user/lib.git" + } + ], + "packages": { + "tool-esptoolpy": "5.2.0" + }, + "platform": "https://github.com/pioarduino/platform-espressif32/releases/download/55.03.38/platform-espressif32.zip" + } + } +} +``` + +## Troubleshooting + +### Lock restore fails + +If `lock-restore` fails due to missing packages: + +```bash +# WARNING: This removes lib_deps, platform, framework, toolchains and tools +# for the env, forcing a full re-download on the next run. +pio pkg uninstall -e myenv +pio run -t lock-restore -e myenv +``` + +### Outdated lockfile + +If `lock-check` reports drift: + +```bash +# Review changes, then update lockfile +pio run -t lock-capture -e myenv +``` + +## See Also + +- [pio-lock GitHub Repository](https://github.com/m-mcgowan/pio-lock) diff --git a/RELINKER_INTEGRATION.md b/docs/RELINKER_INTEGRATION.md similarity index 100% rename from RELINKER_INTEGRATION.md rename to docs/RELINKER_INTEGRATION.md diff --git a/WEAR_LEVELING.md b/docs/WEAR_LEVELING.md similarity index 100% rename from WEAR_LEVELING.md rename to docs/WEAR_LEVELING.md diff --git a/platform.py b/platform.py index 05b78d02d..178450683 100644 --- a/platform.py +++ b/platform.py @@ -67,6 +67,7 @@ get_executable_path = penv_setup_module.get_executable_path has_internet_connection = penv_setup_module.has_internet_connection install_freertos_gdb = penv_setup_module.install_freertos_gdb +install_pio_lock = penv_setup_module.install_pio_lock GDB_TOOL_PACKAGES = penv_setup_module.GDB_TOOL_PACKAGES @@ -859,6 +860,10 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any # Install freertos-gdb after MCU toolchains are installed install_freertos_gdb(self, get_executable_path(str(Path(core_dir) / "penv"), "uv"), penv_python, str(Path(core_dir) / ".cache" / "uv")) + # Install pio-lock if enabled in platformio.ini (via custom_pio_lock = true) + if variables.get("custom_pio_lock", "false").lower() in ("true", "yes", "1"): + install_pio_lock(self, get_executable_path(str(Path(core_dir) / "penv"), "uv"), penv_python, str(Path(core_dir) / ".cache" / "uv")) + if "espidf" in frameworks: self._install_common_idf_packages()