From d622506f508e93a0cb9cc87c797358aa3ea3d092 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sat, 27 Dec 2025 12:56:13 +0100 Subject: [PATCH 1/7] replace mkfatfs tool with native python module --- .github/workflows/examples.yml | 1 + FATFS_INTEGRATION.md | 175 ++++++++ README.md | 30 ++ WEAR_LEVELING.md | 216 ++++++++++ builder/main.py | 473 ++++++++++++++++---- builder/penv_setup.py | 1 + examples/arduino-fatfs/.gitignore | 5 + examples/arduino-fatfs/README.md | 224 ++++++++++ examples/arduino-fatfs/TEST_GUIDE.md | 378 ++++++++++++++++ examples/arduino-fatfs/data/README.md | 224 ++++++++++ examples/arduino-fatfs/data/partitions.csv | 6 + examples/arduino-fatfs/data/platformio.ini | 7 + examples/arduino-fatfs/data/test.txt | 1 + examples/arduino-fatfs/partitions.csv | 6 + examples/arduino-fatfs/platformio.ini | 6 + examples/arduino-fatfs/src/ffat.ino | 478 +++++++++++++++++++++ platform.json | 7 - platform.py | 7 +- 18 files changed, 2152 insertions(+), 93 deletions(-) create mode 100644 FATFS_INTEGRATION.md create mode 100644 WEAR_LEVELING.md create mode 100644 examples/arduino-fatfs/.gitignore create mode 100644 examples/arduino-fatfs/README.md create mode 100644 examples/arduino-fatfs/TEST_GUIDE.md create mode 100644 examples/arduino-fatfs/data/README.md create mode 100644 examples/arduino-fatfs/data/partitions.csv create mode 100644 examples/arduino-fatfs/data/platformio.ini create mode 100644 examples/arduino-fatfs/data/test.txt create mode 100644 examples/arduino-fatfs/partitions.csv create mode 100644 examples/arduino-fatfs/platformio.ini create mode 100644 examples/arduino-fatfs/src/ffat.ino diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index b8eb8bd2b..b306ae0a8 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -15,6 +15,7 @@ jobs: os: [ubuntu-latest, windows-latest, macos-15] example: - "examples/arduino-blink" + - "examples/arduino-fatfs" - "examples/arduino-rmt-blink" - "examples/arduino-usb-keyboard" - "examples/arduino-wifiscan" diff --git a/FATFS_INTEGRATION.md b/FATFS_INTEGRATION.md new file mode 100644 index 000000000..8be1ddb58 --- /dev/null +++ b/FATFS_INTEGRATION.md @@ -0,0 +1,175 @@ +# FatFS Integration for Platform-Espressif32 + +This platform now supports FatFS as a filesystem option, analogous to the existing LittleFS integration. + +## Features + +- **Build FatFS Image**: Creates a FatFS filesystem image from a directory +- **Upload FatFS Image**: Uploads the FatFS image to the ESP32 device +- **Download FatFS Image**: Downloads the FatFS image from the device and extracts it + +## Configuration + +### platformio.ini + +```ini +[env:myenv] +platform = espressif32 +board = esp32dev +framework = arduino + +; Select FatFS as filesystem +board_build.filesystem = fatfs + +; Optional: Directory for extracted files (default: unpacked_fs) +board_build.unpack_dir = unpacked_fs +``` + +### Partition Table + +The partition table must contain a FAT partition (Subtype 0x81): + +```csv +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +ffat, data, fat, 0x290000,0x170000, +``` + +## Usage + +### Build FatFS Image + +```bash +# Place files in data/ directory +mkdir -p data +echo "Hello FatFS" > data/test.txt + +# Build image +pio run -t buildfs +``` + +### Upload FatFS Image + +```bash +pio run -t uploadfs +``` + +### Download FatFS Image from Device + +```bash +pio run -t download_fatfs +``` + +Files will be extracted to the configured directory (default: `unpacked_fs`). + +## Technical Details + +### Python Dependencies + +The integration uses the `fatfs-ng` package, which is automatically installed. + +### Build Process + +1. A RAM disk is created with the configured FAT data size (partition size minus WL overhead) +2. The FatFS is formatted with proper parameters (2 FATs, LFN support) +3. All files from the `data/` directory are copied +4. The FAT image is wrapped with ESP32 Wear Leveling layer +5. The final image is saved as a `.bin` file + +**Important**: The build process automatically adds the ESP32 Wear Leveling layer, which is required by the Arduino FFat library. See [WEAR_LEVELING.md](WEAR_LEVELING.md) for details. + +### Wear Leveling Layer + +ESP32's FFat library requires a wear leveling layer around the FAT filesystem. The build process automatically: +- Reserves sectors for wear leveling metadata +- Wraps the FAT filesystem with WL_State structures +- Calculates proper CRC32 checksums + +### Download Process + +1. The partition table is downloaded from the device +2. The FAT partition is identified (Subtype 0x81) +3. The filesystem image is downloaded +4. The wear leveling layer is automatically detected and removed +5. The FAT data is mounted and extracted + +## Extended Features + +The `pyfatfs` package includes extended features for complete directory traversal: + +- **Complete Directory Traversal**: `walk()`, `listdir()`, `stat()` +- **Path Operations**: `exists()`, `isfile()`, `isdir()` +- **File Operations**: `remove()`, `rmdir()`, `rename()`, `makedirs()` +- **Convenience Methods**: `read_file()`, `write_file()` +- **Bulk Operations**: `copy_tree_from()`, `copy_tree_to()` + +These features enable full filesystem extraction and manipulation. + +## Comparison: LittleFS vs FatFS + +| Feature | LittleFS | FatFS | +|---------|----------|-------| +| Wear Leveling | Yes | Yes | +| Power-Loss Protection | Yes | Limited | +| Compatibility | ESP-IDF specific | Standard FAT | +| Sector Size | 4096 | 4096 | +| Filesystem Size | Flexible | Larger | + +## Example Code (Arduino) + +```cpp +#include + +void setup() { + Serial.begin(115200); + + // Mount FatFS + if (!FFat.begin(true)) { + Serial.println("FFat Mount Failed"); + return; + } + + // Read file + File file = FFat.open("/test.txt", "r"); + if (file) { + Serial.println(file.readString()); + file.close(); + } + + // Write file + file = FFat.open("/output.txt", "w"); + if (file) { + file.println("Hello from ESP32!"); + file.close(); + } +} + +void loop() { + // ... +} +``` + +## Troubleshooting + +### "No FAT filesystem partition found" + +- Check the partition table +- Ensure a partition with subtype `fat` (0x81) exists + +### Build Errors + +```bash +# Recreate Python environment +rm -rf ~/.platformio/penv +pio run +``` + +## Further Information + +- [FatFS Documentation](http://elm-chan.org/fsw/ff/00index_e.html) +- [ESP-IDF FFat Documentation](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/fatfs.html) +- [fatfs-ng Repository](https://github.com/Jason2866/pyfatfs) +- [Original fatfs-python](https://github.com/krakonos/fatfs-python) diff --git a/README.md b/README.md index 8d0312221..c7a5a6ea2 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,36 @@ Prerequisites: [pioarduino Wiki](https://deepwiki.com/pioarduino/platform-espressif32) The Wiki is AI generated and insane detailed and accurate. +# Features + +## Filesystem Support + +This platform supports two native Python filesystem options integrated in Build system menu: + +- **LittleFS** (default) - Wear-leveling filesystem optimized for flash memory +- **FatFS** - Standard FAT filesystem with broad compatibility + +### FatFS Integration + +FatFS is now fully integrated as a Python module, similar to LittleFS. See [FATFS_INTEGRATION.md](FATFS_INTEGRATION.md) for detailed documentation. + +**Quick Start:** + +```ini +[env:myenv] +board_build.filesystem = fatfs +``` + +**Available Commands:** + +```bash +pio run -t buildfs # Build FatFS image +pio run -t uploadfs # Upload FatFS image +pio run -t download_fatfs # Download and extract FatFS from device +``` + +See the [arduino-fatfs example](examples/arduino-fatfs/) for a complete working example. + ### Stable Arduino currently espressif Arduino 3.3.5 and IDF 5.5.1.251215 diff --git a/WEAR_LEVELING.md b/WEAR_LEVELING.md new file mode 100644 index 000000000..ceee00dfa --- /dev/null +++ b/WEAR_LEVELING.md @@ -0,0 +1,216 @@ +# ESP32 Wear Leveling Implementation for FAT Filesystem + +## Overview + +This implementation adds ESP32 Wear Leveling layer support to FAT filesystem images created with `fatfs-python`. The wear leveling layer is required by the ESP32 Arduino Core's `FFat` library, which uses ESP-IDF's `esp_vfs_fat_spiflash_mount_rw_wl()` function. + +## Problem + +The ESP32 Arduino Core expects FAT partitions to be wrapped with a wear leveling layer: +- **Without WL**: Raw FAT filesystem → **Mount fails** +- **With WL**: WL State + FAT filesystem + WL metadata → **Mount succeeds** + +## Solution + +The `esp32_wl.py` module implements the ESP-IDF wear leveling structure to wrap FAT images. + +## Wear Leveling Structure + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Sector 0: WL State Copy 1 │ +├─────────────────────────────────────────────────────────────┤ +│ Sector 1: WL State Copy 2 │ +├─────────────────────────────────────────────────────────────┤ +│ Sector 2-N: FAT Filesystem Data │ +│ (Boot sector, FATs, Root dir, Data area) │ +├─────────────────────────────────────────────────────────────┤ +│ Sector N+1: Temp Sector (for WL operations) │ +├─────────────────────────────────────────────────────────────┤ +│ Sector N+2: WL State Copy 3 │ +├─────────────────────────────────────────────────────────────┤ +│ Sector N+3: WL State Copy 4 │ +└─────────────────────────────────────────────────────────────┘ +``` + +## WL_State Structure (48 bytes) + +```c +typedef struct { + uint32_t pos; // Current position (0) + uint32_t max_pos; // Maximum position (number of FAT sectors) + uint32_t move_count; // Move counter (0) + uint32_t access_count; // Access counter (0) + uint32_t max_count; // Maximum count (update_rate * fat_sectors) + uint32_t block_size; // Block/sector size (4096) + uint32_t version; // WL version (2) + uint32_t device_id; // Device ID (0) + uint8_t reserved[12]; // Reserved (0xFF) + uint32_t crc32; // CRC32 of structure +} WL_State; +``` + +## Configuration + +### Default Values +- **Sector Size**: 4096 bytes (ESP32 standard) +- **Update Rate**: 16 (triggers WL after 16 * sectors writes) +- **WL State Sectors**: 2 copies at start, 2 at end (4 total) +- **Temp Sectors**: 1 sector for WL operations + +### Overhead Calculation +``` +Total Sectors = Partition Size / Sector Size +WL Overhead = (2 + 2 + 1) = 5 sectors +FAT Sectors = Total Sectors - 5 +``` + +Example for 1.5 MB partition: +- Total: 1,507,328 bytes / 4096 = 368 sectors +- WL Overhead: 5 sectors = 20,480 bytes +- FAT Data: 363 sectors = 1,486,848 bytes + +## Usage + +### Building FAT Image with WL + +The `build_fatfs_image()` function in `main.py` automatically wraps FAT images: + +```bash +pio run -t buildfs +``` + +Output: +``` +Building FS image from 'data' directory to .pio/build/esp32dev/fatfs.bin +Wrapping FAT image with ESP32 Wear Leveling layer... + Partition size: 1507328 bytes (368 sectors) + FAT data size: 1486848 bytes (363 sectors) + WL overhead: 5 sectors +Successfully created wear-leveling FAT image +``` + +### Downloading and Extracting + +The `download_fatfs` target automatically detects and extracts WL-wrapped images: + +```bash +pio run -t download_fatfs +``` + +Output: +``` +Detected Wear Leveling layer, extracting FAT data... + Extracted FAT data: 1486848 bytes +Extracting files: + FILE: /test.txt (12 bytes) +Successfully extracted 1 file(s) to unpacked_fs +``` + +## Technical Details + +### CRC32 Calculation + +The WL_State CRC32 is calculated over the first 44 bytes (excluding the CRC field itself): + +```python +state_data = struct.pack('32MB) + +## Verification + +### Check WL State + +```python +from esp32_wl import WearLevelingLayer + +wl = WearLevelingLayer() +with open('fatfs.bin', 'rb') as f: + data = f.read() + +# Verify first state sector +state = data[:48] +is_valid = wl.verify_wl_state(state) +print(f"WL State valid: {is_valid}") +``` + +### Extract FAT Data + +```python +from esp32_wl import extract_fat_from_wl_image + +with open('fatfs.bin', 'rb') as f: + wl_image = f.read() + +fat_data = extract_fat_from_wl_image(wl_image) +if fat_data: + with open('fat_only.bin', 'wb') as f: + f.write(fat_data) +``` + +## Troubleshooting + +### "FFat Mount Failed" + +**Cause**: Image doesn't have wear leveling layer + +**Solution**: Rebuild with updated `build_fatfs_image()`: +```bash +pio run -t buildfs +pio run -t uploadfs +``` + +### "Invalid sector size" + +**Cause**: Sector size mismatch between build and ESP32 config + +**Solution**: Ensure `CONFIG_WL_SECTOR_SIZE=4096` in sdkconfig + +### "Partition too small" + +**Cause**: FAT data + WL overhead exceeds partition size + +**Solution**: Increase partition size in `partitions.csv` or reduce data + +## References + +- [ESP-IDF Wear Levelling Component](https://github.com/espressif/esp-idf/tree/master/components/wear_levelling) +- [ESP-IDF FAT Filesystem](https://github.com/espressif/esp-idf/tree/master/components/fatfs) +- [Arduino-ESP32 FFat Library](https://github.com/espressif/arduino-esp32/tree/master/libraries/FFat) +- [mk_esp32fat Tool](https://github.com/TobleMiner/mk_esp32fat) (alternative C implementation) + +## License + +Same as platform-espressif32 (Apache 2.0) diff --git a/builder/main.py b/builder/main.py index 7a9a3cbf9..24c330228 100644 --- a/builder/main.py +++ b/builder/main.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import importlib.util import locale import os import re @@ -23,6 +22,7 @@ from os.path import isfile, join from pathlib import Path from littlefs import LittleFS +from fatfs import Partition, RamDisk, create_extended_partition from SCons.Script import ( ARGUMENTS, @@ -393,12 +393,6 @@ def fetch_fs_size(env): env["FS_PAGE"] = int("0x100", 16) env["FS_BLOCK"] = int("0x1000", 16) - # FFat specific offsets, see: - # https://github.com/lorol/arduino-esp32fatfs-plugin#notes-for-fatfs - if filesystem == "fatfs": - env["FS_START"] += 4096 - env["FS_SIZE"] -= 4096 - def __fetch_fs_size(target, source, env): """ @@ -515,6 +509,165 @@ def build_fs_image(target, source, env): return 1 +def build_fatfs_image(target, source, env): + """ + Build FatFS filesystem image with ESP32 Wear Leveling support. + + Uses fatfs-ng module to create ESP-IDF compatible WL-wrapped FAT images. + + Args: + target: SCons target (output .bin file) + source: SCons source (directory with files) + env: SCons environment object + + Returns: + int: 0 on success, 1 on failure + """ + + # Get parameters + source_dir = str(source[0]) + target_file = str(target[0]) + fs_size = env["FS_SIZE"] + sector_size = env.get("FS_SECTOR", 4096) + + # ESP-IDF WL layout (following wl_fatfsgen.py): + # [dummy sector] [FAT data] [state1] [state2] [config] + # Total WL sectors: 1 dummy + 2 states + 1 config = 4 sectors + from fatfs import calculate_esp32_wl_overhead + wl_info = calculate_esp32_wl_overhead(fs_size, sector_size) + + wl_reserved_sectors = wl_info['wl_overhead_sectors'] + fat_fs_size = wl_info['fat_size'] + sector_count = wl_info['fat_sectors'] + + try: + # Create RAM disk with the FAT filesystem size (without WL overhead) + storage = bytearray(fat_fs_size) + disk = RamDisk(storage, sector_size=sector_size, sector_count=sector_count) + + # Create partition, format, and mount + base_partition = Partition(disk) + + # Format the filesystem with proper workarea size for LFN support + # Workarea needs to be at least sector_size, use 2x for safety with LFN + from fatfs.wrapper import pyf_mkfs, PY_FR_OK as FR_OK + workarea_size = sector_size * 2 + + # Create filesystem with parameters matching ESP-IDF expectations: + # - n_fat=2: Two FAT copies for redundancy + # - align=0: Auto-align (let FATFS decide) + # - n_root=512: Number of root directory entries (FAT12/16 only, 0 for FAT32) + # - au_size=0: Auto allocation unit size + ret = pyf_mkfs( + base_partition.pname, + n_fat=2, + align=0, + n_root=512, # Standard root entries for FAT16 + au_size=0, # Auto + workarea_size=workarea_size + ) + if ret != FR_OK: + raise Exception(f"Failed to format filesystem: error code {ret}") + + # Mount the filesystem + base_partition.mount() + + # Wrap with extended partition for directory support + from fatfs.partition_extended import PartitionExtended + partition = PartitionExtended(base_partition) + + # Track skipped files + skipped_files = [] + + # Add all files from source directory + source_path = Path(source_dir) + if source_path.exists(): + for item in source_path.rglob("*"): + rel_path = item.relative_to(source_path) + fs_path = "/" + rel_path.as_posix() + + if item.is_dir(): + try: + partition.mkdir(fs_path) + except Exception: + # Directory might already exist or be root + pass + else: + # Ensure parent directories exist + if rel_path.parent != Path("."): + parent_path = "/" + rel_path.parent.as_posix() + try: + partition.mkdir(parent_path) + except Exception: + pass # Directory might already exist + + # Copy file + try: + with partition.open(fs_path, "w") as dest: + dest.write(item.read_bytes()) + except Exception as e: + print(f"Warning: Failed to write file {rel_path}: {e}") + skipped_files.append(str(rel_path)) + + # Unmount filesystem + base_partition.unmount() + + # Read boot sector parameters for validation + import struct + bytes_per_sector = struct.unpack(' 10: + print(f" ... and {len(skipped_files) - 10} more") + + print(f"\nSuccessfully created ESP-IDF WL-wrapped FAT image: {target_file}") + + return 0 + + except Exception as e: + print(f"Error building FatFS image: {e}") + import traceback + traceback.print_exc() + return 1 + + def check_lib_archive_exists(): """ Check if lib_archive is set in platformio.ini configuration. @@ -530,12 +683,13 @@ def check_lib_archive_exists(): def switch_off_ldf(): """ - Disables LDF (Library Dependency Finder) for uploadfs, uploadfsota, buildfs, download_littlefs, and erase targets. + Disables LDF (Library Dependency Finder) for uploadfs, uploadfsota, buildfs, + download_littlefs, download_fatfs, and erase targets. This optimization prevents unnecessary library dependency scanning and compilation when only filesystem operations are performed. """ - fs_targets = {"uploadfs", "uploadfsota", "buildfs", "erase", "download_littlefs"} + fs_targets = {"uploadfs", "uploadfsota", "buildfs", "erase", "download_littlefs", "download_fatfs"} if fs_targets & set(COMMAND_LINE_TARGETS): # Disable LDF by modifying project configuration directly env_section = "env:" + env["PIOENV"] @@ -660,14 +814,16 @@ def switch_off_ldf(): ), DataToBin=Builder( action=env.VerboseAction( - build_fs_image if filesystem == "littlefs" else " ".join( - ['"$MKFSTOOL"', "-c", "$SOURCES", "-s", "$FS_SIZE"] - + ( - ["-p", "$FS_PAGE", "-b", "$FS_BLOCK"] - if filesystem == "spiffs" - else [] + build_fs_image if filesystem == "littlefs" else ( + build_fatfs_image if filesystem == "fatfs" else " ".join( + ['"$MKFSTOOL"', "-c", "$SOURCES", "-s", "$FS_SIZE"] + + ( + ["-p", "$FS_PAGE", "-b", "$FS_BLOCK"] + if filesystem == "spiffs" + else [] + ) + + ["$TARGET"] ) - + ["$TARGET"] ), "Building FS image from '$SOURCES' directory to $TARGET", ), @@ -878,40 +1034,70 @@ def coredump_analysis(target, source, env): print(f'Make sure esp-coredump is installed: uv pip install --python "{PYTHON_EXE}" esp-coredump') -def download_littlefs(target, source, env): +def _get_unpack_dir(env): """ - Download Little filesystem from device and extract to directory. - Only supports LittleFS filesystem. - Usage: pio run -e -t download_littlefs - + Get the unpack directory from project configuration. + Args: - target: SCons target - source: SCons source env: SCons environment object + + Returns: + str: Unpack directory path """ - # Get unpack directory from board config or use default unpack_dir = "unpacked_fs" - + # Read from project config (env-specific or common section) for section in ["env:" + env["PIOENV"], "common"]: if projectconfig.has_option(section, "board_build.unpack_dir"): unpack_dir = projectconfig.get(section, "board_build.unpack_dir") break - + + return unpack_dir + + +def _prepare_unpack_dir(unpack_dir): + """ + Prepare the unpack directory by removing old content and creating fresh directory. + + Args: + unpack_dir: Directory path to prepare + + Returns: + Path: Path object for the unpack directory + """ + unpack_path = Path(get_project_dir()) / unpack_dir + if unpack_path.exists(): + shutil.rmtree(unpack_path) + unpack_path.mkdir(parents=True, exist_ok=True) + return unpack_path + + +def _download_partition_image(env, fs_type_filter=None): + """ + Common function to download partition table and filesystem image from device. + + Args: + env: SCons environment object + fs_type_filter: List of partition subtypes to look for (e.g., [0x82, 0x83] for LittleFS/SPIFFS) + or [0x81] for FAT. If None, accepts any data partition. + + Returns: + tuple: (fs_file_path, fs_start, fs_size, fs_subtype) or (None, None, None, None) on error + """ # Ensure upload port is set if not env.subst("$UPLOAD_PORT"): env.AutodetectUploadPort() - + upload_port = env.subst("$UPLOAD_PORT") download_speed = board.get("download.speed", "115200") - + # Download partition table from device - print(f"Downloading partition table from {upload_port}...") - + print(f"\nDownloading partition table from {upload_port}...\n") + build_dir = Path(env.subst("$BUILD_DIR")) build_dir.mkdir(parents=True, exist_ok=True) partition_file = build_dir / "partition_table_from_flash.bin" - + esptool_cmd = [ uploader_path.strip('"'), "--chip", mcu, @@ -924,64 +1110,57 @@ def download_littlefs(target, source, env): "0x1000", # Partition table size (4KB) str(partition_file) ] - + try: result = subprocess.run(esptool_cmd, check=False) if result.returncode != 0: print("Error: Failed to download partition table") - return 1 + return None, None, None, None except Exception as e: print(f"Error: {e}") - return 1 - - # Parse partition table to find filesystem partition - print("Parsing partition table...") - + return None, None, None, None + with open(partition_file, 'rb') as f: partition_data = f.read() - + # Parse partition entries (format: 0xAA 0x50 followed by entry data) entries = [e for e in partition_data.split(b'\xaaP') if len(e) > 0] - + fs_start = None fs_size = None fs_subtype = None - + for entry in entries: if len(entry) < 32: continue - + # Byte 0: Type (0x01 for data partitions) - # Byte 1: SubType (0x82=SPIFFS, 0x83=LittleFS) + # Byte 1: SubType (0x81=FAT, 0x82=SPIFFS, 0x83=LittleFS) # Bytes 2-5: Offset (4 bytes, little-endian) # Bytes 6-9: Size (4 bytes, little-endian) - + part_subtype = entry[1] - - # Check for SPIFFS (0x82) or LITTLEFS (0x83) - if part_subtype in [0x82, 0x83]: + + # Check if this partition matches our filter + if fs_type_filter is None or part_subtype in fs_type_filter: fs_start = int.from_bytes(entry[2:6], byteorder='little', signed=False) fs_size = int.from_bytes(entry[6:10], byteorder='little', signed=False) fs_subtype = part_subtype break - + if fs_start is None or fs_size is None: - print("Error: No filesystem partition found in partition table") - return 1 + print("Error: No matching filesystem partition found in partition table") + return None, None, None, None - block_size = 0x1000 # 4KB - - print(f"Found filesystem partition (subtype {hex(fs_subtype)}):") + print(f"\nFound filesystem partition (subtype {hex(fs_subtype)}):") print(f" Start: {hex(fs_start)}") print(f" Size: {hex(fs_size)} ({fs_size} bytes)") - print(f" Block size: {hex(block_size)}") - print("Note: This tool only supports LittleFS extraction") - + # Download filesystem image fs_file = build_dir / f"downloaded_fs_{hex(fs_start)}_{hex(fs_size)}.bin" - - print("\nDownloading filesystem from device...") - + + print("\nDownloading filesystem from device...\n") + esptool_cmd = [ uploader_path.strip('"'), "--chip", mcu, @@ -994,35 +1173,54 @@ def download_littlefs(target, source, env): hex(fs_size), str(fs_file) ] - + try: result = subprocess.run(esptool_cmd, check=False) if result.returncode != 0: print(f"Error: Download failed with code {result.returncode}") - return 1 + return None, None, None, None except Exception as e: print(f"Error: {e}") + return None, None, None, None + + print(f"\nDownloaded to {fs_file}") + + return fs_file, fs_start, fs_size, fs_subtype + + +def download_littlefs(target, source, env): + """ + Download Little filesystem from device and extract to directory. + Only supports LittleFS filesystem. + Usage: pio run -e -t download_littlefs + + Args: + target: SCons target + source: SCons source + env: SCons environment object + """ + # Get unpack directory from board config or use default + unpack_dir = _get_unpack_dir(env) + + # Download partition image (LittleFS=0x83, SPIFFS=0x82) + fs_file, fs_start, fs_size, fs_subtype = _download_partition_image(env, [0x82, 0x83]) + + if fs_file is None: return 1 - - print(f"Downloaded to {fs_file}") - - # Extract filesystem - print(f"\nExtracting LittleFS filesystem to {unpack_dir}...") - + + block_size = 0x1000 # 4KB + # Remove old unpack directory - unpack_path = Path(get_project_dir()) / unpack_dir - if unpack_path.exists(): - shutil.rmtree(unpack_path) - unpack_path.mkdir(parents=True, exist_ok=True) - + unpack_path = _prepare_unpack_dir(unpack_dir) + try: # Read the downloaded filesystem image with open(fs_file, 'rb') as f: fs_data = f.read() - + # Calculate block count block_count = fs_size // block_size - + # Create LittleFS instance and mount the image fs = LittleFS( block_size=block_size, @@ -1031,41 +1229,144 @@ def download_littlefs(target, source, env): ) fs.context.buffer = bytearray(fs_data) fs.mount() - + # Extract all files file_count = 0 print("\nExtracted files:") for root, dirs, files in fs.walk("/"): if not root.endswith("/"): root += "/" - + # Create directories for dir_name in dirs: src_path = root + dir_name dst_path = unpack_path / src_path[1:] # Remove leading '/' dst_path.mkdir(parents=True, exist_ok=True) print(f" [DIR] {src_path}") - + # Extract files for file_name in files: src_path = root + file_name dst_path = unpack_path / src_path[1:] # Remove leading '/' dst_path.parent.mkdir(parents=True, exist_ok=True) - + with fs.open(src_path, "rb") as src: file_data = src.read() dst_path.write_bytes(file_data) - + print(f" [FILE] {src_path} ({len(file_data)} bytes)") file_count += 1 - + fs.unmount() print(f"\nSuccessfully extracted {file_count} file(s) to {unpack_dir}") return 0 - + except Exception as e: print(f"Error: Failed to extract LittleFS filesystem: {e}") - print("No support for other filesystems than LittleFS!") + return 1 + + +def download_fatfs(target, source, env): + """ + Download FAT filesystem from device and extract to directory. + Handles ESP32 Wear Leveling layer automatically. + Only supports FatFS filesystem. + Usage: pio run -e -t download_fatfs + + Args: + target: SCons target + source: SCons source + env: SCons environment object + """ + + # Get unpack directory from board config or use default + unpack_dir = _get_unpack_dir(env) + + # Download partition image (FAT=0x81) + fs_file, _fs_start, _fs_size, _fs_subtype = _download_partition_image(env, [0x81]) + + if fs_file is None: + return 1 + + # Remove old unpack directory + unpack_path = _prepare_unpack_dir(unpack_dir) + + try: + # Read the downloaded filesystem image + with open(fs_file, 'rb') as f: + fs_data = bytearray(f.read()) + + # Check if the image looks like a valid FAT filesystem + if len(fs_data) < 512: + print("Error: Downloaded image is too small to be a valid FAT filesystem") + return 1 + + # Import ESP32 WL functions from fatfs + from fatfs import is_esp32_wl_image, extract_fat_from_esp32_wl + + # Try to detect and extract wear leveling layer + sector_size = 4096 # Default ESP32 sector size + + # Check if this is a wear-leveling wrapped image + if is_esp32_wl_image(fs_data, sector_size): + print("\nDetected Wear Leveling layer, extracting FAT data...") + fat_data = extract_fat_from_esp32_wl(fs_data, sector_size) + if fat_data is None: + print("Error: Failed to extract FAT data from wear-leveling image") + return 1 + fs_data = bytearray(fat_data) + print(f" Extracted FAT data: {len(fs_data)} bytes") + else: + print("\nNo Wear Leveling layer detected, treating as raw FAT image...") + + # Read sector size from FAT boot sector (offset 0x0B, 2 bytes, little-endian) + sector_size = int.from_bytes(fs_data[0x0B:0x0D], byteorder='little') + # print(f" Sector size from boot sector: {sector_size} bytes") + + # Validate sector size + if sector_size not in [512, 1024, 2048, 4096]: + print(f"Error: Invalid sector size {sector_size}. Must be 512, 1024, 2048, or 4096") + return 1 + + # Mount with fatfs-python + from fatfs import RamDisk, create_extended_partition + fs_size_adjusted = len(fs_data) + sector_count = fs_size_adjusted // sector_size + disk = RamDisk(fs_data, sector_size=sector_size, sector_count=sector_count) + partition = create_extended_partition(disk) + partition.mount() + + # Extract all files using PartitionExtended.walk() and read_file() + print("\nExtracting files:\n") + extracted_count = 0 + for root, _dirs, files in partition.walk("/"): + # Create directories + rel_root = root[1:] if root.startswith("/") else root + abs_root = unpack_path / rel_root + abs_root.mkdir(parents=True, exist_ok=True) + for filename in files: + src_file = root.rstrip("/") + "/" + filename if root != "/" else "/" + filename + dst_file = abs_root / filename + try: + data = partition.read_file(src_file) + dst_file.write_bytes(data) + print(f" FILE: {src_file} ({len(data)} bytes)") + extracted_count += 1 + except Exception as e: + print(f" Warning: Failed to extract {src_file}: {e}") + partition.unmount() + # Summary + if extracted_count == 0: + print("\nNo files were extracted.") + print("The filesystem may be empty, freshly formatted, or contain only deleted entries.") + else: + print(f"\nSuccessfully extracted {extracted_count} file(s) to {unpack_dir}") + return 0 + + except Exception as e: + print(f"Error: Failed to extract FatFS filesystem: {e}") + import traceback + traceback.print_exc() return 1 # @@ -1313,6 +1614,14 @@ def download_littlefs(target, source, env): "Download and extract LittleFS filesystem from device", ) +# Target: Download FatFS (no build required) +env.AddPlatformTarget( + "download_fatfs", + None, + download_fatfs, + "Download and extract FatFS filesystem from device", +) + # Target: Erase Flash and Upload env.AddPlatformTarget( "erase_upload", diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 2e7fcce9a..5c3c23bff 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -46,6 +46,7 @@ python_deps = { "platformio": "https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip", "littlefs-python": ">=0.16.0", + "fatfs-ng": ">=0.1.14", "pyyaml": ">=6.0.2", "rich-click": ">=1.8.6", "zopfli": ">=0.2.2", diff --git a/examples/arduino-fatfs/.gitignore b/examples/arduino-fatfs/.gitignore new file mode 100644 index 000000000..89cc49cbd --- /dev/null +++ b/examples/arduino-fatfs/.gitignore @@ -0,0 +1,5 @@ +.pio +.vscode/.browse.c_cpp.db* +.vscode/c_cpp_properties.json +.vscode/launch.json +.vscode/ipch diff --git a/examples/arduino-fatfs/README.md b/examples/arduino-fatfs/README.md new file mode 100644 index 000000000..cd049e74d --- /dev/null +++ b/examples/arduino-fatfs/README.md @@ -0,0 +1,224 @@ +# ESP32 FAT Filesystem Test Project + +This project tests the FAT filesystem implementation with ESP32 Wear Leveling support. + +## Overview + +This project demonstrates: +- Building FAT filesystem images with wear leveling layer +- Uploading FAT images to ESP32 +- Mounting and reading FAT filesystem on ESP32 +- Downloading and extracting FAT images from ESP32 + +## Requirements + +- PlatformIO +- ESP32 development board +- USB cable + +## Project Structure + +``` +arduino-fatfs/ +├── data/ # Files to be included in FAT image +│ ├── test.txt +│ ├── README.md +│ └── ... +├── src/ +│ └── ffat.ino # Main Arduino sketch +├── partitions.csv # Partition table with FAT partition +├── platformio.ini # PlatformIO configuration +└── unpacked_fs/ # Downloaded files (created by download_fatfs) +``` + +## Usage + +### 1. Build Firmware + +```bash +pio run +``` + +### 2. Build FAT Filesystem Image + +Place your files in the `data/` directory, then: + +```bash +pio run -t buildfs +``` + +This creates a FAT filesystem image with ESP32 wear leveling layer at: +`.pio/build/esp32dev/fatfs.bin` + +### 3. Upload Firmware and Filesystem + +```bash +# Upload firmware +pio run -t upload + +# Upload filesystem +pio run -t uploadfs +``` + +### 4. Monitor Serial Output + +```bash +pio run -t monitor +``` + +Expected output: +``` +FFat mounted successfully +Test begin +Total space: 1486848 +Free space: 1482752 +Listing directory: / + FILE: test.txt SIZE: 12 + FILE: README.md SIZE: 1234 +Test complete +``` + +### 5. Download Filesystem from Device + +To download and extract the filesystem from the device: + +```bash +pio run -t download_fatfs +``` + +Files will be extracted to `unpacked_fs/` directory. + +## Partition Table + +The `partitions.csv` defines the flash layout: + +```csv +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +ffat, data, fat, 0x290000,0x170000, +``` + +The `ffat` partition: +- **Type**: data +- **SubType**: fat (0x81) +- **Offset**: 0x290000 (2,686,976 bytes) +- **Size**: 0x170000 (1,507,328 bytes = ~1.44 MB) + +## Wear Leveling + +The FAT filesystem is automatically wrapped with ESP32's wear leveling layer: + +- **Total partition**: 1,507,328 bytes (368 sectors × 4096 bytes) +- **WL overhead**: 20,480 bytes (5 sectors) +- **FAT data**: 1,486,848 bytes (363 sectors) + +Structure: +``` +[WL State 1][WL State 2][FAT Data][Temp][WL State 3][WL State 4] +``` + +See [WEAR_LEVELING.md](../platform-espressif32/WEAR_LEVELING.md) for details. + +## Troubleshooting + +### "FFat Mount Failed" + +**Possible causes:** +1. Filesystem not uploaded +2. Wrong partition table +3. Corrupted filesystem + +**Solutions:** +```bash +# Rebuild and upload filesystem +pio run -t buildfs +pio run -t uploadfs + +# Or erase flash and start fresh +pio run -t erase +pio run -t upload +pio run -t uploadfs +``` + +### "No FAT filesystem partition found" + +**Cause:** Partition table doesn't have a FAT partition + +**Solution:** Check `partitions.csv` has a partition with `SubType: fat` + +### Files not appearing + +**Cause:** Files not in `data/` directory when building + +**Solution:** +1. Add files to `data/` directory +2. Rebuild filesystem: `pio run -t buildfs` +3. Upload: `pio run -t uploadfs` + +## Code Example + +```cpp +#include "FFat.h" + +void setup() { + Serial.begin(115200); + + // Mount FAT filesystem + if (!FFat.begin(false)) { + Serial.println("FFat Mount Failed"); + return; + } + + // List files + File root = FFat.open("/"); + File file = root.openNextFile(); + while (file) { + Serial.printf("File: %s, Size: %d\n", + file.name(), file.size()); + file = root.openNextFile(); + } + + // Read file + File f = FFat.open("/test.txt", "r"); + if (f) { + String content = f.readString(); + Serial.println(content); + f.close(); + } + + // Write file + f = FFat.open("/output.txt", "w"); + if (f) { + f.println("Hello from ESP32!"); + f.close(); + } +} + +void loop() { + // Your code here +} +``` + +## Platform Configuration + +This project uses a custom platform with FAT filesystem support: + +```ini +[env:esp32dev] +platform = https://github.com/Jason2866/platform-espressif32.git#fatfs_python +framework = arduino +board = esp32dev +board_build.filesystem = fatfs +board_build.partitions = partitions.csv +``` + +## References + +- [ESP32 FFat Library](https://github.com/espressif/arduino-esp32/tree/master/libraries/FFat) +- [ESP-IDF FAT Filesystem](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/fatfs.html) +- [ESP-IDF Wear Levelling](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/wear-levelling.html) +- [Platform-Espressif32 FAT Integration](../platform-espressif32/FATFS_INTEGRATION.md) +- [Wear Leveling Implementation](../platform-espressif32/WEAR_LEVELING.md) diff --git a/examples/arduino-fatfs/TEST_GUIDE.md b/examples/arduino-fatfs/TEST_GUIDE.md new file mode 100644 index 000000000..f31cd0a61 --- /dev/null +++ b/examples/arduino-fatfs/TEST_GUIDE.md @@ -0,0 +1,378 @@ +# FFat Filesystem Test Guide + +This guide explains how to test the ESP32 FFat filesystem with pre-flashed images using Wear Leveling. + +## Quick Start + +### 1. Prepare Test Files + +Add files to the `data/` directory that you want to include in the filesystem: + +```bash +# Example files already included: +data/ +├── test.txt +├── README.md +├── platformio.ini +└── partitions.csv +``` + +### 2. Build Filesystem Image + +Build the FAT filesystem image with Wear Leveling layer: + +```bash +pio run -t buildfs +``` + +This creates `.pio/build/esp32dev/fatfs.bin` with: +- Your files from `data/` directory +- ESP32 Wear Leveling layer +- Proper FAT filesystem structure + +### 3. Upload Firmware and Filesystem + +```bash +# Upload firmware +pio run -t upload + +# Upload filesystem +pio run -t uploadfs +``` + +### 4. Monitor Serial Output + +```bash +pio run -t monitor +``` + +## Test Configuration + +Edit `src/ffat.ino` to configure tests: + +```cpp +// Set to true to format the partition (erases all data!) +#define FORMAT_FFAT false + +// Test settings +#define TEST_READ_EXISTING true // Test reading pre-flashed files +#define TEST_WRITE_NEW true // Test writing new files +#define TEST_FILE_IO true // Test I/O performance +``` + +## Expected Output + +### Successful Mount + +``` +============================================================ +ESP32 FFat Filesystem Test +Testing pre-flashed image with Wear Leveling +============================================================ + +Mounting FFat filesystem... +✓ FFat mounted successfully! + +=== Filesystem Information === +Total space: 1486848 bytes (1.42 MB) +Used space: 12288 bytes (0.01 MB) +Free space: 1474560 bytes (1.41 MB) +Usage: 0.8% +``` + +### Reading Pre-Flashed Files + +``` +============================================================ + +=== Testing Pre-Flashed Files === + +Files in root directory: +Listing directory: / + FILE: test.txt SIZE: 12 + FILE: README.md SIZE: 1234 + FILE: platformio.ini SIZE: 456 + FILE: partitions.csv SIZE: 234 + +Reading test files: + +--- File: /test.txt --- +Reading file: /test.txt +- read from file: +Hello World! + +--- File: /README.md --- +Reading file: /README.md +- read from file: +[README content...] +``` + +### Write Operations + +``` +============================================================ + +=== Testing Write Operations === + +1. Creating new file... +Writing file: /test_write.txt +- file written + +2. Appending to file... +Appending to file: /test_write.txt +- message appended +Appending to file: /test_write.txt +- message appended + +3. Reading back written file: +Reading file: /test_write.txt +- read from file: +Hello from ESP32! +This line was appended. +And another line. + +4. Testing rename... +Renaming file /test_write.txt to /renamed.txt +- file renamed +Reading file: /renamed.txt +[content...] + +5. Testing delete... +Deleting file: /renamed.txt +- file deleted +File successfully deleted +``` + +### Performance Test + +``` +============================================================ + +Testing file I/O with /benchmark.bin +- writing................................ + - 1048576 bytes written in 2345 ms +- reading................................ + - 1048576 bytes read in 1234 ms +``` + +## Troubleshooting + +### "FFat Mount Failed" + +**Possible causes:** + +1. **No FFat partition in partition table** + - Check `partitions.csv` has a `fat` partition + - Verify partition is flashed + +2. **Filesystem not flashed** + ```bash + pio run -t buildfs + pio run -t uploadfs + ``` + +3. **Missing Wear Leveling layer** + - Ensure you're using the updated platform with WL support + - Rebuild filesystem image + +4. **Corrupted filesystem** + - Set `FORMAT_FFAT true` to reformat + - Or erase flash: `pio run -t erase` + +### "File not found" + +If pre-flashed files are not found: + +1. Check files exist in `data/` directory +2. Rebuild filesystem: `pio run -t buildfs` +3. Upload filesystem: `pio run -t uploadfs` +4. Reset ESP32 + +### "Write failed" + +If write operations fail: + +1. Check filesystem is not full +2. Verify partition has write permissions +3. Check for filesystem corruption + +## Advanced Testing + +### Download and Verify Filesystem + +After running tests, download the filesystem to verify changes: + +```bash +pio run -t download_fatfs +``` + +Files will be extracted to `unpacked_fs/` directory. + +### Compare Original and Downloaded + +```bash +# Compare original files +diff data/test.txt unpacked_fs/test.txt + +# Check for new files created by tests +ls -la unpacked_fs/ +``` + +### Test Wear Leveling + +To verify wear leveling is working: + +1. Write many files +2. Download filesystem +3. Check WL state is valid: + +```python +from fatfs import is_esp32_wl_image + +with open('.pio/build/esp32dev/downloaded_fs_*.bin', 'rb') as f: + data = f.read() + +if is_esp32_wl_image(data): + print("✓ Wear Leveling layer is intact") +else: + print("✗ Wear Leveling layer is missing or corrupted") +``` + +## Test Scenarios + +### Scenario 1: Fresh Filesystem + +```cpp +#define FORMAT_FFAT true +#define TEST_READ_EXISTING false +#define TEST_WRITE_NEW true +#define TEST_FILE_IO true +``` + +Tests creating a new filesystem from scratch. + +### Scenario 2: Pre-Flashed Image (Default) + +```cpp +#define FORMAT_FFAT false +#define TEST_READ_EXISTING true +#define TEST_WRITE_NEW true +#define TEST_FILE_IO true +``` + +Tests reading pre-flashed files and writing new ones. + +### Scenario 3: Read-Only Test + +```cpp +#define FORMAT_FFAT false +#define TEST_READ_EXISTING true +#define TEST_WRITE_NEW false +#define TEST_FILE_IO false +``` + +Only tests reading pre-flashed files without modifications. + +### Scenario 4: Performance Only + +```cpp +#define FORMAT_FFAT false +#define TEST_READ_EXISTING false +#define TEST_WRITE_NEW false +#define TEST_FILE_IO true +``` + +Only tests I/O performance. + +## Continuous Integration + +For automated testing: + +```bash +#!/bin/bash +# test_fatfs.sh + +# Build and upload +pio run -t buildfs +pio run -t upload +pio run -t uploadfs + +# Wait for ESP32 to boot +sleep 2 + +# Monitor output and check for success +pio run -t monitor | tee test_output.log + +# Verify output +if grep -q "✓ All tests completed!" test_output.log; then + echo "Tests PASSED" + exit 0 +else + echo "Tests FAILED" + exit 1 +fi +``` + +## Debugging + +### Enable Debug Output + +```cpp +void setup() { + Serial.begin(115200); + Serial.setDebugOutput(true); // Enable ESP32 debug output + // ... +} +``` + +### Check Partition Table + +```bash +# Read partition table from device +pio run -t monitor + +# In another terminal +esptool.py --port /dev/ttyUSB0 read_flash 0x8000 0x1000 partition_table.bin + +# Parse partition table +python -c " +import struct +with open('partition_table.bin', 'rb') as f: + data = f.read() + for i in range(0, len(data), 32): + entry = data[i:i+32] + if entry[:2] == b'\xAA\x50': + print(f'Partition at {i}: {entry.hex()}') +" +``` + +### Verify Wear Leveling + +```bash +# Download filesystem +pio run -t download_fatfs + +# Check WL structure +python -c " +from fatfs import is_esp32_wl_image, ESP32WearLeveling +import struct + +with open('.pio/build/esp32dev/downloaded_fs_*.bin', 'rb') as f: + data = f.read() + +if is_esp32_wl_image(data): + wl = ESP32WearLeveling() + state = data[:48] + fields = struct.unpack(' 512) { + toRead = 512; + } + file.read(buf, toRead); + if ((i++ & 0x001F) == 0x001F) { + Serial.print("."); + } + len -= toRead; + } + Serial.println(""); + end = millis() - start; + Serial.printf("- %u bytes read in %lu ms\r\n", flen, end); + file.close(); + } else { + Serial.println("- failed to open file for reading"); + } +} + +void testExistingFiles(fs::FS &fs) { + Serial.println("\n=== Testing Pre-Flashed Files ==="); + + // List all files in root + Serial.println("\nFiles in root directory:"); + listDir(fs, "/", 2); + + // Test reading specific files that should exist + const char* testFiles[] = { + "/test.txt", + "/README.md", + "/platformio.ini", + "/partitions.csv" + }; + + Serial.println("\nReading test files:"); + for (int i = 0; i < 4; i++) { + if (fs.exists(testFiles[i])) { + Serial.printf("\n--- File: %s ---\n", testFiles[i]); + readFile(fs, testFiles[i]); + } else { + Serial.printf("File not found: %s\n", testFiles[i]); + } + } +} + +void testWriteOperations(fs::FS &fs) { + Serial.println("\n=== Testing Write Operations ==="); + + // Test creating new file + Serial.println("\n1. Creating new file..."); + writeFile(fs, "/test_write.txt", "Hello from ESP32!\n"); + + // Test appending + Serial.println("\n2. Appending to file..."); + appendFile(fs, "/test_write.txt", "This line was appended.\n"); + appendFile(fs, "/test_write.txt", "And another line.\n"); + + // Read back + Serial.println("\n3. Reading back written file:"); + readFile(fs, "/test_write.txt"); + + // Test rename + Serial.println("\n4. Testing rename..."); + renameFile(fs, "/test_write.txt", "/renamed.txt"); + readFile(fs, "/renamed.txt"); + + // Test delete + Serial.println("\n5. Testing delete..."); + deleteFile(fs, "/renamed.txt"); + + // Verify deletion + if (!fs.exists("/renamed.txt")) { + Serial.println("File successfully deleted"); + } else { + Serial.println("ERROR: File still exists!"); + } +} + +void testFileSystem(fs::FS &fs) { + Serial.println("\n=== Filesystem Information ==="); + Serial.printf("Total space: %10u bytes (%.2f MB)\n", + FFat.totalBytes(), FFat.totalBytes() / 1024.0 / 1024.0); + Serial.printf("Used space: %10u bytes (%.2f MB)\n", + FFat.usedBytes(), FFat.usedBytes() / 1024.0 / 1024.0); + Serial.printf("Free space: %10u bytes (%.2f MB)\n", + FFat.freeBytes(), FFat.freeBytes() / 1024.0 / 1024.0); + + float usage = (FFat.usedBytes() * 100.0) / FFat.totalBytes(); + Serial.printf("Usage: %.1f%%\n", usage); +} + +void printSeparator() { + Serial.println("\n============================================================"); +} + +void setup() { + Serial.begin(115200); + delay(1000); // Wait for serial monitor + + // Enable ESP-IDF debug logging + esp_log_level_set("*", ESP_LOG_DEBUG); + esp_log_level_set("vfs_fat", ESP_LOG_VERBOSE); + esp_log_level_set("wear_levelling", ESP_LOG_VERBOSE); + + Serial.println("\n\n"); + printSeparator(); + Serial.println("ESP32 FFat Filesystem Test"); + Serial.println("Testing pre-flashed image with Wear Leveling"); + printSeparator(); + + // Format if requested + if (FORMAT_FFAT) { + Serial.println("\n============================================================"); + Serial.println("WARNING: Formatting FFat partition..."); + Serial.println("This will erase all data!"); + Serial.println("============================================================"); + delay(2000); + + // First try to mount the WL layer without formatting + Serial.println("\nStep 1: Checking if WL layer can be mounted..."); + if (FFat.begin(false, "/ffat", 10, "ffat")) { + Serial.println("✓ WL layer mounted successfully!"); + Serial.println(" Now formatting filesystem..."); + FFat.end(); + } else { + Serial.println("✗ WL layer mount failed - will try to format anyway"); + } + + Serial.println("\nStep 2: Formatting partition..."); + if (!FFat.format(false, (char*)"ffat")) { + Serial.println("ERROR: FFat Format Failed"); + Serial.println("\nThis means the Wear Leveling layer itself is broken."); + Serial.println("The WL structure on flash is not compatible with ESP-IDF."); + return; + } + Serial.println("✓ FFat formatted successfully"); + + Serial.println("\nStep 3: Mounting formatted partition..."); + if (!FFat.begin(false, "/ffat", 10, "ffat")) { + Serial.println("ERROR: Cannot mount even after format!"); + return; + } + Serial.println("✓ Mounted successfully after format"); + + // Write some test files so we have data to compare + Serial.println("\nWriting test files..."); + writeFile(FFat, "/test.txt", "This is a test file created by ESP32\n"); + writeFile(FFat, "/README.md", "# ESP32 Formatted Filesystem\n\nThis was formatted on device.\n"); + + Serial.println("\n*** IMPORTANT: Now download this filesystem ***"); + Serial.println("Run: pio run -t download_fatfs"); + Serial.println("This will save the ESP32-formatted image for comparison"); + + delay(5000); // Give time to read the message + } + + // Mount the filesystem + Serial.println("\nMounting FFat filesystem..."); + Serial.println("Partition label: 'ffat'"); + Serial.println("Format on fail: false"); + Serial.println("Max open files: 10"); + + if (!FFat.begin(false, "/ffat", 10, "ffat")) { // formatOnFail = false + Serial.println("\nERROR: FFat Mount Failed"); + Serial.println("\nDiagnostics:"); + Serial.println("1. Checking partition..."); + + // Try to get partition info + const esp_partition_t* partition = esp_partition_find_first( + ESP_PARTITION_TYPE_DATA, + ESP_PARTITION_SUBTYPE_DATA_FAT, + "ffat" + ); + + if (partition) { + Serial.printf(" ✓ Partition found: %s\n", partition->label); + Serial.printf(" Address: 0x%06X\n", partition->address); + Serial.printf(" Size: %u bytes (%.2f MB)\n", + partition->size, partition->size / 1024.0 / 1024.0); + + // Check FAT boot sector at offset 0 (RAW FAT, no WL layer in flash) + Serial.println("\n2. Checking FAT boot sector..."); + uint8_t buffer[512]; + if (esp_partition_read(partition, 0, buffer, 512) == ESP_OK) { + Serial.println(" First 64 bytes at offset 0 (RAW FAT):"); + for (int i = 0; i < 64; i++) { + Serial.printf("%02X ", buffer[i]); + if ((i + 1) % 16 == 0) Serial.println(); + } + Serial.println(); + + // Check boot signature + if (buffer[510] == 0x55 && buffer[511] == 0xAA) { + Serial.println(" ✓ Boot signature found (0x55AA)"); + + // Parse boot sector + char oem[9]; + memcpy(oem, buffer + 3, 8); + oem[8] = 0; + uint16_t bytes_per_sector = buffer[11] | (buffer[12] << 8); + uint16_t total_sectors = buffer[19] | (buffer[20] << 8); + + Serial.printf(" OEM Name: '%s'\n", oem); + Serial.printf(" Bytes per sector: %u\n", bytes_per_sector); + Serial.printf(" Total sectors: %u\n", total_sectors); + + if (bytes_per_sector == 4096 && total_sectors == 362) { + Serial.println(" ✓ Correct format for ESP32 FFat!"); + } + } else { + Serial.printf(" ✗ Invalid boot signature: 0x%02X%02X (expected 0x55AA)\n", + buffer[511], buffer[510]); + } + } + + // Check FAT table + Serial.println("\n3. Checking FAT table..."); + if (esp_partition_read(partition, 4096, buffer, 32) == ESP_OK) { + Serial.println(" FAT1 (first 32 bytes at offset 4096):"); + for (int i = 0; i < 32; i++) { + Serial.printf("%02X ", buffer[i]); + if ((i + 1) % 16 == 0) Serial.println(); + } + Serial.println(); + + if (buffer[0] == 0xF8 && buffer[1] == 0xFF && buffer[2] == 0xFF) { + Serial.println(" ✓ Media descriptor correct (F8 FF FF)"); + + // Check if rest is 00 + bool clean = true; + for (int i = 3; i < 32; i++) { + if (buffer[i] != 0x00) { + clean = false; + break; + } + } + + if (clean) { + Serial.println(" ✓ FAT table clean (all zeros after media descriptor)"); + } else { + Serial.println(" ✗ FAT table has non-zero bytes after media descriptor"); + } + } + } + + // Check WL metadata at end of partition + Serial.println("\n4. Checking WL metadata at end..."); + // WL state1 should be at partition_size - 3 * sector_size + uint32_t wl_state1_offset = partition->size - (3 * 4096); + if (esp_partition_read(partition, wl_state1_offset, buffer, 48) == ESP_OK) { + Serial.printf(" WL State at offset 0x%06X (first 48 bytes):\n", wl_state1_offset); + for (int i = 0; i < 48; i++) { + Serial.printf("%02X ", buffer[i]); + if ((i + 1) % 16 == 0) Serial.println(); + } + Serial.println(); + + // Parse WL_State + uint32_t* words = (uint32_t*)buffer; + Serial.printf(" pos: %u\n", words[0]); + Serial.printf(" max_pos: %u\n", words[1]); + Serial.printf(" block_size: %u\n", words[5]); + Serial.printf(" version: %u\n", words[6]); + + if (words[5] == 4096 && words[6] == 2) { + Serial.println(" ✓ WL metadata looks valid"); + } else { + Serial.println(" ✗ WL metadata invalid"); + } + } + } else { + Serial.println(" ✗ Partition 'ffat' not found!"); + Serial.println(" Check partition table in platformio.ini"); + } + + Serial.println("\nPossible causes:"); + Serial.println("- Corrupted FAT filesystem"); + Serial.println("- Wrong sector size (should be 4096)"); + Serial.println("- Wrong sector count (should be 362)"); + Serial.println("\nTry: pio run -t erase && pio run -t upload && pio run -t uploadfs"); + return; + } + + Serial.println("✓ FFat mounted successfully!"); + + // Show filesystem info + testFileSystem(FFat); + + // Test reading existing files + if (TEST_READ_EXISTING) { + printSeparator(); + testExistingFiles(FFat); + } + + // Test write operations + if (TEST_WRITE_NEW) { + printSeparator(); + testWriteOperations(FFat); + + // Show updated filesystem info + printSeparator(); + Serial.println("\nFilesystem after write tests:"); + testFileSystem(FFat); + } + + // Test file I/O performance + if (TEST_FILE_IO) { + printSeparator(); + testFileIO(FFat, "/benchmark.bin"); + + // Clean up benchmark file + deleteFile(FFat, "/benchmark.bin"); + } + + // Final directory listing + printSeparator(); + Serial.println("\nFinal directory listing:"); + listDir(FFat, "/", 2); + + printSeparator(); + Serial.println("\n✓ All tests completed!"); + Serial.println("\nFilesystem remains mounted for further testing."); + Serial.println("You can now:"); + Serial.println("- Download filesystem: pio run -t download_fatfs"); + Serial.println("- Reset to re-run tests"); + printSeparator(); +} + +void loop() { + // Keep the filesystem mounted + // You can add interactive commands here if needed +} diff --git a/platform.json b/platform.json index 3a34c15a9..dde9da9d1 100644 --- a/platform.json +++ b/platform.json @@ -135,13 +135,6 @@ "package-version": "2.230.0", "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/mkspiffs-v2.230.0.zip" }, - "tool-mkfatfs": { - "type": "uploader", - "optional": true, - "owner": "pioarduino", - "package-version": "2.0.1", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/mkfatfs-v2.0.1.zip" - }, "tool-cppcheck": { "type": "tool", "optional": true, diff --git a/platform.py b/platform.py index 531a7e671..9f0aac51c 100644 --- a/platform.py +++ b/platform.py @@ -720,10 +720,9 @@ def _configure_check_tools(self, variables: Dict) -> None: def _install_filesystem_tool(self, filesystem: str) -> None: """Install filesystem-specific tools based on the filesystem type.""" - # LittleFS is handled by littlefs-python, only install tools for other filesystems - if filesystem == "fatfs": - self.install_tool("tool-mkfatfs") - elif filesystem == "spiffs": + # LittleFS and FatFS are handled by Python modules (littlefs-python, fatfs-python) + # Only install tools for other filesystems + if filesystem == "spiffs": self.install_tool("tool-mkspiffs") def _handle_dfuutil_tool(self, variables: Dict, for_download: bool = False) -> None: From e0b543d0f216d1c8d9fdde45bce77cdcb3d41560 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 27 Dec 2025 13:00:03 +0100 Subject: [PATCH 2/7] Remove solution section from WEAR_LEVELING.md Removed the solution section explaining the esp32_wl.py module. --- WEAR_LEVELING.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/WEAR_LEVELING.md b/WEAR_LEVELING.md index ceee00dfa..91663a3e1 100644 --- a/WEAR_LEVELING.md +++ b/WEAR_LEVELING.md @@ -10,10 +10,6 @@ The ESP32 Arduino Core expects FAT partitions to be wrapped with a wear leveling - **Without WL**: Raw FAT filesystem → **Mount fails** - **With WL**: WL State + FAT filesystem + WL metadata → **Mount succeeds** -## Solution - -The `esp32_wl.py` module implements the ESP-IDF wear leveling structure to wrap FAT images. - ## Wear Leveling Structure ``` From 875e081b7380136a3a1baa8e09854a5bbb099b6b Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 27 Dec 2025 13:06:13 +0100 Subject: [PATCH 3/7] Update state_data struct packing format --- WEAR_LEVELING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WEAR_LEVELING.md b/WEAR_LEVELING.md index 91663a3e1..7fc2df87f 100644 --- a/WEAR_LEVELING.md +++ b/WEAR_LEVELING.md @@ -110,7 +110,7 @@ Successfully extracted 1 file(s) to unpacked_fs The WL_State CRC32 is calculated over the first 44 bytes (excluding the CRC field itself): ```python -state_data = struct.pack(' Date: Sat, 27 Dec 2025 13:22:45 +0100 Subject: [PATCH 4/7] Update links in README.md for FATFS documentation --- examples/arduino-fatfs/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/arduino-fatfs/README.md b/examples/arduino-fatfs/README.md index cd049e74d..62fde9f12 100644 --- a/examples/arduino-fatfs/README.md +++ b/examples/arduino-fatfs/README.md @@ -120,7 +120,7 @@ Structure: [WL State 1][WL State 2][FAT Data][Temp][WL State 3][WL State 4] ``` -See [WEAR_LEVELING.md](../platform-espressif32/WEAR_LEVELING.md) for details. +See [WEAR_LEVELING.md](../../WEAR_LEVELING.md) for details. ## Troubleshooting @@ -208,7 +208,7 @@ This project uses a custom platform with FAT filesystem support: ```ini [env:esp32dev] -platform = https://github.com/Jason2866/platform-espressif32.git#fatfs_python +platform = espressif32 framework = arduino board = esp32dev board_build.filesystem = fatfs @@ -220,5 +220,5 @@ board_build.partitions = partitions.csv - [ESP32 FFat Library](https://github.com/espressif/arduino-esp32/tree/master/libraries/FFat) - [ESP-IDF FAT Filesystem](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/fatfs.html) - [ESP-IDF Wear Levelling](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/wear-levelling.html) -- [Platform-Espressif32 FAT Integration](../platform-espressif32/FATFS_INTEGRATION.md) -- [Wear Leveling Implementation](../platform-espressif32/WEAR_LEVELING.md) +- [Platform-Espressif32 FAT Integration](../../FATFS_INTEGRATION.md) +- [Wear Leveling Implementation](../../platform-espressif32/WEAR_LEVELING.md) From 9e930e907d534c8413d2de54915fd075fadbe75d Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 27 Dec 2025 13:26:13 +0100 Subject: [PATCH 5/7] Remove verification section from WEAR_LEVELING.md Removed verification section including code examples for checking WL state and extracting FAT data. --- WEAR_LEVELING.md | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/WEAR_LEVELING.md b/WEAR_LEVELING.md index 7fc2df87f..22cf70860 100644 --- a/WEAR_LEVELING.md +++ b/WEAR_LEVELING.md @@ -145,37 +145,6 @@ Unused areas are filled with `0xFF` (erased flash state): - FAT16 (medium partitions) - FAT32 (large partitions, >32MB) -## Verification - -### Check WL State - -```python -from esp32_wl import WearLevelingLayer - -wl = WearLevelingLayer() -with open('fatfs.bin', 'rb') as f: - data = f.read() - -# Verify first state sector -state = data[:48] -is_valid = wl.verify_wl_state(state) -print(f"WL State valid: {is_valid}") -``` - -### Extract FAT Data - -```python -from esp32_wl import extract_fat_from_wl_image - -with open('fatfs.bin', 'rb') as f: - wl_image = f.read() - -fat_data = extract_fat_from_wl_image(wl_image) -if fat_data: - with open('fat_only.bin', 'wb') as f: - f.write(fat_data) -``` - ## Troubleshooting ### "FFat Mount Failed" From a4a48fab41f7ae8e7e22c2d044bffad30289d321 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 27 Dec 2025 13:28:59 +0100 Subject: [PATCH 6/7] Modify WL structure check to support multiple files Updated Python script to handle multiple downloaded FS files. --- examples/arduino-fatfs/TEST_GUIDE.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/arduino-fatfs/TEST_GUIDE.md b/examples/arduino-fatfs/TEST_GUIDE.md index f31cd0a61..33e681fe8 100644 --- a/examples/arduino-fatfs/TEST_GUIDE.md +++ b/examples/arduino-fatfs/TEST_GUIDE.md @@ -354,10 +354,12 @@ pio run -t download_fatfs # Check WL structure python -c " +import glob from fatfs import is_esp32_wl_image, ESP32WearLeveling import struct -with open('.pio/build/esp32dev/downloaded_fs_*.bin', 'rb') as f: +files = glob.glob('.pio/build/esp32dev/downloaded_fs_*.bin') +with open(files[0], 'rb') as f: data = f.read() if is_esp32_wl_image(data): From 25ef3b60b7641847eecacfd8bbb6648c86b7cdbb Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 27 Dec 2025 13:32:37 +0100 Subject: [PATCH 7/7] Update platform URL for ESP32 development environment --- examples/arduino-fatfs/data/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/arduino-fatfs/data/README.md b/examples/arduino-fatfs/data/README.md index ba7929bdb..38284025d 100644 --- a/examples/arduino-fatfs/data/README.md +++ b/examples/arduino-fatfs/data/README.md @@ -208,7 +208,7 @@ This project uses a custom platform with FAT filesystem support: ```ini [env:esp32dev] -platform = https://github.com/Jason2866/platform-espressif32.git#fatfs_python +platform = espressif32 framework = arduino board = esp32dev board_build.filesystem = fatfs