From a72ac07ce7c3f5001a03fbbd2f6adac4b02a312a Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Dec 2025 20:56:59 +0100 Subject: [PATCH 1/6] Add download_littlefs target and function Updated the switch_off_ldf function to include 'download_littlefs' in the targets. Added a new function 'download_littlefs' to download and extract the LittleFS filesystem from the device. --- builder/main.py | 221 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 219 insertions(+), 2 deletions(-) diff --git a/builder/main.py b/builder/main.py index ebb444227..fe181fbf5 100644 --- a/builder/main.py +++ b/builder/main.py @@ -529,12 +529,12 @@ def check_lib_archive_exists(): def switch_off_ldf(): """ - Disables LDF (Library Dependency Finder) for uploadfs, uploadfsota, and buildfs targets. + Disables LDF (Library Dependency Finder) for uploadfs, uploadfsota, buildfs, download_littlefs, and erase targets. This optimization prevents unnecessary library dependency scanning and compilation when only filesystem operations are performed. """ - fs_targets = {"uploadfs", "uploadfsota", "buildfs", "erase"} + fs_targets = {"uploadfs", "uploadfsota", "buildfs", "erase", "download_littlefs"} if fs_targets & set(COMMAND_LINE_TARGETS): # Disable LDF by modifying project configuration directly env_section = "env:" + env["PIOENV"] @@ -876,6 +876,215 @@ def coredump_analysis(target, source, env): print(f"Error: Failed to run coredump analysis: {e}") print(f'Make sure esp-coredump is installed: uv pip install --python "{PYTHON_EXE}" esp-coredump') + +def download_littlefs(target, source, env): + """ + Download Little filesystem from device and extract to directory. + Only supports LittleFS filesystem. + Usage: pio run -t downloadfs + + Args: + target: SCons target + source: SCons source + env: SCons environment object + """ + # Get unpack directory from project config or use default + unpack_dir = env.GetProjectOption("custom_unpack_dir", "unpacked_fs") + + # Ensure upload port is set + if not env.subst("$UPLOAD_PORT"): + env.AutodetectUploadPort() + + upload_port = env.subst("$UPLOAD_PORT") + download_speed = board.get("download.speed", "115200") + + # Download partition table from device + print(f"Downloading partition table from {upload_port}...") + + build_dir = Path(env.subst("$BUILD_DIR")) + build_dir.mkdir(parents=True, exist_ok=True) + partition_file = build_dir / "partition_table_from_flash.bin" + + esptool_cmd = [ + uploader_path.strip('"'), + "--chip", mcu, + "--port", upload_port, + "--baud", str(download_speed), + "--before", "default-reset", + "--after", "hard-reset", + "read-flash", + "0x8000", # Partition table offset + "0x1000", # Partition table size (4KB) + str(partition_file) + ] + + try: + result = subprocess.run(esptool_cmd, check=True) + if result.returncode != 0: + print(f"Error: Failed to download partition table") + return 1 + except subprocess.CalledProcessError as e: + print(f"Error: Download failed: {e}") + return 1 + except Exception as e: + print(f"Error: {e}") + return 1 + + # Parse partition table to find filesystem partition + print("Parsing partition table...") + + with open(partition_file, 'rb') as f: + partition_data = f.read() + + # Parse partition entries (format: 0xAA 0x50 followed by entry data) + entries = [e for e in partition_data.split(b'\xaaP') if len(e) > 0] + + fs_start = None + fs_size = None + fs_type_name = None + fs_subtype = None + + for entry in entries: + if len(entry) < 32: + continue + + # Partition entry format (after 0xAA 0x50 magic): + # The entry structure after split is: + # Byte 0: Unknown/padding + # Byte 1: Type/Subtype combined + # Bytes 2-4: Offset (3 bytes, little-endian) + # Bytes 5: Unknown/padding + # Bytes 6-8: Size (3 bytes, little-endian) + + part_type = entry[1] + + # Check for SPIFFS (0x82) or LITTLEFS (0x83) + if part_type in [0x82, 0x83]: + fs_start = int.from_bytes(entry[2:5], byteorder='little', signed=False) + fs_size = int.from_bytes(entry[6:9], byteorder='little', signed=False) + fs_subtype = part_type + fs_type_name = "LittleFS" if part_type == 0x83 else "SPIFFS" + break + + if fs_start is None or fs_size is None: + print("Error: No filesystem partition found in partition table") + return 1 + + # Check if filesystem is supported + # Note: LittleFS can use subtype 0x82 or 0x83 + # We only support LittleFS extraction, not SPIFFS + # The actual filesystem type will be detected when mounting + if fs_subtype not in [0x82, 0x83]: + print(f"Error: Unsupported filesystem partition type") + return 1 + + block_size = 0x1000 # 4KB + page_size = 0x100 # 256 bytes + + print(f"Found filesystem partition (subtype {hex(fs_subtype)}):") + print(f" Start: {hex(fs_start)}") + print(f" Size: {hex(fs_size)} ({fs_size} bytes)") + print(f" Block size: {hex(block_size)}") + print(f"Note: This tool only supports LittleFS extraction") + + # Download filesystem image + fs_file = build_dir / f"downloaded_fs_{hex(fs_start)}_{hex(fs_size)}.bin" + + print(f"\nDownloading filesystem from device...") + + esptool_cmd = [ + uploader_path.strip('"'), + "--chip", mcu, + "--port", upload_port, + "--baud", str(download_speed), + "--before", "default-reset", + "--after", "hard-reset", + "read-flash", + hex(fs_start), + hex(fs_size), + str(fs_file) + ] + + try: + result = subprocess.run(esptool_cmd, check=True) + if result.returncode != 0: + print(f"Error: Download failed with code {result.returncode}") + return 1 + except subprocess.CalledProcessError as e: + print(f"Error: Download failed: {e}") + return 1 + except Exception as e: + print(f"Error: {e}") + return 1 + + print(f"Downloaded to {fs_file}") + + # Extract filesystem + print(f"\nExtracting LittleFS filesystem to {unpack_dir}...") + + # Remove old unpack directory + unpack_path = Path(get_project_dir()) / unpack_dir + if unpack_path.exists(): + import shutil + shutil.rmtree(unpack_path) + unpack_path.mkdir(parents=True, exist_ok=True) + + try: + # Read the downloaded filesystem image + with open(fs_file, 'rb') as f: + fs_data = f.read() + + # Calculate block count + block_count = fs_size // block_size + + # Create LittleFS instance and mount the image + fs = LittleFS( + block_size=block_size, + block_count=block_count, + mount=False + ) + fs.context.buffer = bytearray(fs_data) + fs.mount() + + # Extract all files + file_count = 0 + print("\nExtracted files:") + for root, dirs, files in fs.walk("/"): + if not root.endswith("/"): + root += "/" + + # Create directories + for dir_name in dirs: + src_path = root + dir_name + dst_path = unpack_path / src_path[1:] # Remove leading '/' + dst_path.mkdir(parents=True, exist_ok=True) + print(f" [DIR] {src_path}") + + # Extract files + for file_name in files: + src_path = root + file_name + dst_path = unpack_path / src_path[1:] # Remove leading '/' + dst_path.parent.mkdir(parents=True, exist_ok=True) + + with fs.open(src_path, "rb") as src: + file_data = src.read() + dst_path.write_bytes(file_data) + + print(f" [FILE] {src_path} ({len(file_data)} bytes)") + file_count += 1 + + fs.unmount() + print(f"\nSuccessfully extracted {file_count} file(s) to {unpack_dir}") + return 0 + + except Exception as e: + print(f"Error: Failed to extract LittleFS filesystem: {e}") + print("This tool only supports LittleFS. If you have SPIFFS, please convert to LittleFS.") + print("Make sure the device has a valid LittleFS filesystem.") + import traceback + traceback.print_exc() + return 1 + # # Target: Build executable and linkable firmware or FS image # @@ -1113,6 +1322,14 @@ def coredump_analysis(target, source, env): "Upload Filesystem Image OTA", ) +# Target: Download LittleFS (no build required) +env.AddPlatformTarget( + "download_littlefs", + None, + download_littlefs, + "Download and extract LittleFS filesystem from device", +) + # Target: Erase Flash and Upload env.AddPlatformTarget( "erase_upload", From 77c9b7c700bcee474f24ffcb1832d54f1e0c487a Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Dec 2025 21:00:19 +0100 Subject: [PATCH 2/6] Fix usage command in download_littlefs function --- builder/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/main.py b/builder/main.py index fe181fbf5..17eb2f632 100644 --- a/builder/main.py +++ b/builder/main.py @@ -881,7 +881,7 @@ def download_littlefs(target, source, env): """ Download Little filesystem from device and extract to directory. Only supports LittleFS filesystem. - Usage: pio run -t downloadfs + Usage: pio run -t download_littlefs Args: target: SCons target From 6a2c23e7d62bd2780f04c7117b6d6b3f799628bc Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Dec 2025 21:30:49 +0100 Subject: [PATCH 3/6] Comment out arduino-zigbee-switch example Comment out the 'arduino-zigbee-switch' example in the workflow. --- .github/workflows/examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index a8cde986d..b8eb8bd2b 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -19,7 +19,7 @@ jobs: - "examples/arduino-usb-keyboard" - "examples/arduino-wifiscan" - "examples/arduino-zigbee-light" - - "examples/arduino-zigbee-switch" +# - "examples/arduino-zigbee-switch" - "examples/arduino-NimBLE-SampleScan" - "examples/arduino-matter-light" - "examples/tasmota" From a2d6c2759fbe00d34f3f9fea31a4d5c5d8da7692 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Dec 2025 21:37:40 +0100 Subject: [PATCH 4/6] Update main.py --- builder/main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/builder/main.py b/builder/main.py index 17eb2f632..661e27335 100644 --- a/builder/main.py +++ b/builder/main.py @@ -941,7 +941,6 @@ def download_littlefs(target, source, env): fs_start = None fs_size = None - fs_type_name = None fs_subtype = None for entry in entries: @@ -963,7 +962,6 @@ def download_littlefs(target, source, env): fs_start = int.from_bytes(entry[2:5], byteorder='little', signed=False) fs_size = int.from_bytes(entry[6:9], byteorder='little', signed=False) fs_subtype = part_type - fs_type_name = "LittleFS" if part_type == 0x83 else "SPIFFS" break if fs_start is None or fs_size is None: @@ -979,7 +977,6 @@ def download_littlefs(target, source, env): return 1 block_size = 0x1000 # 4KB - page_size = 0x100 # 256 bytes print(f"Found filesystem partition (subtype {hex(fs_subtype)}):") print(f" Start: {hex(fs_start)}") From e0393fa658cac1e028de7df946e47ea8e5c606a0 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Dec 2025 21:40:56 +0100 Subject: [PATCH 5/6] Change subprocess.run to not check for errors --- builder/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/main.py b/builder/main.py index 661e27335..7ada796d8 100644 --- a/builder/main.py +++ b/builder/main.py @@ -919,7 +919,7 @@ def download_littlefs(target, source, env): ] try: - result = subprocess.run(esptool_cmd, check=True) + result = subprocess.run(esptool_cmd, check=False) if result.returncode != 0: print(f"Error: Failed to download partition table") return 1 @@ -1003,7 +1003,7 @@ def download_littlefs(target, source, env): ] try: - result = subprocess.run(esptool_cmd, check=True) + result = subprocess.run(esptool_cmd, check=False) if result.returncode != 0: print(f"Error: Download failed with code {result.returncode}") return 1 From 270397c6bb748af0f291dfbf72bf0196d831783e Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Dec 2025 22:08:51 +0100 Subject: [PATCH 6/6] Refactor error handling and partition entry parsing --- builder/main.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/builder/main.py b/builder/main.py index 7ada796d8..0fc457adf 100644 --- a/builder/main.py +++ b/builder/main.py @@ -921,11 +921,8 @@ def download_littlefs(target, source, env): try: result = subprocess.run(esptool_cmd, check=False) if result.returncode != 0: - print(f"Error: Failed to download partition table") + print("Error: Failed to download partition table") return 1 - except subprocess.CalledProcessError as e: - print(f"Error: Download failed: {e}") - return 1 except Exception as e: print(f"Error: {e}") return 1 @@ -947,21 +944,18 @@ def download_littlefs(target, source, env): if len(entry) < 32: continue - # Partition entry format (after 0xAA 0x50 magic): - # The entry structure after split is: - # Byte 0: Unknown/padding - # Byte 1: Type/Subtype combined - # Bytes 2-4: Offset (3 bytes, little-endian) - # Bytes 5: Unknown/padding - # Bytes 6-8: Size (3 bytes, little-endian) + # Byte 0: Type (0x01 for data partitions) + # Byte 1: SubType (0x82=SPIFFS, 0x83=LittleFS) + # Bytes 2-5: Offset (4 bytes, little-endian) + # Bytes 6-9: Size (4 bytes, little-endian) - part_type = entry[1] + part_subtype = entry[1] # Check for SPIFFS (0x82) or LITTLEFS (0x83) - if part_type in [0x82, 0x83]: - fs_start = int.from_bytes(entry[2:5], byteorder='little', signed=False) - fs_size = int.from_bytes(entry[6:9], byteorder='little', signed=False) - fs_subtype = part_type + if part_subtype in [0x82, 0x83]: + fs_start = int.from_bytes(entry[2:6], byteorder='little', signed=False) + fs_size = int.from_bytes(entry[6:10], byteorder='little', signed=False) + fs_subtype = part_subtype break if fs_start is None or fs_size is None: @@ -1007,9 +1001,6 @@ def download_littlefs(target, source, env): if result.returncode != 0: print(f"Error: Download failed with code {result.returncode}") return 1 - except subprocess.CalledProcessError as e: - print(f"Error: Download failed: {e}") - return 1 except Exception as e: print(f"Error: {e}") return 1