From 76bc2f05ec9c7b0add32c1aac31bfe6a3363137b Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 28 Dec 2025 15:26:57 +0100 Subject: [PATCH 1/5] try to use target dl fs without env --- builder/main.py | 119 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 86 insertions(+), 33 deletions(-) diff --git a/builder/main.py b/builder/main.py index 2ba9ca4b0..3573bf7f7 100644 --- a/builder/main.py +++ b/builder/main.py @@ -1334,7 +1334,7 @@ def download_fs(target, source, env): if fs_type == "littlefs": return _extract_littlefs(fs_file, fs_size, unpack_path, unpack_dir) elif fs_type == "spiffs": - return _extract_spiffs(fs_file, fs_size, unpack_path, unpack_dir, env) + return _extract_spiffs(fs_file, fs_size, unpack_path, unpack_dir) elif fs_type == "fatfs": return _extract_fatfs(fs_file, unpack_path, unpack_dir) else: @@ -1521,53 +1521,106 @@ def _extract_littlefs(fs_file, fs_size, unpack_path, unpack_dir): return 0 -def _extract_spiffs(fs_file, fs_size, unpack_path, unpack_dir, env): - """Extract SPIFFS filesystem.""" - # Get SPIFFS configuration - page_size = 256 - block_size = 4096 - obj_name_len = 32 - meta_len = 4 - use_magic = True - use_magic_len = True - aligned_obj_ix_tables = False +def _parse_spiffs_config(fs_data, fs_size): + """ + Auto-detect SPIFFS configuration from the image. + Tries common configurations and validates against the image. + + Returns: + dict: SPIFFS configuration parameters or None + """ + # Common ESP32/ESP8266 SPIFFS configurations + common_configs = [ + # ESP32/ESP8266 defaults + {'page_size': 256, 'block_size': 4096, 'obj_name_len': 32}, + # Alternative configurations + {'page_size': 256, 'block_size': 8192, 'obj_name_len': 32}, + {'page_size': 512, 'block_size': 4096, 'obj_name_len': 32}, + {'page_size': 256, 'block_size': 4096, 'obj_name_len': 64}, + ] + + print("\nAuto-detecting SPIFFS configuration...") + + for config in common_configs: + try: + # Try to parse with this configuration + spiffs_build_config = SpiffsBuildConfig( + page_size=config['page_size'], + page_ix_len=2, + block_size=config['block_size'], + block_ix_len=2, + meta_len=4, + obj_name_len=config['obj_name_len'], + obj_id_len=2, + span_ix_len=2, + packed=True, + aligned=True, + endianness='little', + use_magic=True, + use_magic_len=True, + aligned_obj_ix_tables=False + ) + + # Try to create and parse the filesystem + spiffs = SpiffsFS(fs_size, spiffs_build_config) + spiffs.from_binary(fs_data) + + # If we got here without exception, this config works + print(" Detected SPIFFS configuration:") + print(f" Page size: {config['page_size']} bytes") + print(f" Block size: {config['block_size']} bytes") + print(f" Max filename length: {config['obj_name_len']}") + + return { + 'page_size': config['page_size'], + 'block_size': config['block_size'], + 'obj_name_len': config['obj_name_len'], + 'meta_len': 4, + 'use_magic': True, + 'use_magic_len': True, + 'aligned_obj_ix_tables': False + } + except Exception: + continue + + # If no config worked, return defaults + print(" Could not auto-detect configuration, using ESP32/ESP8266 defaults") + return { + 'page_size': 256, + 'block_size': 4096, + 'obj_name_len': 32, + 'meta_len': 4, + 'use_magic': True, + 'use_magic_len': True, + 'aligned_obj_ix_tables': False + } - for section in ["common", "env:" + env["PIOENV"]]: - if projectconfig.has_option(section, "board_build.spiffs.page_size"): - page_size = int(projectconfig.get(section, "board_build.spiffs.page_size")) - if projectconfig.has_option(section, "board_build.spiffs.block_size"): - block_size = int(projectconfig.get(section, "board_build.spiffs.block_size")) - if projectconfig.has_option(section, "board_build.spiffs.obj_name_len"): - obj_name_len = int(projectconfig.get(section, "board_build.spiffs.obj_name_len")) - if projectconfig.has_option(section, "board_build.spiffs.meta_len"): - meta_len = int(projectconfig.get(section, "board_build.spiffs.meta_len")) - if projectconfig.has_option(section, "board_build.spiffs.use_magic"): - use_magic = projectconfig.getboolean(section, "board_build.spiffs.use_magic") - if projectconfig.has_option(section, "board_build.spiffs.use_magic_len"): - use_magic_len = projectconfig.getboolean(section, "board_build.spiffs.use_magic_len") - if projectconfig.has_option(section, "board_build.spiffs.aligned_obj_ix_tables"): - aligned_obj_ix_tables = projectconfig.getboolean(section, "board_build.spiffs.aligned_obj_ix_tables") +def _extract_spiffs(fs_file, fs_size, unpack_path, unpack_dir): + """Extract SPIFFS filesystem with auto-detected configuration.""" # Read the downloaded filesystem image with open(fs_file, 'rb') as f: fs_data = f.read() + # Auto-detect SPIFFS configuration + config = _parse_spiffs_config(fs_data, fs_size) + # Create SPIFFS build configuration spiffs_build_config = SpiffsBuildConfig( - page_size=page_size, + page_size=config['page_size'], page_ix_len=2, - block_size=block_size, + block_size=config['block_size'], block_ix_len=2, - meta_len=meta_len, - obj_name_len=obj_name_len, + meta_len=config['meta_len'], + obj_name_len=config['obj_name_len'], obj_id_len=2, span_ix_len=2, packed=True, aligned=True, endianness='little', - use_magic=use_magic, - use_magic_len=use_magic_len, - aligned_obj_ix_tables=aligned_obj_ix_tables + use_magic=config['use_magic'], + use_magic_len=config['use_magic_len'], + aligned_obj_ix_tables=config['aligned_obj_ix_tables'] ) # Create SPIFFS filesystem and parse the image From 51975ec408a2095ebeea2b4a3b004d7d66481f1c Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 28 Dec 2025 16:07:08 +0100 Subject: [PATCH 2/5] no SCons for unpack --- builder/main.py | 123 ++++++++++++++++++------------------------------ 1 file changed, 47 insertions(+), 76 deletions(-) diff --git a/builder/main.py b/builder/main.py index 3573bf7f7..5985d3c35 100644 --- a/builder/main.py +++ b/builder/main.py @@ -1270,81 +1270,6 @@ def _download_partition_image(env, fs_type_filter=None): return fs_file, fs_start, fs_size, fs_subtype -def download_fs(target, source, env): - """ - Download filesystem from device and extract to directory. - Automatically detects filesystem type (LittleFS, SPIFFS, or FatFS). - Usage: pio run -e -t download_fs - - Args: - target: SCons target (unused) - source: SCons source (unused) - env: SCons environment object - """ - # Get unpack directory from board config or use default - unpack_dir = _get_unpack_dir(env) - - # Download partition image (accept any data partition) - fs_file, _fs_start, fs_size, fs_subtype = _download_partition_image(env, None) - - if fs_file is None: - return 1 - - # Detect filesystem type based on partition subtype and signature - fs_type = None - - if fs_subtype == 0x81: - # FAT partition - fs_type = "fatfs" - print(f"\nDetected filesystem type: FAT (partition subtype 0x{fs_subtype:02X})") - elif fs_subtype == 0x82: - # Could be SPIFFS or LittleFS - need to check signature - print(f"\nPartition subtype 0x{fs_subtype:02X} detected - checking filesystem signature...") - - try: - with open(fs_file, 'rb') as f: - # Read first 8KB to scan for filesystem signature - header = f.read(8192) - - # Check for "littlefs" ASCII string - if b'littlefs' in header: - fs_type = "littlefs" - else: - fs_type = "spiffs" - except Exception as e: - print(f"Error reading filesystem signature: {e}") - print(" Defaulting to SPIFFS") - fs_type = "spiffs" - elif fs_subtype == 0x83: - # LittleFS partition - fs_type = "littlefs" - print(f"\nDetected filesystem type: LittleFS (partition subtype 0x{fs_subtype:02X})") - else: - print(f"\nError: Unknown partition subtype 0x{fs_subtype:02X}") - print("Supported types: 0x81 (FAT), 0x82 (SPIFFS/LittleFS), 0x83 (LittleFS)") - return 1 - - # Remove old unpack directory - unpack_path = _prepare_unpack_dir(unpack_dir) - - # Call appropriate extraction function - print(f"\nExtracting {fs_type.upper()} filesystem...\n") - - try: - if fs_type == "littlefs": - return _extract_littlefs(fs_file, fs_size, unpack_path, unpack_dir) - elif fs_type == "spiffs": - return _extract_spiffs(fs_file, fs_size, unpack_path, unpack_dir) - elif fs_type == "fatfs": - return _extract_fatfs(fs_file, unpack_path, unpack_dir) - else: - print(f"Error: Unsupported filesystem type '{fs_type}'") - return 1 - except Exception as e: - print(f"Error: Failed to extract {fs_type.upper()} filesystem: {e}") - return 1 - - def _parse_littlefs_superblock(fs_data): """Parse LittleFS superblock to extract filesystem parameters. @@ -1710,6 +1635,49 @@ def _extract_fatfs(fs_file, unpack_path, unpack_dir): return 0 +def download_fs_action(): + """Download and extract filesystem from device.""" + # Get unpack directory + unpack_dir = _get_unpack_dir(env) + + # Download partition image + fs_file, _fs_start, fs_size, fs_subtype = _download_partition_image(env, None) + + if fs_file is None: + return 1 + + # Detect filesystem type + fs_type = None + if fs_subtype == 0x81: + fs_type = "fatfs" + elif fs_subtype == 0x82: + with open(fs_file, 'rb') as f: + header = f.read(8192) + fs_type = "littlefs" if b'littlefs' in header else "spiffs" + elif fs_subtype == 0x83: + fs_type = "littlefs" + else: + print(f"Error: Unknown partition subtype 0x{fs_subtype:02X}") + return 1 + + print(f"\nDetected filesystem: {fs_type.upper()}") + + # Prepare unpack directory + unpack_path = _prepare_unpack_dir(unpack_dir) + + # Extract filesystem + try: + if fs_type == "littlefs": + return _extract_littlefs(fs_file, fs_size, unpack_path, unpack_dir) + elif fs_type == "spiffs": + return _extract_spiffs(fs_file, fs_size, unpack_path, unpack_dir) + elif fs_type == "fatfs": + return _extract_fatfs(fs_file, unpack_path, unpack_dir) + except Exception as e: + print(f"Error: {e}") + return 1 + + # # Target: Build executable and linkable firmware or FS image @@ -1952,7 +1920,10 @@ def _extract_fatfs(fs_file, unpack_path, unpack_dir): env.AddPlatformTarget( "download_fs", None, - download_fs, + [ + env.VerboseAction(BeforeUpload, "Looking for upload port..."), + env.VerboseAction(download_fs_action, "Downloading and extracting filesystem") + ], "Download and extract filesystem from device", ) From 78f5ddb04294baf43dd58635233bd1c98e17774a Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 28 Dec 2025 16:32:10 +0100 Subject: [PATCH 3/5] fix dl extract to work without env --- builder/main.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/builder/main.py b/builder/main.py index 5985d3c35..82ec2f074 100644 --- a/builder/main.py +++ b/builder/main.py @@ -1182,7 +1182,6 @@ def _download_partition_image(env, fs_type_filter=None): esptool_cmd = [ uploader_path.strip('"'), - "--chip", mcu, "--port", upload_port, "--baud", str(download_speed), "--before", "default-reset", @@ -1245,7 +1244,6 @@ def _download_partition_image(env, fs_type_filter=None): esptool_cmd = [ uploader_path.strip('"'), - "--chip", mcu, "--port", upload_port, "--baud", str(download_speed), "--before", "default-reset", @@ -1635,9 +1633,9 @@ def _extract_fatfs(fs_file, unpack_path, unpack_dir): return 0 -def download_fs_action(): +def download_fs_action(target, source, env): """Download and extract filesystem from device.""" - # Get unpack directory + # Get unpack directory (use global env, not the parameter) unpack_dir = _get_unpack_dir(env) # Download partition image From 5acd77aa60ba9a360648a8c03590121f863733c3 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 28 Dec 2025 16:46:42 +0100 Subject: [PATCH 4/5] change defaults for LittleFS --- builder/main.py | 107 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 77 insertions(+), 30 deletions(-) diff --git a/builder/main.py b/builder/main.py index 82ec2f074..d92a11e99 100644 --- a/builder/main.py +++ b/builder/main.py @@ -1372,18 +1372,80 @@ def _extract_littlefs(fs_file, fs_size, unpack_path, unpack_dir): block_size = superblock['block_size'] block_count = superblock['block_count'] name_max = superblock['name_max'] - print("\nUsing auto-detected LittleFS parameters") + print("\nUsing auto-detected LittleFS parameters:") + print(f" Block size: {block_size} bytes") + print(f" Block count: {block_count}") + print(f" Max filename length: {name_max}") else: - # Fall back to defaults - print(f"\nWarning: Could not auto-detect LittleFS parameters, using defaults") - block_size = 0x1000 # 4KB default - block_count = fs_size // block_size - name_max = 64 - print(f" Block size: {block_size} bytes (default)") - print(f" Block count: {block_count} (calculated)") - print(f" Max filename length: {name_max} (default)") - - # Create LittleFS instance and mount the image + # Try common configurations + print("\nWarning: Could not auto-detect LittleFS parameters") + print("Trying common configurations...") + + common_configs = [ + # ESP-IDF defaults (most common) + {'block_size': 4096, 'name_max': 64}, + # Alternative configurations + {'block_size': 4096, 'name_max': 32}, + {'block_size': 8192, 'name_max': 64}, + {'block_size': 8192, 'name_max': 32}, + ] + + for config in common_configs: + block_size = config['block_size'] + block_count = fs_size // block_size + name_max = config['name_max'] + + try: + print(f" Trying: block_size={block_size}, name_max={name_max}...", end=" ") + fs = LittleFS( + block_size=block_size, + block_count=block_count, + name_max=name_max, + mount=False + ) + fs.context.buffer = bytearray(fs_data) + fs.mount() + print("Success!") + + # Extract files with this configuration + file_count = 0 + print("\nExtracted files:") + for root, dirs, files in fs.walk("/"): + if not root.endswith("/"): + root += "/" + + for dir_name in dirs: + src_path = root + dir_name + dst_path = unpack_path / src_path[1:] + dst_path.mkdir(parents=True, exist_ok=True) + print(f" [DIR] {src_path}") + + for file_name in files: + src_path = root + file_name + dst_path = unpack_path / src_path[1:] + dst_path.parent.mkdir(parents=True, exist_ok=True) + + with fs.open(src_path, "rb") as src: + file_data = src.read() + dst_path.write_bytes(file_data) + + print(f" [FILE] {src_path} ({len(file_data)} bytes)") + file_count += 1 + + fs.unmount() + print(f"\nSuccessfully extracted {file_count} file(s) to {unpack_dir}") + return 0 + + except Exception as e: + print(f"Failed: {e}") + continue + + # If all configs failed + print("\nError: Could not mount LittleFS with any common configuration") + print("The filesystem may be corrupted or use non-standard parameters") + return 1 + + # Create LittleFS instance and mount the image with detected parameters try: fs = LittleFS( block_size=block_size, @@ -1394,23 +1456,8 @@ def _extract_littlefs(fs_file, fs_size, unpack_path, unpack_dir): fs.context.buffer = bytearray(fs_data) fs.mount() except Exception as e: - # If mount fails with detected parameters, try defaults - if superblock: - print(f"\nWarning: Mount failed with detected parameters: {e}") - print("Retrying with default parameters...") - block_size = 0x1000 - block_count = fs_size // block_size - name_max = 64 - fs = LittleFS( - block_size=block_size, - block_count=block_count, - name_max=name_max, - mount=False - ) - fs.context.buffer = bytearray(fs_data) - fs.mount() - else: - raise + print(f"\nError: Mount failed with detected parameters: {e}") + return 1 # Extract all files file_count = 0 @@ -1422,14 +1469,14 @@ def _extract_littlefs(fs_file, fs_size, unpack_path, unpack_dir): # Create directories for dir_name in dirs: src_path = root + dir_name - dst_path = unpack_path / src_path[1:] # Remove leading '/' + dst_path = unpack_path / src_path[1:] dst_path.mkdir(parents=True, exist_ok=True) print(f" [DIR] {src_path}") # Extract files for file_name in files: src_path = root + file_name - dst_path = unpack_path / src_path[1:] # Remove leading '/' + dst_path = unpack_path / src_path[1:] dst_path.parent.mkdir(parents=True, exist_ok=True) with fs.open(src_path, "rb") as src: From 91b85ecf63d23b1300b85a4e2101613ba116bd20 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 28 Dec 2025 16:58:43 +0100 Subject: [PATCH 5/5] remove non working LittleFS auto parameter detection --- builder/main.py | 110 ++++++------------------------------------------ 1 file changed, 14 insertions(+), 96 deletions(-) diff --git a/builder/main.py b/builder/main.py index d92a11e99..042b61195 100644 --- a/builder/main.py +++ b/builder/main.py @@ -1359,105 +1359,23 @@ def _parse_littlefs_superblock(fs_data): def _extract_littlefs(fs_file, fs_size, unpack_path, unpack_dir): - """Extract LittleFS filesystem with auto-detected parameters.""" + """Extract LittleFS filesystem.""" # Read the downloaded filesystem image with open(fs_file, 'rb') as f: fs_data = f.read() - # Try to auto-detect filesystem parameters from superblock - superblock = _parse_littlefs_superblock(fs_data) - - if superblock: - # Use detected parameters - block_size = superblock['block_size'] - block_count = superblock['block_count'] - name_max = superblock['name_max'] - print("\nUsing auto-detected LittleFS parameters:") - print(f" Block size: {block_size} bytes") - print(f" Block count: {block_count}") - print(f" Max filename length: {name_max}") - else: - # Try common configurations - print("\nWarning: Could not auto-detect LittleFS parameters") - print("Trying common configurations...") - - common_configs = [ - # ESP-IDF defaults (most common) - {'block_size': 4096, 'name_max': 64}, - # Alternative configurations - {'block_size': 4096, 'name_max': 32}, - {'block_size': 8192, 'name_max': 64}, - {'block_size': 8192, 'name_max': 32}, - ] - - for config in common_configs: - block_size = config['block_size'] - block_count = fs_size // block_size - name_max = config['name_max'] - - try: - print(f" Trying: block_size={block_size}, name_max={name_max}...", end=" ") - fs = LittleFS( - block_size=block_size, - block_count=block_count, - name_max=name_max, - mount=False - ) - fs.context.buffer = bytearray(fs_data) - fs.mount() - print("Success!") - - # Extract files with this configuration - file_count = 0 - print("\nExtracted files:") - for root, dirs, files in fs.walk("/"): - if not root.endswith("/"): - root += "/" - - for dir_name in dirs: - src_path = root + dir_name - dst_path = unpack_path / src_path[1:] - dst_path.mkdir(parents=True, exist_ok=True) - print(f" [DIR] {src_path}") - - for file_name in files: - src_path = root + file_name - dst_path = unpack_path / src_path[1:] - dst_path.parent.mkdir(parents=True, exist_ok=True) - - with fs.open(src_path, "rb") as src: - file_data = src.read() - dst_path.write_bytes(file_data) - - print(f" [FILE] {src_path} ({len(file_data)} bytes)") - file_count += 1 - - fs.unmount() - print(f"\nSuccessfully extracted {file_count} file(s) to {unpack_dir}") - return 0 - - except Exception as e: - print(f"Failed: {e}") - continue - - # If all configs failed - print("\nError: Could not mount LittleFS with any common configuration") - print("The filesystem may be corrupted or use non-standard parameters") - return 1 + # Use ESP-IDF defaults + block_size = 0x1000 # 4KB + block_count = fs_size // block_size - # Create LittleFS instance and mount the image with detected parameters - try: - fs = LittleFS( - block_size=block_size, - block_count=block_count, - name_max=name_max, - mount=False - ) - fs.context.buffer = bytearray(fs_data) - fs.mount() - except Exception as e: - print(f"\nError: Mount failed with detected parameters: {e}") - return 1 + # Create LittleFS instance and mount the image + fs = LittleFS( + block_size=block_size, + block_count=block_count, + mount=False + ) + fs.context.buffer = bytearray(fs_data) + fs.mount() # Extract all files file_count = 0 @@ -1469,14 +1387,14 @@ def _extract_littlefs(fs_file, fs_size, unpack_path, unpack_dir): # Create directories for dir_name in dirs: src_path = root + dir_name - dst_path = unpack_path / src_path[1:] + dst_path = unpack_path / src_path[1:] # Remove leading '/' dst_path.mkdir(parents=True, exist_ok=True) print(f" [DIR] {src_path}") # Extract files for file_name in files: src_path = root + file_name - dst_path = unpack_path / src_path[1:] + dst_path = unpack_path / src_path[1:] # Remove leading '/' dst_path.parent.mkdir(parents=True, exist_ok=True) with fs.open(src_path, "rb") as src: