Fix up usermod libArchive settings

pio-scripts/load_usermods.py

@@ -2,16 +2,21 @@
 import os.path
 from collections import deque
 from pathlib import Path   # For OS-agnostic path manipulation
+from click import secho
+from SCons.Script import Exit
+from platformio.builder.tools.piolib import LibBuilderBase
 from platformio.package.manager.library import LibraryPackageManager
 
 usermod_dir = Path(env["PROJECT_DIR"]) / "usermods"
-all_usermods = [f for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()]
 
+# "usermods" environment: expand list of usermods to everything in the folder
 if env['PIOENV'] == "usermods":
    # Add all usermods
+   all_usermods = [f for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()]
    env.GetProjectConfig().set(f"env:usermods", 'custom_usermods', " ".join([f.name for f in all_usermods]))
 
-def find_usermod(mod: str):
+# Utility functions
+def find_usermod(mod: str) -> Path:
   """Locate this library in the usermods folder.
      We do this to avoid needing to rename a bunch of folders;
      this could be removed later
@@ -28,6 +33,13 @@ def find_usermod(mod: str):
     return mp
   raise RuntimeError(f"Couldn't locate module {mod} in usermods directory!")
 
+def is_wled_module(dep: LibBuilderBase) -> bool:
+  """Returns true if the specified library is a wled module
+  """
+  return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
+
+## Script starts here
+# Process usermod option
 usermods = env.GetProjectOption("custom_usermods","")
 if usermods:
   # Inject usermods in to project lib_deps
@@ -82,13 +94,6 @@ def cached_add_includes(dep, dep_cache: set, includes: deque):
 
 # Our new wrapper
 def wrapped_ConfigureProjectLibBuilder(xenv):
-  # Update usermod properties
-  # Set libArchive before build actions are added
-  for um in (um for um in xenv.GetLibBuilders() if usermod_dir in Path(um.src_dir).parents):
-    build = um._manifest.get("build", {})
-    build["libArchive"] = False
-    um._manifest["build"] = build
-
   # Call the wrapped function
   result = old_ConfigureProjectLibBuilder.clone(xenv)()
 
@@ -102,12 +107,25 @@ def wrapped_ConfigureProjectLibBuilder(xenv):
   for dep in result.depbuilders:
      cached_add_includes(dep, processed_deps, extra_include_dirs)
 
-  for um in [dep for dep in result.depbuilders if usermod_dir in Path(dep.src_dir).parents]:
-    # Add the wled folder to the include path
-    um.env.PrependUnique(CPPPATH=wled_dir)
-    # Add WLED's own dependencies
-    for dir in extra_include_dirs:
-      um.env.PrependUnique(CPPPATH=dir)
+  broken_usermods = []
+  for dep in result.depbuilders:
+    if is_wled_module(dep):
+      # Add the wled folder to the include path
+      dep.env.PrependUnique(CPPPATH=str(wled_dir))
+      # Add WLED's own dependencies
+      for dir in extra_include_dirs:
+        dep.env.PrependUnique(CPPPATH=str(dir))
+      # Enforce that libArchive is not set; we must link them directly to the executable
+      if dep.lib_archive:
+        broken_usermods.append(dep)
+
+  if broken_usermods:
+    broken_usermods = [usermod.name for usermod in broken_usermods]
+    secho(
+      f"ERROR: libArchive=false is missing on usermod(s) {' '.join(broken_usermods)} -- modules will not compile in correctly",
+      fg="red",
+      err=True)    
+    Exit(1)
 
   return result
 

pio-scripts/validate_usermods.py

@@ -0,0 +1,92 @@
+import re
+import sys
+from pathlib import Path   # For OS-agnostic path manipulation
+from click import secho
+from SCons.Script import Action, Exit
+from platformio import util
+
+def read_lines(p: Path):
+    """ Read in the contents of a file for analysis """
+    with p.open("r", encoding="utf-8", errors="ignore") as f:
+        return f.readlines()
+
+def check_map_file_objects(map_file: list[str], usermod_dirs: list[str]) -> set[str]:
+    """ Checks that an object file from each usermod_dir appears in the linked output
+
+        Returns the (sub)set of usermod_dirs that are found in the output ELF
+    """
+    # Pattern to match symbols in object directories
+    # Join directories into alternation
+    usermod_dir_regex = "|".join([re.escape(dir) for dir in usermod_dirs])
+    # Matches nonzero address, any size, and any path in a matching directory
+    object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+/(" + usermod_dir_regex + r")/\S+\.o")
+
+    found = set()
+    for line in map_file:
+        matches = object_path_regex.findall(line)
+        for m in matches:
+            found.add(m)
+    return found
+
+def count_registered_usermods(map_file: list[str]) -> int:
+    """ Returns the number of usermod objects in the usermod list """
+    # Count the number of entries in the usermods table section
+    return len([x for x in map_file if ".dtors.tbl.usermods.1" in x])
+
+
+def validate_map_file(source, target, env):
+    """ Validate that all usermods appear in the output build """
+    build_dir = Path(env.subst("$BUILD_DIR"))
+    map_file_path = build_dir /  env.subst("${PROGNAME}.map")
+
+    if not map_file_path.exists():
+        secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True)
+        Exit(1)
+
+    # Load project settings
+    usermods = env.GetProjectOption("custom_usermods","").split()
+    libdeps = env.GetProjectOption("lib_deps", [])
+    lib_builders = env.GetLibBuilders()
+
+    secho(f"INFO: Expecting {len(usermods)} usermods: {', '.join(usermods)}")
+
+    # Map the usermods to libdeps; every usermod should have one
+    usermod_dirs = []
+    for mod in usermods:
+        modstr = f"{mod} = symlink://"
+        this_mod_libdeps = [libdep[len(modstr):] for libdep in libdeps if libdep.startswith(modstr)]
+        if not this_mod_libdeps:
+            secho(
+                f"ERROR: Usermod {mod} not found in build libdeps!",
+                fg="red",
+                err=True)
+            Exit(1)
+        # Save only the final folder name
+        usermod_dir = Path(this_mod_libdeps[0]).name
+        # Search lib_builders
+        this_mod_builders = [builder for builder in lib_builders if Path(builder.src_dir).name == usermod_dir]
+        if not this_mod_builders:
+            secho(
+                f"ERROR: Usermod {mod} not found in library builders!",
+                fg="red",
+                err=True)
+            Exit(1)
+        usermod_dirs.append(usermod_dir)
+
+    # Now parse the map file
+    map_file_contents = read_lines(map_file_path)
+    confirmed_usermods = check_map_file_objects(map_file_contents, usermod_dirs)
+    usermod_object_count = count_registered_usermods(map_file_contents)
+
+    secho(f"INFO: {len(usermod_dirs)}/{len(usermods)} libraries linked via custom_usermods, producing {usermod_object_count} usermod object entries")
+    missing_usermods = confirmed_usermods.difference(usermod_dirs)
+    if missing_usermods:
+        secho(
+            f"ERROR: No object files from {missing_usermods} found in linked output!",
+            fg="red",
+            err=True)
+        Exit(1)
+    return None
+
+Import("env")
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking map file...'))

platformio.ini

@@ -116,6 +116,7 @@ extra_scripts =
   pre:pio-scripts/user_config_copy.py
   pre:pio-scripts/load_usermods.py
   pre:pio-scripts/build_ui.py
+  post:pio-scripts/validate_usermods.py  ;; double-check the build output usermods
   ; post:pio-scripts/obj-dump.py  ;; convenience script to create a disassembly dump of the firmware (hardcore debugging)
 
 # ------------------------------------------------------------------------------

usermods/ADS1115_v2/library.json

@@ -1,5 +1,6 @@
 {
   "name": "ADS1115_v2",
+  "build": { "libArchive": false },
   "dependencies": {
     "Adafruit BusIO": "https://github.com/adafruit/Adafruit_BusIO#1.13.2",
     "Adafruit ADS1X15": "https://github.com/adafruit/Adafruit_ADS1X15#2.4.0"

usermods/AHT10_v2/library.json

@@ -1,5 +1,6 @@
 {
   "name": "AHT10_v2",
+  "build": { "libArchive": false },  
   "dependencies": {
     "enjoyneering/AHT10":"~1.1.0"
   }

usermods/Analog_Clock/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "Analog_Clock"
+  "name": "Analog_Clock",
+  "build": { "libArchive": false }
 }

usermods/Animated_Staircase/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "Animated_Staircase"
+  "name": "Animated_Staircase",
+  "build": { "libArchive": false }
 }

usermods/BH1750_v2/library.json

@@ -1,5 +1,6 @@
 {
   "name": "BH1750_v2",
+  "build": { "libArchive": false },
   "dependencies": {
     "claws/BH1750":"^1.2.0"
   }

usermods/BME280_v2/library.json

@@ -1,5 +1,6 @@
 {
   "name": "BME280_v2",
+  "build": { "libArchive": false },
   "dependencies": {
     "finitespace/BME280":"~3.0.0"
   }

usermods/BME68X_v2/library.json

@@ -1,5 +1,6 @@
 {
   "name": "BME68X",
+  "build": { "libArchive": false },
   "dependencies": {
     "boschsensortec/BSEC Software Library":"^1.8.1492"
   }

usermods/Battery/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "Battery"
+  "name": "Battery",
+  "build": { "libArchive": false }
 }

usermods/Cronixie/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "Cronixie"
+  "name": "Cronixie",
+  "build": { "libArchive": false }
 }

usermods/EXAMPLE/library.json

@@ -1,4 +1,5 @@
 {
   "name": "EXAMPLE",
+  "build": { "libArchive": false },
   "dependencies": {}
 }

usermods/EleksTube_IPS/library.json.disabled

@@ -1,5 +1,6 @@
 {
   "name:": "EleksTube_IPS",
+  "build": { "libArchive": false },
   "dependencies": {
     "TFT_eSPI" : "2.5.33"
   } 

usermods/INA226_v2/library.json

@@ -1,5 +1,6 @@
 {
   "name": "INA226_v2",
+  "build": { "libArchive": false },
   "dependencies": {
     "wollewald/INA226_WE":"~1.2.9"
   }

usermods/Internal_Temperature_v2/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "Internal_Temperature_v2"
+  "name": "Internal_Temperature_v2",
+  "build": { "libArchive": false }
 }

usermods/LD2410_v2/library.json

@@ -1,5 +1,6 @@
 {
   "name": "LD2410_v2",
+  "build": { "libArchive": false },
   "dependencies": {
     "ncmreynolds/ld2410":"^0.1.3"
   }

usermods/LDR_Dusk_Dawn_v2/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "LDR_Dusk_Dawn_v2"
+  "name": "LDR_Dusk_Dawn_v2",
+  "build": { "libArchive": false }
 }

usermods/MY9291/library.json

@@ -1,4 +1,5 @@
 {
   "name": "MY9291",
+  "build": { "libArchive": false },
   "platforms": ["espressif8266"]
 }

usermods/PIR_sensor_switch/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "PIR_sensor_switch"
+  "name": "PIR_sensor_switch",
+  "build": { "libArchive": false }
 }

usermods/PWM_fan/library.json

@@ -1,6 +1,7 @@
 {
   "name": "PWM_fan",
   "build": {
+    "libArchive": false,
     "extraScript": "setup_deps.py"
   }
 }

usermods/RTC/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "RTC"
+  "name": "RTC",
+  "build": { "libArchive": false }
 }

usermods/SN_Photoresistor/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "SN_Photoresistor"
+  "name": "SN_Photoresistor",
+  "build": { "libArchive": false }
 }

usermods/ST7789_display/library.json.disabled

@@ -1,3 +1,4 @@
 {
-  "name:": "ST7789_display"
+  "name:": "ST7789_display",
+  "build": { "libArchive": false }
 }

usermods/Si7021_MQTT_HA/library.json

@@ -1,5 +1,6 @@
 {
   "name": "Si7021_MQTT_HA",
+  "build": { "libArchive": false },
   "dependencies": {
     "finitespace/BME280":"3.0.0",
     "adafruit/Adafruit Si7021 Library" : "1.5.3"

usermods/TetrisAI_v2/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "TetrisAI_v2"
+  "name": "TetrisAI_v2",
+  "build": { "libArchive": false }
 }

usermods/boblight/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "boblight"
+  "name": "boblight",
+  "build": { "libArchive": false }
 }

usermods/buzzer/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "buzzer"
+  "name": "buzzer",
+  "build": { "libArchive": false }
 }

usermods/deep_sleep/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "deep_sleep"
+  "name": "deep_sleep",
+  "build": { "libArchive": false }
 }

usermods/multi_relay/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "multi_relay"
+  "name": "multi_relay",
+  "build": { "libArchive": false }
 }

usermods/pwm_outputs/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "pwm_outputs"
+  "name": "pwm_outputs",
+  "build": { "libArchive": false }
 }

usermods/sd_card/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "sd_card"
+  "name": "sd_card",
+  "build": { "libArchive": false }
 }

usermods/seven_segment_display/library.json

@@ -1,3 +1,4 @@
 {
-  "name": "seven_segment_display"
+  "name": "seven_segment_display",
+  "build": { "libArchive": false }
 }