Dynarray fixes

pio-scripts/dynarray.py

@@ -2,18 +2,85 @@
 # This is implemented as a pio post-script to ensure that we can
 # place our linker script at the correct point in the command arguments.
 Import("env")
+import shutil
 from pathlib import Path
 
-platform = env.get("PIOPLATFORM")
-script_file = Path(f"tools/dynarray_{platform}.ld")
-if script_file.is_file():
-    linker_script = f"-T{script_file}"
-    if platform == "espressif32":
-        # For ESP32, the script must be added at the right point in the list
-        linkflags = env.get("LINKFLAGS", [])
-        idx = linkflags.index("memory.ld")    
-        linkflags.insert(idx+1, linker_script)    
-        env.Replace(LINKFLAGS=linkflags)
+# Linker script fragment injected into the rodata output section of whichever
+# platform we're building for.  Placed just before the end-of-rodata marker so
+# that the dynarray entries land in flash rodata and are correctly sorted.
+DYNARRAY_INJECTION = (
+    "\n    /* dynarray: WLED dynamic module arrays */\n"
+    "    . = ALIGN(0x10);\n"
+    "    KEEP(*(SORT_BY_INIT_PRIORITY(.dynarray.*)))\n"
+    "    "
+)
+
+
+def inject_before_marker(path, marker):
+    """Patch a linker script file in-place, inserting DYNARRAY_INJECTION before marker."""
+    original = path.read_text()
+    marker_pos = original.find(marker)
+    if marker_pos < 0:
+        raise RuntimeError(
+            f"DYNARRAY injection marker not found in linker script: path={path}, marker={marker!r}"
+        )
+    patched = original[:marker_pos] + DYNARRAY_INJECTION + original[marker_pos:]
+    path.write_text(patched)
+
+
+if env.get("PIOPLATFORM") == "espressif32":
+    # Find sections.ld on the linker search path (LIBPATH).
+    sections_ld_path = None
+    for ld_dir in env.get("LIBPATH", []):
+        candidate = Path(str(ld_dir)) / "sections.ld"
+        if candidate.exists():
+            sections_ld_path = candidate
+            break
+
+    if sections_ld_path is not None:
+        # Inject inside the existing .flash.rodata output section, just before
+        # _rodata_end.  IDF v5 enforces zero gaps between adjacent output
+        # sections via ASSERT statements, so INSERT AFTER .flash.rodata would
+        # fail.  Injecting inside the section creates no new output section and
+        # leaves the ASSERTs satisfied.
+        build_dir = Path(env.subst("$BUILD_DIR"))
+        patched_path = build_dir / "dynarray_sections.ld"
+        shutil.copy(sections_ld_path, patched_path)
+        inject_before_marker(patched_path, "_rodata_end = ABSOLUTE(.);")
+
+        # Replace "sections.ld" in LINKFLAGS with an absolute path to our
+        # patched copy.  The flag may appear as a bare token, combined as
+        # "-Tsections.ld", or split across two tokens ("-T", "sections.ld").
+        patched_str = str(patched_path)
+        new_flags = []
+        skip_next = False
+        for flag in env.get("LINKFLAGS", []):
+            if skip_next:
+                new_flags.append(patched_str if flag == "sections.ld" else flag)
+                skip_next = False
+            elif flag == "-T":
+                new_flags.append(flag)
+                skip_next = True
+            else:
+                new_flags.append(flag.replace("sections.ld", patched_str))
+        env.Replace(LINKFLAGS=new_flags)
     else:
-        # For other platforms, put it in last
-        env.Append(LINKFLAGS=[linker_script])
+        # Assume sections.ld will be built (ESP-IDF format); add a post-action to patch it
+        # TODO: consider using ESP-IDF linker fragment (https://docs.espressif.com/projects/esp-idf/en/stable/esp32/api-guides/linker-script-generation.html)
+        # For now, patch after building
+        sections_ld = Path(env.subst("$BUILD_DIR")) / "sections.ld"
+        def patch_sections_ld(target, source, env):
+            inject_before_marker(sections_ld, "_rodata_end = ABSOLUTE(.);")
+        env.AddPostAction(str(sections_ld), patch_sections_ld)
+
+elif env.get("PIOPLATFORM") == "espressif8266":
+    # The ESP8266 framework preprocesses eagle.app.v6.common.ld.h into
+    # local.eagle.app.v6.common.ld in $BUILD_DIR/ld/ at build time.  Register
+    # a post-action on that generated file so the injection happens after
+    # C-preprocessing but before linking.
+    build_ld = Path(env.subst("$BUILD_DIR")) / "ld" / "local.eagle.app.v6.common.ld"
+
+    def patch_esp8266_ld(target, source, env):
+        inject_before_marker(build_ld, "_irom0_text_end = ABSOLUTE(.);")
+
+    env.AddPostAction(str(build_ld), patch_esp8266_ld)

pio-scripts/validate_modules.py

@@ -1,117 +1,169 @@
-import os
 import re
 import subprocess
-from pathlib import Path   # For OS-agnostic path manipulation
+from pathlib import Path
 from click import secho
 from SCons.Script import Action, Exit
 Import("env")
 
+_ATTR = re.compile(r'\bDW_AT_(name|comp_dir)\b')
+
 
 def read_lines(p: Path):
     """ Read in the contents of a file for analysis """
     with p.open("r", encoding="utf-8", errors="ignore") as f:
         return f.readlines()
 
 
-def _get_nm_path(env) -> str:
-    """ Derive the nm tool path from the build environment """
-    if "NM" in env:
-        return env.subst("$NM")
-    # Derive from the C compiler: xtensa-esp32-elf-gcc → xtensa-esp32-elf-nm
-    cc = env.subst("$CC")
-    nm = re.sub(r'(gcc|g\+\+)$', 'nm', os.path.basename(cc))
-    return os.path.join(os.path.dirname(cc), nm)
+def _get_readelf_path(env) -> str:
+    """ Derive the readelf tool path from the build environment """
+    # Derive from the C compiler: xtensa-esp32-elf-gcc → xtensa-esp32-elf-readelf
+    cc = Path(env.subst("$CC"))
+    return str(cc.with_name(re.sub(r'(gcc|g\+\+)$', 'readelf', cc.name)))
 
 
 def check_elf_modules(elf_path: Path, env, module_lib_builders) -> set[str]:
-    """ Check which modules have at least one defined symbol placed in the ELF.
+    """ Check which modules have at least one compilation unit in the ELF.
 
         The map file is not a reliable source for this: with LTO, original object
         file paths are replaced by temporary ltrans.o partitions in all output
         sections, making per-module attribution impossible from the map alone.
-        Instead we invoke nm --defined-only -l on the ELF, which uses DWARF debug
-        info to attribute each placed symbol to its original source file.
-
-        Requires usermod libraries to be compiled with -g so that DWARF sections
-        are present in the ELF.  load_usermods.py injects -g for all WLED modules
-        via dep.env.AppendUnique(CCFLAGS=["-g"]).
+        Instead we invoke readelf --debug-dump=info --dwarf-depth=1 on the ELF,
+        which reads only the top-level compilation-unit DIEs from .debug_info.
+        Each CU corresponds to one source file; matching DW_AT_comp_dir +
+        DW_AT_name against the module src_dirs is sufficient to confirm a module
+        was compiled into the ELF.  The output volume is proportional to the
+        number of source files, not the number of symbols.
 
         Returns the set of build_dir basenames for confirmed modules.
     """
-    nm_path = _get_nm_path(env)
+    readelf_path = _get_readelf_path(env)
     try:
         result = subprocess.run(
-            [nm_path, "--defined-only", "-l", str(elf_path)],
+            [readelf_path, "--debug-dump=info", "--dwarf-depth=1", str(elf_path)],
             capture_output=True, text=True, errors="ignore", timeout=120,
         )
-        nm_output = result.stdout
+        output = result.stdout
+        if result.returncode != 0 or result.stderr.strip():
+            secho(f"WARNING: readelf exited {result.returncode}: {result.stderr.strip()}", fg="yellow", err=True)
     except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as e:
-        secho(f"WARNING: nm failed ({e}); skipping per-module validation", fg="yellow", err=True)
+        secho(f"WARNING: readelf failed ({e}); skipping per-module validation", fg="yellow", err=True)
         return {Path(b.build_dir).name for b in module_lib_builders}  # conservative pass
 
-    # Match placed symbols against builders as we parse nm output, exiting early
-    # once all builders are accounted for.
-    # nm --defined-only still includes debugging symbols (type 'N') such as the
-    # per-CU markers GCC emits in .debug_info (e.g. "usermod_example_cpp_6734d48d").
-    # These live at address 0x00000000 in their debug section — not in any load
-    # segment — so filtering them out leaves only genuinely placed symbols.
-    # nm -l appends a tab-separated "file:lineno" location to each symbol line.
     remaining = {Path(str(b.src_dir)): Path(b.build_dir).name for b in module_lib_builders}
     found = set()
-
-    for line in nm_output.splitlines():
-        if not remaining:
-            break  # all builders matched
-        addr, _, _ = line.partition(' ')
-        if not addr.lstrip('0'):
-            continue  # zero address — skip debug-section marker
-        if '\t' not in line:
-            continue
-        loc = line.rsplit('\t', 1)[1]
-        # Strip trailing :lineno  (e.g. "/path/to/foo.cpp:42" → "/path/to/foo.cpp")
-        src_path = Path(loc.rsplit(':', 1)[0])
-        # Path.is_relative_to() handles OS-specific separators correctly without
-        # any regex, avoiding Windows path escaping issues.
+    project_dir = Path(env.subst("$PROJECT_DIR"))
+
+    def _flush_cu(comp_dir: str | None, name: str | None) -> None:
+        """Match one completed CU against remaining builders."""
+        if not name or not remaining:
+            return
+        p = Path(name)
+        src_path = (Path(comp_dir) / p) if (comp_dir and not p.is_absolute()) else p
+        # In arduino+espidf dual-framework builds the IDF toolchain sets DW_AT_comp_dir
+        # to the virtual path "/IDF_PROJECT" rather than the real project root, so
+        # src_path won't match.  Pre-compute a fallback using $PROJECT_DIR and check
+        # both candidates in a single pass.
+        use_fallback = not p.is_absolute() and comp_dir and Path(comp_dir) != project_dir
+        src_path_real = project_dir / p if use_fallback else None
         for src_dir in list(remaining):
-            if src_path.is_relative_to(src_dir):
+            if src_path.is_relative_to(src_dir) or (src_path_real and src_path_real.is_relative_to(src_dir)):
                 found.add(remaining.pop(src_dir))
-                break
+                return
+
+    # readelf emits one DW_TAG_compile_unit DIE per source file.  Attributes
+    # of interest:
+    #   DW_AT_name     — source file (absolute, or relative to comp_dir)
+    #   DW_AT_comp_dir — compile working directory
+    # Both appear as either a direct string or an indirect string:
+    #   DW_AT_name     : foo.cpp
+    #   DW_AT_name     : (indirect string, offset: 0x…): foo.cpp
+    # Taking the portion after the *last* ": " on the line handles both forms.
+
+    comp_dir = name = None
+    for line in output.splitlines():
+        if 'Compilation Unit @' in line:
+            _flush_cu(comp_dir, name)
+            comp_dir = name = None
+            continue
+        if not remaining:
+            break  # all builders matched
+        m = _ATTR.search(line)
+        if m:
+            _, _, val = line.rpartition(': ')
+            val = val.strip()
+            if m.group(1) == 'name':
+                name = val
+            else:
+                comp_dir = val
+    _flush_cu(comp_dir, name)  # flush the last CU
 
     return found
 
 
-DYNARRAY_SECTION = ".dtors" if env.get("PIOPLATFORM") == "espressif8266" else ".dynarray"
-USERMODS_SECTION = f"{DYNARRAY_SECTION}.usermods.1"
-
 def count_usermod_objects(map_file: list[str]) -> int:
-    """ Returns the number of usermod objects in the usermod list """
-    # Count the number of entries in the usermods table section
-    return len([x for x in map_file if USERMODS_SECTION in x])
+    """ Returns the number of usermod objects in the usermod list.
+
+    Computes the count from the address span between the .dynarray.usermods.0
+    and .dynarray.usermods.99999 sentinel sections.  This mirrors the
+    DYNARRAY_LENGTH macro and is reliable under LTO, where all entries are
+    merged into a single ltrans partition so counting section occurrences
+    always yields 1 regardless of the true count.
+    """
+    ENTRY_SIZE = 4  # sizeof(Usermod*) on 32-bit targets
+    addr_begin = None
+    addr_end = None
+
+    for i, line in enumerate(map_file):
+        stripped = line.strip()
+        if stripped == '.dynarray.usermods.0':
+            if i + 1 < len(map_file):
+                m = re.search(r'0x([0-9a-fA-F]+)', map_file[i + 1])
+                if m:
+                    addr_begin = int(m.group(1), 16)
+        elif stripped == '.dynarray.usermods.99999':
+            if i + 1 < len(map_file):
+                m = re.search(r'0x([0-9a-fA-F]+)', map_file[i + 1])
+                if m:
+                    addr_end = int(m.group(1), 16)
+        if addr_begin is not None and addr_end is not None:
+            break
+
+    if addr_begin is None or addr_end is None:
+        return 0
+    return (addr_end - addr_begin) // ENTRY_SIZE
 
 
 def validate_map_file(source, target, env):
     """ Validate that all modules appear in the output build """
     build_dir = Path(env.subst("$BUILD_DIR"))
-    map_file_path = build_dir /  env.subst("${PROGNAME}.map")
+    map_file_path = build_dir / env.subst("${PROGNAME}.map")
 
     if not map_file_path.exists():
         secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True)
         Exit(1)
 
     # Identify the WLED module builders, set by load_usermods.py
-    module_lib_builders = env['WLED_MODULES']
+    module_lib_builders = env.get('WLED_MODULES')
+    if module_lib_builders is None:
+        secho("ERROR: WLED_MODULES not set — ensure load_usermods.py is run as a pre: script", fg="red", err=True)
+        Exit(1)
 
     # Extract the values we care about
     modules = {Path(builder.build_dir).name: builder.name for builder in module_lib_builders}
-    secho(f"INFO: {len(modules)} libraries linked as WLED optional/user modules")
+    secho(f"INFO: {len(modules)} libraries included as WLED optional/user modules")
 
     # Now parse the map file
     map_file_contents = read_lines(map_file_path)
     usermod_object_count = count_usermod_objects(map_file_contents)
-    secho(f"INFO: {usermod_object_count} usermod object entries")
+    secho(f"INFO: {usermod_object_count} usermod object entries found")
 
     elf_path = build_dir / env.subst("${PROGNAME}.elf")
+
     confirmed_modules = check_elf_modules(elf_path, env, module_lib_builders)
+    if confirmed_modules:
+        secho(f"INFO: Code from usermod libraries found in binary: {', '.join(confirmed_modules)}")
+    # else - if there's no usermods found, don't generate a message.  If we're legitimately missing all entries, the error report on the
+    # next line will trip; and if the usermod set is expected to be empty, then there's no need for yet another null message.
 
     missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules]
     if missing_modules:
@@ -120,7 +172,6 @@ def validate_map_file(source, target, env):
             fg="red",
             err=True)
         Exit(1)
-    return None
 
 env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
 env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))

wled00/dynarray.h

@@ -20,15 +20,4 @@ Macros for generating a "dynamic array", a static array of objects declared in d
 #define DYNARRAY_END(array_name) array_name##_end
 #define DYNARRAY_LENGTH(array_name) (&DYNARRAY_END(array_name)[0] - &DYNARRAY_BEGIN(array_name)[0])
 
-#ifdef ESP8266
-// ESP8266 linker script cannot be extended with a unique section for dynamic arrays.
-// We instead pack them in the ".dtors" section, as it's sorted and uploaded to the flash
-// (but will never be used in the embedded system)
-#define DYNARRAY_SECTION ".dtors"
-
-#else /* ESP8266 */
-
-// Use a unique named section; the linker script must be extended to ensure it's correctly placed.
 #define DYNARRAY_SECTION ".dynarray"
-
-#endif