[build] Use lbuild queries to share code and data

repo.lb

@@ -34,7 +34,7 @@ except Exception as e:
     exit(1)
 
 import lbuild
-min_lbuild_version = "1.6.0"
+min_lbuild_version = "1.7.0"
 if StrictVersion(getattr(lbuild, "__version__", "0.1.0")) < StrictVersion(min_lbuild_version):
     print("modm requires at least lbuild v{}, please upgrade!\n"
           "    pip3 install -U lbuild".format(min_lbuild_version))

tools/build_script_generator/cmake/module.lb

@@ -12,9 +12,6 @@
 
 import os
 from os.path import join
-# import all common code
-with open(localpath("../common.py")) as common:
-    exec(common.read())
 
 def init(module):
     module.parent = "build"
@@ -43,13 +40,13 @@ def build(env):
 def post_build(env, buildlog):
     target = env["modm:target"]
     # get CPU information
-    subs = common_target(target)
+    subs = env.query("::device")
     # Extract all source code files
-    subs["sources"] = sorted(p for sources in common_source_files(env, buildlog).values() for p in sources)
+    subs["sources"] = sorted(p for sources in env.query("::source_files")(buildlog).values() for p in sources)
     # get memory information
-    subs["memories"] = common_memories(target)
+    subs["memories"] = env.query("::memories")
     # get memory information
-    subs["flags"] = common_metadata_flags(buildlog.repo_metadata, "modm")
+    subs["flags"] = env.query("::metadata_flags")(buildlog, "modm")
 
     # Add SCons specific data
     subs.update({

tools/build_script_generator/common.py

@@ -13,7 +13,13 @@
 import os
 from collections import defaultdict
 
-def common_source_files(env, buildlog):
+def common_source_files(buildlog):
+    """
+    Builds a list of files that need to be compiled per repository.
+
+    :param buildlog: the buildlog object available in the post_build step
+    :returns: a dictionary of sorted lists of filenames, keyed by repository.
+    """
     files_to_build = defaultdict(list)
 
     for operation in buildlog:
@@ -28,21 +34,43 @@ def common_source_files(env, buildlog):
         files_to_build[repo].sort()
     return files_to_build
 
-def common_target(target):
-    core = target.get_driver("core")["type"]
+def common_target(env):
+    """
+    Extracts common properties from a modm:target device:
+      - platform
+      - family
+      - partname
+      - core
+      - mcu (AVR only)
+
+    :returns: a dictionary of common properties.
+    """
+    device = env["modm:target"]
+    core = device.get_driver("core")["type"]
     core = core.replace("fd", "").replace("f", "")
-    mcu = target._properties.get("mcu", "")
+    mcu = device._properties.get("mcu", "")
     p = {
         "core": core,
         "mcu": mcu,
-        "platform": target.identifier["platform"],
-        "family": target.identifier["family"],
-        "partname": target.partname,
+        "platform": device.identifier["platform"],
+        "family": device.identifier["family"],
+        "partname": device.partname,
     }
     return p
 
-def common_memories(target):
-    core_driver = target.get_driver("core")
+def common_memories(env):
+    """
+    Extracts the memory map of the device.
+    A memory region is a dictionary containing:
+      - `name` of region
+      - `start` address of region
+      - `size` of region
+      - `access` of region
+
+    :returns: a list of memory regions.
+    """
+    device = env["modm:target"]
+    core_driver = device.get_driver("core")
     memories = []
     if "memory" in core_driver:
         memories.extend([
@@ -54,16 +82,34 @@ def common_memories(target):
         ])
     return memories
 
-def common_metadata_flags(metadata, repo=None):
+def common_metadata_flags(buildlog, repo):
+    """
+    Scans the metadata for module compile flags.
+    Converts them into SCons-compatible names and places them into a dictionary
+    of the form: flags[name][profile] = list(values).
+
+    :param buildlog: the post_build step buildlog
+    :param repo: the repository to search for
+    :returns: compile flags dictionary
+    """
     flags = defaultdict(lambda: defaultdict(list))
-    for key, values in metadata.items():
+    for key, values in buildlog.repo_metadata.items():
         if key.startswith("flags."):
             key = key.split(".")[1:]
-            if repo: values = values[repo];
+            values = values[repo];
             flags[key[0].upper()]["" if len(key) < 2 else key[1]].extend(list(values))
     return flags
 
 def common_file_flags(buildlog, filename):
+    """
+    Scans the metadata for file compile flags.
+    Converts them into SCons-compatible names and places them into a dictionary
+    of the form: flags[name][profile] = list(values).
+
+    :param buildlog: the post_build step buildlog
+    :param filename: the operation filename to search for
+    :returns: compile flags dictionary
+    """
     flags = defaultdict(lambda: defaultdict(list))
     for key, data in buildlog.operation_metadata.items():
         if key.startswith("flags."):

tools/build_script_generator/module.lb

@@ -55,6 +55,17 @@ def prepare(module, options):
             StringOption(name="openocd.cfg", default="",
                          description=descr_openocd_cfg))
 
+    module.add_query(
+        Query(name="source_files", function=common_source_files))
+    module.add_query(
+        EnvironmentQuery(name="device", factory=common_target))
+    module.add_query(
+        EnvironmentQuery(name="memories", factory=common_memories))
+    module.add_query(
+        Query(name="metadata_flags", function=common_metadata_flags))
+    module.add_query(
+        Query(name="file_flags", function=common_file_flags))
+
     return True
 
 

tools/build_script_generator/scons/module.lb

@@ -10,10 +10,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 # -----------------------------------------------------------------------------
 
-from os.path import join, relpath
-# import all common code
-with open(localpath("../common.py")) as common:
-    exec(common.read())
+from os.path import join, relpath, isdir
 
 def init(module):
     module.parent = "build"
@@ -73,12 +70,12 @@ def post_build(env, buildlog):
     is_unittest = env.has_module(":test")
     has_xpcc_generator = env.has_module(":communication:xpcc:generator")
     has_image_source = len(env["image.source"])
-    repositories = [p for p in buildlog.repositories if os.path.isdir(env.outpath(p, basepath="."))]
+    repositories = [p for p in buildlog.repositories if isdir(env.outpath(p, basepath="."))]
     repositories = sorted(repositories, key=lambda name: "0" if name == "modm" else name)
 
     target = env["modm:target"]
-    subs = common_target(target)
-    sources = common_source_files(env, buildlog)
+    subs = env.query("::device")
+    sources = env.query("::source_files")(buildlog)
 
     build_tools = [
         "settings_buildpath",
@@ -111,7 +108,7 @@ def post_build(env, buildlog):
         if "build/" in cache_dir:
             cache_dir = "{}build/cache".format(cache_dir.split("build/")[0])
     # get memory information
-    subs["memories"] = common_memories(target)
+    subs["memories"] = env.query("::memories")
     # Add SCons specific data
     subs.update({
         "metadata": buildlog.metadata,
@@ -134,6 +131,8 @@ def post_build(env, buildlog):
     # Set these substitutions for all templates
     env.substitutions = subs
 
+    common_file_flags = env.query("::file_flags")
+    common_metadata_flags = env.query("::metadata_flags")
     for repo in repositories:
         files = []
         for f in sources[repo]:
@@ -145,7 +144,7 @@ def post_build(env, buildlog):
 
         subs.update({
             "repo": repo,
-            "flags": common_metadata_flags(buildlog.repo_metadata, repo),
+            "flags": common_metadata_flags(buildlog, repo),
             "sources": files,
             "libraries": buildlog.repo_metadata["required_library"][repo],
             "library_paths": [env.reloutpath(p, repo) for p in buildlog.repo_metadata["required_library_path"][repo]],