[protobuf] SCons integration of nanopb

.github/workflows/linux.yml

@@ -185,7 +185,7 @@ jobs:
         uses: actions/checkout@v2
       - name: Checkout code and update modm tools
         run: |
-          (git submodule sync && git submodule update --init --jobs 8) & pip3 install --upgrade --upgrade-strategy=eager modm & wait
+          (git submodule sync && git submodule update --init --jobs 8) & pip3 install --upgrade --upgrade-strategy=eager modm protobuf==3.20.1 grpcio-tools & wait
       - name: Examples STM32F4 Without Discovery Board
         if: always()
         run: |

.gitmodules

@@ -40,3 +40,6 @@
 [submodule "ext/eyalroz/printf"]
 	path = ext/eyalroz/printf
 	url = https://github.com/modm-ext/printf-partial.git
+[submodule "ext/nanopb/nanopb"]
+	path = ext/nanopb/nanopb
+	url = https://github.com/modm-ext/nanopb-partial.git

examples/nucleo_f429zi/nanopb/main.cpp

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2022, Lucas Moesch
+ * Copyright (c) 2018, Petteri Aimonen <jpa at nanopb.mail.kapsi.fi>
+ *
+ * This file is part of the modm project.
+ *
+ * This file originated from the nanopb project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <pb_encode.h>
+#include <pb_decode.h>
+#include "protocol/example.pb.hpp"
+
+int main()
+{
+    /* This is the buffer where we will store our message. */
+    uint8_t buffer[128];
+    size_t message_length;
+    bool status;
+
+    {
+        /* Allocate space on the stack to store the message data.
+         *
+         * Nanopb generates simple struct definitions for all the messages.
+         * - check out the contents of simple.pb.h!
+         * It is a good idea to always initialize your structures
+         * so that you do not have garbage data from RAM in there.
+         */
+        SimpleMessage message = SimpleMessage_init_zero;
+
+        /* Create a stream that will write to our buffer. */
+        pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+
+        /* Fill in the lucky number */
+        message.lucky_number = 13;
+
+        /* Now we are ready to encode the message! */
+        status = pb_encode(&stream, SimpleMessage_fields, &message);
+        message_length = stream.bytes_written;
+    }
+
+    {
+        /* Now we could transmit the message over network, store it in a file or
+         * wrap it to a pigeon's leg.
+         */
+
+        /* But because we are lazy, we will just decode it immediately. */
+
+        /* Allocate space for the decoded message. */
+        SimpleMessage message = SimpleMessage_init_zero;
+
+        /* Create a stream that reads from the buffer. */
+        pb_istream_t stream = pb_istream_from_buffer(buffer, message_length);
+
+        /* Now we are ready to decode the message. */
+        status = pb_decode(&stream, SimpleMessage_fields, &message);
+    }
+
+    while (true) {
+
+    }
+
+    return 0;
+}

examples/nucleo_f429zi/nanopb/project.xml

@@ -0,0 +1,11 @@
+<library>
+  <extends>modm:nucleo-f429zi</extends>
+  <options>
+    <option name="modm:build:build.path">../../../build/nucleo_f429zi/nanopb</option>
+    <option name="modm:nanopb:protofile.path">./protocol/example.proto</option>
+  </options>
+  <modules>
+    <module>modm:build:scons</module>
+    <module>modm:nanopb</module>
+  </modules>
+</library>

examples/nucleo_f429zi/nanopb/protocol/example.proto

@@ -0,0 +1,16 @@
+// -*- coding: utf-8 -*-
+//
+// Copyright (c) 2022, Lucas Moesch
+//
+// This file is part of the modm project.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+// -----------------------------------------------------------------------------
+syntax = "proto3";
+
+message SimpleMessage {
+    int32 lucky_number = 1;
+}
+

ext/nanopb/nanopb.lb

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2021, Lucas Moesch
+#
+# This file is part of the modm project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# -----------------------------------------------------------------------------
+import os
+import subprocess
+import tempfile
+
+from pathlib import Path
+
+def init(module):
+    module.name = ":nanopb"
+    module.description = """
+# Nanopb - Protocol Buffers for Embedded Systems
+Nanopb is a small code-size Protocol Buffers (protobuf) implementation in ansi C. It is especially suitable
+for use in microcontrollers, but fits any memory restricted system.
+
+- https://github.com/nanopb/nanopb
+
+Protofiles can be added to the build system either by using the `modm:nanopb:protofile.path` option or by extending
+`proto_sources` in the SConstruct file.
+
+"""
+
+def prepare(module, options):
+    module.add_option(PathOption(name="protofile.path", description="Path to a protofile.", empty_ok=True, absolute=True, default=""))
+
+    return True
+
+def build(env):
+    env.collect(":build:path.include", "modm/ext/nanopb")
+    env.outbasepath = "modm/ext/nanopb"
+
+    env.copy("nanopb/pb_common.c", dest="pb_common.c")
+    env.copy("nanopb/pb_common.h", dest="pb_common.h")
+    env.copy("nanopb/pb_decode.c", dest="pb_decode.c")
+    env.copy("nanopb/pb_decode.h", dest="pb_decode.h")
+    env.copy("nanopb/pb_encode.c", dest="pb_encode.c")
+    env.copy("nanopb/pb_encode.h", dest="pb_encode.h")
+    env.copy("nanopb/pb.h", dest="pb.h")

tools/build_script_generator/scons/module.lb

@@ -108,6 +108,9 @@ def build(env):
     elif device["core"].startswith("avr"):
         tools.update({"size", "avrdude"})
 
+    if env.has_module(":nanopb"):
+        tools.add("nanopb")
+
     env.collect("path.tools", *toolpaths)
     env.collect("tools", *tools)
 
@@ -117,9 +120,15 @@ def build(env):
         path = "site_tools/{}.py".format(tool)
         if exists(localpath(path)):
             env.copy(path)
+
     # Copy support files
     env.copy("site_tools/qtcreator/")
 
+    if env.has_module(":nanopb"):
+        env.copy("site_tools/nanopb_builder/")
+        env.copy(repopath("ext/nanopb/nanopb/generator"), dest="site_tools/nanopb_builder/generator")
+        env.copy(repopath("ext/nanopb/nanopb/tests/site_scons/site_tools/nanopb.py"), dest="site_tools/nanopb_builder/nanopb.py")
+
     # Generate the env.BuildTarget tool
     linkerscript = env.get(":platform:cortex-m:linkerscript.override")
     linkerscript = env.relcwdoutpath(linkerscript) if linkerscript \
@@ -139,6 +148,7 @@ def post_build(env):
     is_unittest = len(env["::unittest.source"])
     has_xpcc_generator = env.has_module(":communication:xpcc:generator")
     has_image_source = len(env["::image.source"])
+    has_nanopb = env.has_module(":nanopb")
     repositories = [p for p in env.buildlog.repositories if isdir(env.real_outpath(p, basepath="."))]
     repositories.sort(key=lambda name: "0" if name == "modm" else name)
     generated_paths = [join(env.relcwdoutpath(""), r) for r in repositories]
@@ -161,7 +171,6 @@ def post_build(env):
         "artifact_path": env.relcwdoutpath(env["path.artifact"]),
         "generated_paths": generated_paths,
         "is_unittest": is_unittest,
-
         "has_image_source": has_image_source,
         "has_xpcc_generator": has_xpcc_generator,
     })
@@ -220,6 +229,16 @@ def post_build(env):
             "tools": tools,
             "is_modm": repo == "modm",
         })
+
+        # Add protofiles
+        subs['build_proto'] = has_nanopb
+        if has_nanopb:
+            protofile = env.get("::protofile.path")
+            if protofile:
+                subs["proto_source"] = [ "%s" % env.relcwdoutpath(protofile) ]
+            else:
+                subs["proto_source"] = []
+
         # Generate library SConscript
         env.outbasepath = repo
         env.template("resources/SConscript.in", "SConscript",

tools/build_script_generator/scons/module.md

@@ -599,6 +599,20 @@ are supported, this is only meant for using the IDE as an editor.
 
 !!! warning "Consider this an unstable feature"
 
+#### scons nanopb
+
+```
+scons nanopb
+```
+
+Generates *.pb.cpp and *.pb.hpp files from the *.proto files listed in `proto_sources`
+in the `SConstruct` file within their source directories.
+
+```
+$ scons nanopb
+"\project\ext\modm\scons\site_tools\nanopb_builder\generator\protoc" "--plugin=protoc-gen-nanopb=\project\ext\modm\scons\site_tools\nanopb_builder\protoc-gen-nanopb" -I".\protocol" -I"." -I"modm\scons\site_tools\enerator\proto" --nanopb_out=".\protocol" --nanopb_opt=--source-extension=.cpp,--header-extension=.hpp "example.proto"
+```
+
 
 ## XPCC Generator Tool
 

tools/build_script_generator/scons/resources/SConstruct.in

@@ -98,4 +98,10 @@ sources += env.FindSourceFiles(".", ignorePaths=ignored)
 #       <option name="modm:build:scons:include_sconstruct">False</option>
 #   7. Anyone using your project now also benefits from your environment changes.
 
+%% if build_proto
+# Get nanopb protofile sources and build them
+proto_sources = {{ proto_source }}
+env.BuildNanopbProto(proto_sources, sources)
+%% endif
+
 env.BuildTarget(sources)

tools/build_script_generator/scons/site_tools/nanopb.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2022, Lucas Moesch
+#
+# This file is part of the modm project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# -----------------------------------------------------------------------------
+
+import SCons.Action
+import SCons.Builder
+import SCons.Util
+from SCons.Script import Dir, File
+
+import os.path
+import platform
+import sys
+
+import nanopb_builder
+
+# -----------------------------------------------------------------------------
+# Support for nanopb protobuf compilation.
+# This suits as a wrapper for nanopb's own SCons build integration, which is
+# integrated via the nanopb_builder module. The SCons integration can be found
+# in modm/ext/nanopb/nanopb/tests/site_scons/site_tools/nanopb.py.
+# -----------------------------------------------------------------------------
+
+def relpath(path, start):
+    os_relpath = os.path.relpath(path, start)
+
+    if os_relpath[0] == ".":
+        return os_relpath
+    else:
+        return os.path.join(".", os_relpath)
+
+def inject_dependencies(target, env):
+    if not any(str(t) == str(target) for t in env["CPP_SOURCES"]):
+        env["CPP_SOURCES"].append(target)
+        env.Depends(env["CPP_SOURCES"][-1], "./modm/ext/nanopb/pb.h")
+
+def _nanopb_proto_actions(source, target, env, for_signature):
+    if source:
+        esc = env['ESCAPE']
+        actionlist = []
+
+        for protofile in source:
+            # Make protoc build inside the SConscript directory
+            srcfile = esc(os.path.basename(str(protofile)))
+            srcdir = esc(relpath(os.path.dirname(str(protofile)), "."))
+            include_dirs = "-I" + srcdir
+
+            for d in env['PROTOCPATH']:
+                d = env.GetBuildPath(d)
+                if not os.path.isabs(d): d = os.path.relpath(d, ".")
+                include_dirs += ' -I' + esc(d)
+
+            # when generating .pb.cpp sources, instead of pb.h generate .pb.hpp headers
+            source_extension = os.path.splitext(str(target[0]))[1]
+            header_extension = '.h' + source_extension[2:]
+            nanopb_flags = env['NANOPBFLAGS']
+            if nanopb_flags:
+                nanopb_flags = '--source-extension=%s,--header-extension=%s,%s' % (source_extension, header_extension, nanopb_flags)
+            else:
+                nanopb_flags = '--source-extension=%s,--header-extension=%s' % (source_extension, header_extension)
+
+            actionlist.append(SCons.Action.CommandAction('$PROTOC $PROTOCFLAGS %s --nanopb_out=%s --nanopb_opt=%s %s' % (include_dirs, srcdir, nanopb_flags, srcfile)))
+        return SCons.Action.ListAction(actionlist)
+    else:
+        return SCons.Action.CommandAction("")
+
+def _nanopb_proto_emitter(target, source, env):
+    if source:
+        for protofile in source:
+            basename = os.path.splitext(str(protofile))[0]
+            source_extension = os.path.splitext(str(target[0]))[1]
+            header_extension = '.h' + source_extension[2:]
+            cpp_target = basename + '.pb' + source_extension
+
+            if not any(str(t) == cpp_target for t in target):
+                target.append(cpp_target)
+                inject_dependencies(cpp_target, env)
+            else:
+                inject_dependencies(target[0], env)
+
+            target.append(basename + '.pb' + header_extension)
+
+            if os.path.exists(basename + '.options'):
+                source.append(basename + '.options')
+
+        return target, source
+    else:
+        return [], []
+
+_nanopb_proto_cpp_builder = SCons.Builder.Builder(
+    generator = _nanopb_proto_actions,
+    suffix = '.pb.cpp',
+    src_suffix = '.proto',
+    emitter = _nanopb_proto_emitter)
+
+def build_proto(env, sources, cpp_sources):
+    env["CPP_SOURCES"] = cpp_sources
+    env.Alias("nanopb", env.NanopbProtoCpp(sources))
+    env.Default("nanopb")
+
+def generate(env, **kw):
+    env['NANOPB'] = os.path.abspath(os.path.join("./", "scons/site_tools/nanopb_builder"))
+    env['PROTOC'] = nanopb_builder._detect_protoc(env)
+    env['PROTOCFLAGS'] = nanopb_builder._detect_protocflags(env)
+
+    env.SetDefault(NANOPBFLAGS = '')
+    env.SetDefault(PROTOCPATH = [".", os.path.join(env['NANOPB'], 'generator', 'proto')])
+
+    env['BUILDERS']['NanopbProtoCpp'] = _nanopb_proto_cpp_builder
+
+    env.AddMethod(build_proto, "BuildNanopbProto")
+
+def exists(env):
+	return True

tools/build_script_generator/scons/site_tools/nanopb_builder/__init__.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2022, Lucas Moesch
+#
+# This file is part of the modm project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# -----------------------------------------------------------------------------
+
+from .nanopb import _nanopb_proto_actions, _detect_protocflags, _detect_protoc