[protobuf] SCons integration of nanopb

Co-authored-by: Niklas Hauser <[email protected]>

.github/workflows/linux.yml

@@ -185,7 +185,7 @@ jobs:
         uses: actions/checkout@v2
       - name: Checkout code and update modm tools
         run: |
-          (git submodule sync && git submodule update --init --jobs 8) & pip3 install --upgrade --upgrade-strategy=eager modm & wait
+          (git submodule sync && git submodule update --init --jobs 8) & pip3 install --upgrade --upgrade-strategy=eager modm protobuf==3.20.1 grpcio-tools & wait
       - name: Examples STM32F4 Without Discovery Board
         if: always()
         run: |

.gitmodules

@@ -40,3 +40,6 @@
 [submodule "ext/eyalroz/printf"]
 	path = ext/eyalroz/printf
 	url = https://github.com/modm-ext/printf-partial.git
+[submodule "ext/nanopb/nanopb"]
+	path = ext/nanopb/nanopb
+	url = https://github.com/modm-ext/nanopb-partial.git

examples/nucleo_f429zi/nanopb/main.cpp

@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2022, Lucas Moesch
+ * Copyright (c) 2022, Niklas Hauser
+ *
+ * This file is part of the modm project.
+ *
+ * This file originated from the nanopb project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <modm/board.hpp>
+#include <pb_encode.h>
+#include <pb_decode.h>
+#include <simple.pb.hpp>
+#include <complex.pb.hpp>
+
+int main()
+{
+    Board::initialize();
+
+    /* This is the buffer where we will store our message. */
+    uint8_t buffer[128];
+    size_t message_length;
+    bool status;
+
+    {
+        /* Allocate space on the stack to store the message data.
+         *
+         * Nanopb generates simple struct definitions for all the messages.
+         * - check out the contents of simple.pb.h!
+         * It is a good idea to always initialize your structures
+         * so that you do not have garbage data from RAM in there.
+         */
+        SimpleMessage message = SimpleMessage_init_zero;
+
+        /* Create a stream that will write to our buffer. */
+        pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+
+        /* Fill in the lucky number */
+        message.lucky_number = 42;
+
+        /* Now we are ready to encode the message! */
+        status = pb_encode(&stream, SimpleMessage_fields, &message);
+        modm_assert(status, "pb.enc", "Encoding SimpleMessage failed!");
+        message_length = stream.bytes_written;
+        modm_assert(message_length, "pb.len", "Empty SimpleMessage buffer!");
+    }
+    {
+        /* Now we could transmit the message over network, store it in a file or
+         * wrap it to a pigeon's leg.
+         */
+
+        /* But because we are lazy, we will just decode it immediately. */
+
+        /* Allocate space for the decoded message. */
+        SimpleMessage message = SimpleMessage_init_zero;
+
+        /* Create a stream that reads from the buffer. */
+        pb_istream_t stream = pb_istream_from_buffer(buffer, message_length);
+
+        /* Now we are ready to decode the message. */
+        status = pb_decode(&stream, SimpleMessage_fields, &message);
+        modm_assert(status, "pb.dec", "Decoding SimpleMessage failed!");
+        modm_assert(message.lucky_number == 42, "lucky_number", "Incorrect SimpleMessage values!");
+    }
+
+    {
+        /* Allocate space on the stack to store the message data.
+         *
+         * Nanopb generates simple struct definitions for all the messages.
+         * - check out the contents of simple.pb.h!
+         * It is a good idea to always initialize your structures
+         * so that you do not have garbage data from RAM in there.
+         */
+        ComplexMessage message = ComplexMessage_init_zero;
+
+        /* Create a stream that will write to our buffer. */
+        pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+
+        /* Fill in the unlucky number */
+        std::strcpy(message.query, "Hello World");
+        message.unlucky_number = 13;
+        message.toggle = true;
+        message.value = 4.00012f;
+
+        /* Now we are ready to encode the message! */
+        status = pb_encode(&stream, ComplexMessage_fields, &message);
+        modm_assert(status, "pb.enc", "Encoding ComplexMessage failed!");
+        message_length = stream.bytes_written;
+        modm_assert(message_length, "pb.len", "Empty ComplexMessage buffer!");
+    }
+    {
+        /* Now we could transmit the message over network, store it in a file or
+         * wrap it to a pigeon's leg.
+         */
+
+        /* But because we are lazy, we will just decode it immediately. */
+
+        /* Allocate space for the decoded message. */
+        ComplexMessage message = ComplexMessage_init_zero;
+
+        /* Create a stream that reads from the buffer. */
+        pb_istream_t stream = pb_istream_from_buffer(buffer, message_length);
+
+        /* Now we are ready to decode the message. */
+        status = pb_decode(&stream, ComplexMessage_fields, &message);
+        modm_assert(status, "pb.dec", "Decoding ComplexMessage failed!");
+
+        modm_assert(!strcmp(message.query, "Hello World"), "query", "Incorrect ComplexMessage values!");
+        modm_assert(message.unlucky_number == 13, "unlucky_number", "Incorrect ComplexMessage values!");
+        modm_assert(message.toggle == true, "toggle", "Incorrect ComplexMessage values!");
+        modm_assert(int(message.value) == 4, "value", "Incorrect ComplexMessage values!");
+    }
+
+    while (true)
+    {
+        /* tumbleweed */
+    }
+    return 0;
+}

examples/nucleo_f429zi/nanopb/project.xml

@@ -0,0 +1,11 @@
+<library>
+  <extends>modm:nucleo-f429zi</extends>
+  <options>
+    <option name="modm:build:build.path">../../../build/nucleo_f429zi/nanopb</option>
+    <option name="modm:nanopb:sources">protocol/simple.proto,protocol/complex.proto</option>
+  </options>
+  <modules>
+    <module>modm:build:scons</module>
+    <module>modm:nanopb</module>
+  </modules>
+</library>

examples/nucleo_f429zi/nanopb/protocol/complex.proto

@@ -0,0 +1,21 @@
+// -*- coding: utf-8 -*-
+//
+// Copyright (c) 2022, Niklas Hauser
+//
+// This file is part of the modm project.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+// -----------------------------------------------------------------------------
+syntax = "proto3";
+
+import "nanopb.proto";
+
+message ComplexMessage {
+    string query = 1 [(nanopb).max_size = 40];
+    int32 unlucky_number = 2;
+    bool toggle = 3;
+    double value = 4;
+}
+

examples/nucleo_f429zi/nanopb/protocol/simple.proto

@@ -0,0 +1,16 @@
+// -*- coding: utf-8 -*-
+//
+// Copyright (c) 2022, Lucas Moesch
+//
+// This file is part of the modm project.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+// -----------------------------------------------------------------------------
+syntax = "proto3";
+
+message SimpleMessage {
+    int32 lucky_number = 1;
+}
+

ext/nanopb/nanopb.lb

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2021, Lucas Moesch
+# Copyright (c) 2022, Niklas Hauser
+#
+# This file is part of the modm project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# -----------------------------------------------------------------------------
+import os
+import subprocess
+import tempfile
+
+from pathlib import Path
+
+def init(module):
+    module.name = ":nanopb"
+    module.description = """
+# Nanopb - Protocol Buffers for Embedded Systems
+
+Nanopb is a small code-size Protocol Buffers (protobuf) implementation in ansi
+C. It is especially suitable for use in microcontrollers, but fits any memory
+restricted system.
+
+See https://github.com/nanopb/nanopb.
+
+
+## Build System Integration
+
+You can optionally point the build system to multiple protofiles using a
+comma-separated list of paths in the `modm:nanopb:source` option.
+You can specify the output location using the `modm:nanopb:path` option.
+See the `modm:build` submodules for further details.
+
+!!! bug "Currently only with SCons support"
+    Only the `modm:build:scons` module currently supports this feature.
+"""
+
+def prepare(module, options):
+    module.add_set_option(
+        PathOption(name="sources", absolute=True, empty_ok=True,
+                   description="Comma-separated list of paths to protofiles."),
+        default="")
+    module.add_option(
+        PathOption(name="path", default="generated/nanopb", absolute=True,
+                   description="Path to the generated messages folder"))
+    return True
+
+def build(env):
+    env.collect(":build:path.include", "modm/ext/nanopb")
+    env.outbasepath = "modm/ext/nanopb"
+
+    env.copy("nanopb/pb_common.c", dest="pb_common.c")
+    env.copy("nanopb/pb_common.h", dest="pb_common.h")
+    env.copy("nanopb/pb_decode.c", dest="pb_decode.c")
+    env.copy("nanopb/pb_decode.h", dest="pb_decode.h")
+    env.copy("nanopb/pb_encode.c", dest="pb_encode.c")
+    env.copy("nanopb/pb_encode.h", dest="pb_encode.h")
+    env.copy("nanopb/pb.h", dest="pb.h")

tools/build_script_generator/scons/module.lb

@@ -3,6 +3,7 @@
 #
 # Copyright (c) 2017-2018, Niklas Hauser
 # Copyright (c) 2019, Raphael Lehmann
+# Copyright (c) 2022, Lucas Moesch
 #
 # This file is part of the modm project.
 #
@@ -108,6 +109,9 @@ def build(env):
     elif device["core"].startswith("avr"):
         tools.update({"size", "avrdude"})
 
+    if env.has_module(":nanopb"):
+        tools.add("nanopb")
+
     env.collect("path.tools", *toolpaths)
     env.collect("tools", *tools)
 
@@ -117,9 +121,15 @@ def build(env):
         path = "site_tools/{}.py".format(tool)
         if exists(localpath(path)):
             env.copy(path)
+
     # Copy support files
     env.copy("site_tools/qtcreator/")
 
+    if env.has_module(":nanopb"):
+        env.copy("site_tools/nanopb_builder/")
+        env.copy(repopath("ext/nanopb/nanopb/generator"), dest="site_tools/nanopb_builder/generator")
+        env.copy(repopath("ext/nanopb/nanopb/tests/site_scons/site_tools/nanopb.py"), dest="site_tools/nanopb_builder/nanopb.py")
+
     # Generate the env.BuildTarget tool
     linkerscript = env.get(":platform:cortex-m:linkerscript.override")
     linkerscript = env.relcwdoutpath(linkerscript) if linkerscript \
@@ -139,6 +149,7 @@ def post_build(env):
     is_unittest = len(env["::unittest.source"])
     has_xpcc_generator = env.has_module(":communication:xpcc:generator")
     has_image_source = len(env["::image.source"])
+    has_nanopb = env.has_module(":nanopb")
     repositories = [p for p in env.buildlog.repositories if isdir(env.real_outpath(p, basepath="."))]
     repositories.sort(key=lambda name: "0" if name == "modm" else name)
     generated_paths = [join(env.relcwdoutpath(""), r) for r in repositories]
@@ -161,9 +172,9 @@ def post_build(env):
         "artifact_path": env.relcwdoutpath(env["path.artifact"]),
         "generated_paths": generated_paths,
         "is_unittest": is_unittest,
-
         "has_image_source": has_image_source,
         "has_xpcc_generator": has_xpcc_generator,
+        "has_nanopb": has_nanopb,
     })
     if has_image_source:
         subs["image_source"] = env.relcwdoutpath(env["::image.source"])
@@ -176,6 +187,9 @@ def post_build(env):
             "generator_path": env.relcwdoutpath(env.get(":communication:xpcc:generator:path", "")),
             "generator_namespace": env.get(":communication:xpcc:generator:namespace", ""),
         })
+    if has_nanopb:
+        subs["nanopb_source"] = [env.relcwdoutpath(p) for p in env.get(":nanopb:sources", [])]
+        subs["nanopb_path"] = env.relcwdoutpath(env.get(":nanopb:path", "."))
     if subs["platform"] == "avr":
         subs.update(env.query("::avrdude_options"))
     if subs["platform"] == "sam":
@@ -220,6 +234,7 @@ def post_build(env):
             "tools": tools,
             "is_modm": repo == "modm",
         })
+
         # Generate library SConscript
         env.outbasepath = repo
         env.template("resources/SConscript.in", "SConscript",

tools/build_script_generator/scons/module.md

@@ -600,6 +600,24 @@ are supported, this is only meant for using the IDE as an editor.
 !!! warning "Consider this an unstable feature"
 
 
+## Protobuf Generator Tool
+
+The `modm:nanopb` module contains a Python generator to translate the messages
+defined in `*.proto` files by the `modm:nanopb:source` option into `*.pb.cpp`
+and `*.pb.hpp` files.
+This module contains a SCons wrapper tool that automatically updates the
+generated files when it becomes necessary:
+
+```py
+cpp_sources += env.NanopbProtofile(
+    sources=options[":nanopb:sources"],
+    path=options[":nanopb:path"],
+)
+```
+
+The generated files are available as a top-level `#include <protofile.pb.hpp>`.
+
+
 ## XPCC Generator Tool
 
 The `modm:communication:xpcc:generator` module contains the Python tools to
@@ -611,7 +629,7 @@ The wrapper tool is automatically used when the generator module is detected,
 and its options are evaluated for the wrapper as follows:
 
 ```py
-env.XpccCommunication(
+cpp_sources += env.XpccCommunication(
     xmlfile=options["::xpcc:generator:source"],
     container=options["::xpcc:generator:container"],
     path=options["::xpcc:generator:path"],

tools/build_script_generator/scons/resources/SConstruct.in

@@ -66,6 +66,14 @@ sources += env.XpccCommunication(
     namespace="{{ generator_namespace }}",
     include_paths=["."])
 %% endif
+%% if has_nanopb
+# Generating Nanopb messages
+env.Append(CPPPATH=abspath("{{ nanopb_path }}"))
+ignored.append("{{ nanopb_path }}")
+sources += env.NanopbProtofile(
+    sources={{ nanopb_source }},
+    path=abspath("{{ nanopb_path }}"))
+%% endif
 
 %% if options["::info.git"] != "Disabled"
 sources.append(env.InfoGit(with_status={{ "True" if "Status" in options["::info.git"] else "False" }}))

tools/build_script_generator/scons/site_tools/comstr.py

@@ -96,6 +96,9 @@ def generate(env, **kw):
         env["XPCC_TASK_CALLER_COMSTR"] = "%s╭─────XPCC───── %s$SOURCE\n" \
                                          "%s╰─Task─Caller─> %s$TARGET%s" % template
 
+        env["NANOPB_MESSAGES_COMSTR"] =  "%s╭────Nanopb──── %s$SOURCE\n" \
+                                         "%s╰───Messages──> %s$TARGET%s" % template
+
         # modm tools installing
         env["ARTIFACT_COMSTR"] =         "%s╭───Artifact─── %s$SOURCE\n" \
                                          "%s╰────Cache────> %s$ARTIFACT_FILEPATH%s" % install