sdk/modules/tflmrt: Run TFLMRT on subcore

examples/dnnrt_lenet/worker/Makefile

@@ -39,7 +39,7 @@ SUBDIRS = $(dir $(WORKER_ELFS))
 
 define DIR_template
 $(1)_$(2):
-	+$(Q) $(MAKE) -C $(1) $(3) TOPDIR="$(TOPDIR)" APPDIR="$(APPDIR)" SDKDIR="$(SDKDIR)" CROSSDEV=$(CROSSDEV)
+	+$(Q) $(MAKE) -C $(1) $(3) TOPDIR="$(TOPDIR)" SDKDIR="$(SDKDIR)" APPDIR="$(APPDIR)" CROSSDEV=$(CROSSDEV)
 endef
 
 .PHONY: clean

examples/tflmrt_lenet/Makefile

@@ -49,3 +49,17 @@ MAINSRC = tflmrt_lenet_main.c
 CSRCS   = loader_tflite.c pnm_util.c model.c image0.c
 
 include $(APPDIR)/Application.mk
+
+ifeq ($(CONFIG_TFLM_RT_MPCOMM),y)
+
+build_worker:
+	@$(MAKE) -C worker TOPDIR="$(TOPDIR)" SDKDIR="$(SDKDIR)" APPDIR="$(APPDIR)" CROSSDEV=$(CROSSDEV)
+
+$(OBJS): build_worker
+
+clean:: clean_worker
+
+clean_worker:
+	@$(MAKE) -C worker TOPDIR="$(TOPDIR)" SDKDIR="$(SDKDIR)" APPDIR="$(APPDIR)" CROSSDEV=$(CROSSDEV) clean
+
+endif

examples/tflmrt_lenet/worker/Makefile

@@ -0,0 +1,57 @@
+############################################################################
+# tflmrt_lenet/worker/Makefile
+#
+#   Copyright 2022 Sony Semiconductor Solutions Corporation
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of Sony Semiconductor Solutions Corporation nor
+#    the names of its contributors may be used to endorse or promote
+#    products derived from this software without specific prior written
+#    permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+############################################################################
+
+WORKER_ELFS = tflmrt-mp/DNNRT-MP
+
+SUBDIRS = $(dir $(WORKER_ELFS))
+
+define DIR_template
+$(1)_$(2):
+	+$(Q) $(MAKE) -C $(1) $(3) TOPDIR="$(TOPDIR)" SDKDIR="$(SDKDIR)" APPDIR="$(APPDIR)" CROSSDEV=$(CROSSDEV)
+endef
+
+.PHONY: clean
+all: $(WORKER_ELFS)
+
+# Build workers
+
+$(WORKER_ELFS):
+	$(Q) $(MAKE) -C $(dir $@) TOPDIR="$(TOPDIR)" SDKDIR="$(SDKDIR)" APPDIR="$(APPDIR)" CROSSDEV=$(CROSSDEV)
+
+# Clean each subdirectory
+
+clean: $(foreach DIR, $(SUBDIRS), $(DIR)_clean)
+
+$(foreach DIR, $(SUBDIRS), $(eval $(call DIR_template,$(DIR),clean,clean)))

examples/tflmrt_lenet/worker/tflmrt-mp/.gitignore

@@ -0,0 +1 @@
+TFLMRT-MP

examples/tflmrt_lenet/worker/tflmrt-mp/Makefile

@@ -0,0 +1,59 @@
+############################################################################
+# tflmrt_lenet/worker/tflmrt-mp/Makefile
+#
+#   Copyright 2022 Sony Semiconductor Solutions Corporation
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of Sony Semiconductor Solutions Corporation nor
+#    the names of its contributors may be used to endorse or promote
+#    products derived from this software without specific prior written
+#    permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+############################################################################
+
+include $(APPDIR)/Make.defs
+-include $(SDKDIR)/Make.defs
+
+BIN = TFLMRT-MP
+WORKER_DIR = $(SDKDIR)$(DELIM)modules$(DELIM)tflmrt$(DELIM)src-mpcomm$(DELIM)worker
+WORKER = $(WORKER_DIR)$(DELIM)tflmrt-mp$(DELIM)$(BIN)
+
+.PHONY: depend clean distclean
+all: $(BIN)
+
+depend:
+	$(Q) $(MAKE) -C $(WORKER_DIR) TOPDIR="$(TOPDIR)" SDKDIR="$(SDKDIR)" depend
+
+$(WORKER): depend
+	$(Q) $(MAKE) -C $(WORKER_DIR) TOPDIR="$(TOPDIR)" SDKDIR="$(SDKDIR)" APPDIR="$(APPDIR)"
+
+$(BIN): $(WORKER)
+	$(Q) install $< $@
+
+clean:
+	$(call DELFILE, $(BIN))
+	$(call CLEAN)
+
+distclean: clean

externals/tensorflow/c-runtime/tf_runtime.cc

@@ -48,13 +48,17 @@
 #include "tensorflow/lite/micro/system_setup.h"
 #include "tensorflow/lite/schema/schema_generated.h"
 
+void *(*tf_rt_malloc_func)(size_t size) = malloc;
+void (*tf_rt_free_func)(void *ptr) = free;
+
 /****************************************************************************
  * Public Functions
  ****************************************************************************/
 
 int tf_rt_allocate_context(tf_rt_context_pointer *context)
 {
-  tf_rt_context_t *c = (tf_rt_context_t *) malloc(sizeof(tf_rt_context_t));
+  tf_rt_context_t *c =
+    (tf_rt_context_t *) tf_rt_malloc_func(sizeof(tf_rt_context_t));
   if (c == 0)
     {
       return -ENOMEM;
@@ -92,7 +96,7 @@ int tf_rt_initialize_context(tf_rt_context_pointer context,
   tflite::AllOpsResolver resolver;
 
   c->tensor_arena_size = size;
-  c->tensor_arena = (uint8_t *) malloc(c->tensor_arena_size);
+  c->tensor_arena = (uint8_t *) tf_rt_malloc_func(c->tensor_arena_size);
   if (c->tensor_arena == 0)
     {
       return -ENOMEM;
@@ -125,12 +129,12 @@ int tf_rt_free_context(tf_rt_context_pointer *context)
 
   if (c->tensor_arena)
     {
-      free(c->tensor_arena);
+      tf_rt_free_func(c->tensor_arena);
       c->tensor_arena = NULL;
       c->tensor_arena_size = 0;
     }
 
-  free(*context);
+  tf_rt_free_func(*context);
 
   return 0;
 }
@@ -218,3 +222,27 @@ int tf_rt_forward(tf_rt_context_pointer context)
 
   return 0;
 }
+
+void tf_rt_set_malloc(void *(*user_malloc)(size_t size))
+{
+  if (user_malloc == 0)
+    {
+      tf_rt_malloc_func = malloc;
+    }
+  else
+    {
+      tf_rt_malloc_func = user_malloc;
+    }
+}
+
+void tf_rt_set_free(void (*user_free)(void *ptr))
+{
+  if (user_free == 0)
+    {
+      tf_rt_free_func = free;
+    }
+  else
+    {
+      tf_rt_free_func = user_free;
+    }
+}

externals/tensorflow/c-runtime/tf_runtime.h

@@ -77,6 +77,8 @@ TfLiteTensor *tf_rt_input_variable(tf_rt_context_pointer context,
 TfLiteTensor *tf_rt_output_variable(tf_rt_context_pointer context,
                                     size_t index);
 int tf_rt_forward(tf_rt_context_pointer context);
+void tf_rt_set_malloc(void *(*user_malloc)(size_t size));
+void tf_rt_set_free(void (*user_free)(void *ptr));
 
 #ifdef __cplusplus
 }

sdk/modules/include/tflmrt/runtime.h

@@ -104,7 +104,7 @@ typedef struct tflm_mallinfo
  * Initialize the whole tflmrt subsystem
  *
  * @param [in] config: configuration of multicore processing. <br>
- *                     If CONFIG_TFLM_RT_MP=y, tflm_config_t::cpu_num
+ *                     If CONFIG_TFLM_RT_MPCOMM=y, tflm_config_t::cpu_num
  *                     must be 1 or more, <br>
  *                     otherwise tflm_config_t::cpu_num must be 1.
  *
@@ -336,7 +336,7 @@ int tflm_nuttx_mallinfo(tflm_mallinfo_t *info);
  *                            This value must be tflm_config_t::cpu_num
  * @param [out] info_array:   Array to store memory allocation stats
  *
- * @return 0 on success. -EPERM if CONFIG_TFLM_RT_MP=n
+ * @return 0 on success. -EPERM if CONFIG_TFLM_RT_MPCOMM=n
  */
 
 int tflm_asmp_mallinfo(unsigned char array_length,

sdk/modules/mpcomm/worker/mpcomm.c

@@ -380,7 +380,7 @@ void *mpcomm_memory_virt_to_phys(void *addr)
 {
   mpcomm_context_t *ctx = get_mpcomm_context();
 
-  return addr + (uint32_t)ctx->loadaddr;
+  return addr < CONFIG_RAM_START ? addr + (uint32_t)ctx->loadaddr : addr;
 }
 
 int mpcomm_send_malloc(void **ptr, size_t size)

sdk/modules/tflmrt/Kconfig

@@ -11,4 +11,25 @@ config TFLM_RT
 	---help---
 		Enable or disable TensorFlow Lite Micro runtime library.
 
+if TFLM_RT
+
+config TFLM_RT_MPCOMM
+	bool "Use multicore processing"
+	select MPCOMM
+	default n
+	---help---
+		Enable or disable multicore processing.
+
+if TFLM_RT_MPCOMM
+
+config TFLM_RT_MPCOMM_PATH
+	string "MPCOMM TFLMRT-MP ELF file path"
+	default "/mnt/sd0/BIN/TFLMRT-MP"
+	---help---
+		Set path to MPCOMM TFLMRT-MP worker program ELF file on the board.
+
+endif
+
+endif
+
 endmenu # TFLM_RT

sdk/modules/tflmrt/Makefile

@@ -39,13 +39,34 @@ RUNTIMEDIR := $(SDKDIR)/../externals/tensorflow/c-runtime
 
 MODNAME = tflmrt
 
+ifeq ($(CONFIG_TFLM_RT_MPCOMM),y)
+CSRCS += runtime_client.c
+
+VPATH += src-mpcomm/runtime
+DEPPATH = --dep-path src-mpcomm/runtime
+else
 CSRCS := runtime_tensorflow.c
 
 CFLAGS += -Isrc/runtime
 
 VPATH = src/runtime
 DEPPATH = --dep-path src/runtime
+endif
 
 CFLAGS += ${shell $(INCDIR) $(INCDIROPT) "$(CC)" $(RUNTIMEDIR)}
 
 include $(SDKDIR)/modules/Module.mk
+
+ifeq ($(CONFIG_TFLM_RT_MPCOMM),y)
+
+build_worker:
+	@$(MAKE) -C src-mpcomm/worker TOPDIR="$(TOPDIR)" SDKDIR="$(SDKDIR)" APPDIR="$(APPDIR)" CROSSDEV=$(CROSSDEV)
+
+$(OBJS): build_worker
+
+clean:: clean_worker
+
+clean_worker:
+	@$(MAKE) -C src-mpcomm/worker TOPDIR="$(TOPDIR)" SDKDIR="$(SDKDIR)" APPDIR="$(APPDIR)" CROSSDEV=$(CROSSDEV) clean
+
+endif