Added tracing_functionality for esp32 platform and integrated the tra…

…cing with esp_insights. (#28466)

Added a esp32_trace directory to provide esp32 specific basic implementation of the scoped tracing macros.
Integrated esp-insights on a config option in the esp32 lighting-app.
This is an initial implemenation of the esp32 specific tracing functionality.

config/esp32/components/chip/CMakeLists.txt

@@ -260,6 +260,10 @@ if (CONFIG_SEC_CERT_DAC_PROVIDER)
     chip_gn_arg_append("chip_use_secure_cert_dac_provider"                 "true")
 endif()
 
+if (CONFIG_ENABLE_ESP_INSIGHTS_TRACE)
+    chip_gn_arg_append("matter_enable_esp_insights_trace"                    "true")
+endif()
+
 if (CONFIG_USE_ESP32_ECDSA_PERIPHERAL)
     chip_gn_arg_append("chip_use_esp32_ecdsa_peripheral" "true")
 endif()
@@ -367,6 +371,10 @@ target_include_directories(${COMPONENT_LIB} INTERFACE
     "${CHIP_ROOT}/config/esp32/${CONFIG_CHIP_EXTERNAL_PLATFORM_DIR}/../../"
 )
 
+if (CONFIG_ENABLE_ESP_INSIGHTS_TRACE)
+    target_include_directories(${COMPONENT_LIB} INTERFACE "${CHIP_ROOT}/src/tracing/esp32_trace/include")
+endif()
+
 idf_component_get_property(mbedtls_lib mbedtls COMPONENT_LIB)
 
 idf_build_get_property(idf_target IDF_TARGET)
@@ -422,6 +430,11 @@ if (CONFIG_SEC_CERT_DAC_PROVIDER)
     list(APPEND chip_libraries $<TARGET_FILE:${esp32_secure_cert_mgr_lib}>)
 endif()
 
+if (CONFIG_ENABLE_ESP_INSIGHTS_TRACE)
+    idf_component_get_property(esp_insights_lib espressif__esp_insights COMPONENT_LIB)
+    list(APPEND chip_libraries $<TARGET_FILE:${esp_insights_lib}>)
+endif()
+
 idf_component_get_property(lwip_lib lwip COMPONENT_LIB)
 list(APPEND chip_libraries $<TARGET_FILE:${lwip_lib}>)
 

config/esp32/components/chip/Kconfig

@@ -798,6 +798,15 @@ menu "CHIP Device Layer"
                 then this option gets enabled.
                 Also, please disable ESP_SECURE_CERT_DS_PERIPHERAL from the menuconfig when this option is disabled
 
+       config ENABLE_ESP_INSIGHTS_TRACE
+           bool "Enable Matter ESP Insights"
+           depends on ESP_INSIGHTS_ENABLED
+           default y
+           help
+               ESP Insights is a remote diagnostics solution to monitor the health of ESP devices in the field.
+               Enabling the above option will enable the esp32 specific tracing functionality and report the
+               diagnostic information to the insights cloud.
+
     endmenu
 
 

config/esp32/components/chip/idf_component.yml

@@ -15,3 +15,7 @@ dependencies:
         version: "2.0.3"
         rules:
             - if: "idf_version >=4.4"
+
+    espressif/esp_insights:
+        version: "1.0.1"
+        require: public

examples/all-clusters-app/esp32/sdkconfig_m5stack_rpc.defaults

@@ -80,3 +80,5 @@ CONFIG_FREERTOS_PLACE_FUNCTIONS_INTO_FLASH=y
 
 # to avoid dram overflow, reduce the critical loggin buffer to 1K
 CONFIG_EVENT_LOGGING_CRIT_BUFFER_SIZE=1024
+
+CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP=y

examples/all-clusters-minimal-app/esp32/sdkconfig_m5stack_rpc.defaults

@@ -80,3 +80,5 @@ CONFIG_FREERTOS_PLACE_FUNCTIONS_INTO_FLASH=y
 
 # to avoid dram overflow, reduce the critical loggin buffer to 1K
 CONFIG_EVENT_LOGGING_CRIT_BUFFER_SIZE=1024
+
+CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP=y

examples/chef/esp32/sdkconfig_rpc.defaults

@@ -61,3 +61,5 @@ CONFIG_MBEDTLS_HKDF_C=y
 
 # IRAM optimizations
 CONFIG_FREERTOS_PLACE_FUNCTIONS_INTO_FLASH=y
+
+CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP=y

examples/lighting-app/esp32/README.md

@@ -8,6 +8,25 @@ and refer
 [building and commissioning](../../../docs/guides/esp32/build_app_and_commission.md)
 guides to get started.
 
+### Enabling ESP-Insights:
+
+-   Before building the app, enable the option: ESP_INSIGHTS_ENABLED through
+    menuconfig.
+
+-   Create a file named insights_auth_key.txt in the main directory of the
+    example.
+
+-   Follow the steps
+    present[here](https://github.com/espressif/esp-insights/blob/main/examples/README.md#set-up-esp-insights-account)
+    to set up an insights_account and the auth key created while setting it up
+    will be used in the example.
+
+-   Download the auth key and copy Auth Key to the example
+
+```
+cp /path/to/auth/key.txt path/to/connectedhomeip/examples/lighting-app/esp32/main/insights_auth_key.txt
+```
+
 ---
 
 -   [Cluster Control](#cluster-control)

examples/lighting-app/esp32/main/CMakeLists.txt

@@ -106,6 +106,10 @@ include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake")
 chip_app_component_codegen("${CHIP_ROOT}/examples/lighting-app/lighting-common/lighting-app.matter")
 chip_app_component_zapgen("${CHIP_ROOT}/examples/lighting-app/lighting-common/lighting-app.zap")
 
+if (CONFIG_ENABLE_ESP_INSIGHTS_TRACE)
+    target_add_binary_data(${COMPONENT_TARGET} "insights_auth_key.txt" TEXT)
+endif()
+
 set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H")
 target_compile_options(${COMPONENT_LIB} PUBLIC

examples/lighting-app/esp32/main/main.cpp

@@ -59,11 +59,20 @@
 #include <platform/ESP32/ESP32SecureCertDACProvider.h>
 #endif
 
+#if CONFIG_ENABLE_ESP_INSIGHTS_TRACE
+#include <esp_insights.h>
+#endif
+
 using namespace ::chip;
 using namespace ::chip::Credentials;
 using namespace ::chip::DeviceManager;
 using namespace ::chip::DeviceLayer;
 
+#if CONFIG_ENABLE_ESP_INSIGHTS_TRACE
+extern const char insights_auth_key_start[] asm("_binary_insights_auth_key_txt_start");
+extern const char insights_auth_key_end[] asm("_binary_insights_auth_key_txt_end");
+#endif
+
 static const char * TAG = "light-app";
 
 static AppDeviceCallbacks EchoCallbacks;
@@ -125,6 +134,20 @@ extern "C" void app_main()
     chip::rpc::Init();
 #endif
 
+#if CONFIG_ENABLE_ESP_INSIGHTS_TRACE
+    esp_insights_config_t config = {
+        .log_type = ESP_DIAG_LOG_TYPE_ERROR | ESP_DIAG_LOG_TYPE_WARNING | ESP_DIAG_LOG_TYPE_EVENT,
+        .auth_key = insights_auth_key_start,
+    };
+
+    esp_err_t ret = esp_insights_init(&config);
+
+    if (ret != ESP_OK)
+    {
+        ESP_LOGE(TAG, "Failed to initialize ESP Insights, err:0x%x", ret);
+    }
+#endif
+
     ESP_LOGI(TAG, "==================================================");
     ESP_LOGI(TAG, "chip-esp32-light-example starting");
     ESP_LOGI(TAG, "==================================================");

examples/lighting-app/esp32/sdkconfig_rpc.defaults

@@ -53,3 +53,5 @@ CONFIG_ENABLE_PW_RPC=y
 
 # Enable HKDF in mbedtls
 CONFIG_MBEDTLS_HKDF_C=y
+
+CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP=y

examples/ota-requestor-app/esp32/sdkconfig_m5stack_rpc.defaults

@@ -73,3 +73,5 @@ CONFIG_ENABLE_PW_RPC=y
 
 # Enable HKDF in mbedtls
 CONFIG_MBEDTLS_HKDF_C=y
+
+CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP=y

examples/ota-requestor-app/esp32/sdkconfig_rpc.defaults

@@ -66,3 +66,5 @@ CONFIG_ENABLE_PW_RPC=y
 
 # Enable HKDF in mbedtls
 CONFIG_MBEDTLS_HKDF_C=y
+
+CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP=y

examples/pigweed-app/esp32/sdkconfig.defaults

@@ -43,3 +43,5 @@ CONFIG_DEVICE_PRODUCT_ID=0x800B
 
 # Enable HKDF in mbedtls
 CONFIG_MBEDTLS_HKDF_C=y
+
+CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP=y

examples/platform/esp32/PigweedLogger.cpp

@@ -23,6 +23,10 @@
 #include <pw_hdlc/encoder.h>
 #include <pw_stream/sys_io_stream.h>
 
+#if CONFIG_ENABLE_ESP_INSIGHTS_TRACE && CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP
+#include <esp_diagnostics.h>
+#endif
+
 namespace PigweedLogger {
 namespace {
 
@@ -126,6 +130,10 @@ extern "C" void __wrap_esp_log_write(esp_log_level_t level, const char * tag, co
     }
 #endif
 
+#if CONFIG_ENABLE_ESP_INSIGHTS_TRACE && CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP
+    esp_diag_log_writev(level, tag, format, v);
+#endif
+
     va_end(v);
 }
 
@@ -153,6 +161,10 @@ extern "C" void __wrap_esp_log_writev(esp_log_level_t level, const char * tag, c
         PigweedLogger::putString(logResetColor, strlen(logResetColor));
     }
 #endif
+
+#if CONFIG_ENABLE_ESP_INSIGHTS_TRACE && CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP
+    esp_diag_log_write(level, tag, format, v);
+#endif
 }
 
 } // namespace PigweedLogger

examples/temperature-measurement-app/esp32/sdkconfig.optimize.defaults

@@ -79,3 +79,5 @@ CONFIG_TCP_SYNMAXRTX=6
 
 # Enable HKDF in mbedtls
 CONFIG_MBEDTLS_HKDF_C=y
+
+CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP=y

examples/temperature-measurement-app/esp32/sdkconfig_rpc.defaults

@@ -94,3 +94,5 @@ CONFIG_ENABLE_PW_RPC=y
 
 # Enable HKDF in mbedtls
 CONFIG_MBEDTLS_HKDF_C=y
+
+CONFIG_DIAG_USE_EXTERNAL_LOG_WRAP=y

src/tracing/esp32_trace/BUILD.gn

@@ -0,0 +1,27 @@
+#
+#Copyright (c) 2023 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build_overrides/build.gni")
+import("//build_overrides/chip.gni")
+
+config("tracing") {
+  include_dirs = [ "include" ]
+}
+
+source_set("esp32_trace") {
+  public = [ "include/matter/tracing/macros_impl.h" ]
+  sources = [ "include/matter/tracing/macros_impl.cpp" ]
+  public_configs = [ ":tracing" ]
+}

src/tracing/esp32_trace/include/matter/tracing/macros_impl.cpp

@@ -0,0 +1,44 @@
+/*
+ *    Copyright (c) 2023 Project CHIP Authors
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#include "macros_impl.h"
+#include <esp_heap_caps.h>
+#include <esp_insights.h>
+namespace Insights {
+
+#define LOG_HEAP_INFO(label, group, entry_exit)                                                                                    \
+    do                                                                                                                             \
+    {                                                                                                                              \
+        ESP_DIAG_EVENT("MTR_TRC", "%s - %s - %s Min Free heap - %u - LFB - %u Start free heap - %u", entry_exit, label, group,     \
+                       heap_caps_get_minimum_free_size(MALLOC_CAP_8BIT),                                                           \
+                       heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),                                    \
+                       heap_caps_get_free_size(MALLOC_CAP_8BIT));                                                                  \
+    } while (0)
+
+ESP32Backend::ESP32Backend(const char * str, ...)
+{
+    va_list args;
+    va_start(args, str);
+    mlabel = str;
+    mgroup = va_arg(args, const char *);
+    LOG_HEAP_INFO(mlabel, mgroup, "Entry");
+}
+
+ESP32Backend::~ESP32Backend()
+{
+    LOG_HEAP_INFO(mlabel, mgroup, "Exit");
+}
+} // namespace Insights

src/tracing/esp32_trace/include/matter/tracing/macros_impl.h

@@ -0,0 +1,50 @@
+/*
+ *    Copyright (c) 2023 Project CHIP Authors
+ *    All rights reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#pragma once
+
+/* Ensure we do not have double tracing macros defined */
+#if defined(MATTER_TRACE_BEGIN)
+#error "Tracing macros seem to be double defined"
+#endif
+
+namespace Insights {
+class ESP32Backend
+{
+public:
+    ESP32Backend(const char * str, ...);
+    ~ESP32Backend();
+
+private:
+    const char * mlabel;
+    const char * mgroup;
+};
+} // namespace Insights
+
+#define MATTER_TRACE_SCOPE(...)                                                                                                    \
+    do                                                                                                                             \
+    {                                                                                                                              \
+        Insights::ESP32Backend backend(__VA_ARGS__);                                                                               \
+    } while (0)
+
+#define _MATTER_TRACE_DISABLE(...)                                                                                                 \
+    do                                                                                                                             \
+    {                                                                                                                              \
+    } while (false)
+
+#define MATTER_TRACE_BEGIN(...) _MATTER_TRACE_DISABLE(__VA_ARGS__)
+#define MATTER_TRACE_END(...) _MATTER_TRACE_DISABLE(__VA_ARGS__)
+#define MATTER_TRACE_INSTANT(...) _MATTER_TRACE_DISABLE(__VA_ARGS__)

src/tracing/tracing_args.gni

@@ -14,6 +14,7 @@
 import("//build_overrides/build.gni")
 import("//build_overrides/chip.gni")
 import("${build_root}/config/compiler/compiler.gni")
+import("${chip_root}/src/platform/device.gni")
 
 declare_args() {
   # Tracing can have a non-zero size impact to binaries. We try to make
@@ -22,7 +23,7 @@ declare_args() {
   #
   # Additionally, if tracing is enabled, the main() function has to add
   # backends explicitly
-  matter_enable_tracing_support = current_os == "android"
+  matter_enable_tracing_support = false
 
   # Defines the trace backend. Current matter tracing splits the logic
   # into two parts:
@@ -43,9 +44,20 @@ declare_args() {
   # since tracing is very noisy, we generally expect it to be explicitly
   # set up.
   #
-  if (current_os == "linux" || current_os == "android") {
-    matter_trace_config = "${chip_root}/src/tracing/perfetto:perfetto_tracing"
-  } else {
-    matter_trace_config = "${chip_root}/src/tracing/none"
-  }
+  matter_trace_config = ""
+  matter_enable_esp_insights_trace = false
+}
+
+if (current_os == "android" ||
+    (chip_device_platform == "esp32" && matter_enable_esp_insights_trace)) {
+  matter_enable_tracing_support = true
+}
+
+if (current_os == "linux" || current_os == "android") {
+  matter_trace_config = "${chip_root}/src/tracing/perfetto:perfetto_tracing"
+} else if (chip_device_platform == "esp32" &&
+           matter_enable_esp_insights_trace) {
+  matter_trace_config = "${chip_root}/src/tracing/esp32_trace"
+} else {
+  matter_trace_config = "${chip_root}/src/tracing/none"
 }