Merge pull request #1 from multiplemonomials/mbed-ce-build-system-ref…

…actor

Mbed CE Build System Refactor

.github/workflows/host_tests.yml

@@ -0,0 +1,20 @@
+name: Run host tests with CMake
+
+on: [pull_request]
+
+jobs:
+  host-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install dev tools
+        run: sudo apt-get install -y cmake ninja-build
+
+      - name: Compile and test for host
+        run: |
+            mkdir __build && cd __build
+            cmake .. -GNinja -DBUILD_GREENTEA_TESTS=FALSE -DBUILD_TESTING=TRUE
+            ninja
+            ctest . --output-on-failure

.gitignore

@@ -104,3 +104,6 @@ cmake_install.cmake
 CMakeFiles/
 cmake_build/
 Testing/
+
+# CLion
+cmake-build-*/

CMakeLists.txt

@@ -4,24 +4,86 @@
 # This is the boilerplate for Mbed OS
 
 cmake_minimum_required(VERSION 3.19.0 FATAL_ERROR)
+cmake_policy(VERSION 3.16...3.22)
+
+# Setup build type (target type, tests/unit tests/real build) ----------------------------------------------------------------------------------
+# This block sets up the following variables for all subdirs to use:
+# - MBED_OS_IS_STANDALONE: True if Mbed OS is the top-level project.  False if Mbed is being built as part of an application.
+# - MBED_IS_NATIVE_BUILD: True if we are building for the host machine.  False if we are building for a microcontroller
+# - MBED_OS_ENABLE_TESTS: True if we are building *any* internal Mbed OS tests at all.  Enabled by -DBUILD_TESTING=TRUE (which is enabled by default when standalone).
+# - BUILD_GREENTEA_TESTS: True to build greentea on-target tests.  False to build host UNITTESTS.  Defaults to false when standalone.
+
+
+if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+    # We are the top level project, so tests or unittests are being built.
+    set(MBED_OS_IS_STANDALONE TRUE)
+else()
+    # Not the top level project
+    set(MBED_OS_IS_STANDALONE FALSE)
+endif()
 
-option(BUILD_GREENTEA_TESTS "Build greentea tests only." OFF)
+# Set up options for testing
+option(BUILD_TESTING "Whether to enable CTest tests in this project" ${MBED_OS_IS_STANDALONE}) # This option is also created by include(CTest) but we need it here earlier on.
+if(MBED_OS_IS_STANDALONE AND BUILD_TESTING)
+    set(MBED_OS_ENABLE_TESTS TRUE)
+    option(BUILD_GREENTEA_TESTS "Build greentea tests instead of unit tests" FALSE)
+endif()
 
-if(BUILD_GREENTEA_TESTS)
-    # Usually we rely on the application to set MBED_CONFIG_PATH and include
-    # app.cmake. They are both required if we're building an application to run
-    # on an mbed-target.
+# Figure out if this is a native build
+if(MBED_OS_IS_STANDALONE)
+
+    # Standalone build, use BUILD_GREENTEA_TESTS to determine if we are building for native or not (user can select)
+    if(BUILD_GREENTEA_TESTS)
+        set(MBED_IS_NATIVE_BUILD FALSE)
+    else()
+        set(MBED_IS_NATIVE_BUILD TRUE)
+    endif()
+
+else()
+
+    # Building as a subdir.  This means that the top-level project will already have called project(), so we can
+    # rely on CMake's platform detection.
+    if(CMAKE_CROSSCOMPILING)
+        set(MBED_IS_NATIVE_BUILD FALSE)
+    else()
+        set(MBED_IS_NATIVE_BUILD TRUE)
+    endif()
+
+endif()
+
+if(MBED_OS_IS_STANDALONE AND NOT MBED_IS_NATIVE_BUILD)
+    # For standalone builds, default to looking for mbed-config.cmake in the binary dir
     set(MBED_CONFIG_PATH ${CMAKE_CURRENT_BINARY_DIR} CACHE STRING "")
-    # TODO: Remove when https://github.com/ARMmbed/mbed-os/issues/14518 is fixed
+
+    # Initialize Mbed build system
     include(${CMAKE_CURRENT_LIST_DIR}/tools/cmake/app.cmake)
 endif()
 
-if(${CMAKE_CROSSCOMPILING})
+if(MBED_IS_NATIVE_BUILD)
+    # Pick up some include files that are needed later
+    list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/tools/cmake)
+    include(mbed_create_distro)
+else()
+    # Grab the Mbed configs for this target
     include(${MBED_CONFIG_PATH}/mbed_config.cmake)
     include(mbed_set_linker_script)
 endif()
 
-project(mbed-os)
+# Print build type
+if(MBED_OS_ENABLE_TESTS)
+    if(BUILD_GREENTEA_TESTS)
+        message(STATUS "Mbed: Compiling Greentea on-target tests for ${MBED_TARGET}")
+    else()
+        message(STATUS "Mbed: Compiling host UNITTESTS for native execution")
+    endif()
+else()
+    message(STATUS "Mbed: Not building any Mbed OS tests.")
+endif()
+
+# Create core Mbed OS targets and set up build flags  ----------------------------------------------------------------------------------
+
+# Create project and find compilers (if not already found)
+project(mbed-os LANGUAGES C CXX ASM)
 
 # Add all paths to the list files within Mbed OS
 list(APPEND CMAKE_MODULE_PATH
@@ -30,7 +92,7 @@ list(APPEND CMAKE_MODULE_PATH
 
 add_subdirectory(extern)
 
-if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
+if(MBED_OS_IS_STANDALONE)
     include(CTest)
 
     if((NOT BUILD_GREENTEA_TESTS) AND BUILD_TESTING)
@@ -40,25 +102,22 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
     endif()
 endif()
 
-add_library(mbed-core INTERFACE)
-
-add_library(mbed-os INTERFACE)
-
-target_link_libraries(mbed-os
-    INTERFACE
-        mbed-rtos
-        mbed-core
-)
+if(BUILD_GREENTEA_TESTS)
+    include(mbed_greentea)
+endif()
 
-add_library(mbed-baremetal INTERFACE)
+# These targets are made visible here so their source files which
+# are spread in different directories can be referenced and can be linked against
+# by libraries that depend on them.
+add_library(mbed-device_key INTERFACE)
+add_library(mbed-rtos-flags INTERFACE) # Collects source files that are in mbed-os but not mbed-baremetal
+add_library(mbed-rtos-sources INTERFACE) # Collects flags that are in mbed-os but not mbed-baremetal
+add_library(mbed-core-flags INTERFACE) # Collects flags common to mbed-baremetal and mbed-os
+add_library(mbed-core-sources INTERFACE) # Collects source files common to mbed-baremetal and mbed-os
 
-target_link_libraries(mbed-baremetal
-    INTERFACE
-        mbed-core
-)
 # Validate selected C library type
 # The C library type selected has to match the library that the target can support
-if(${CMAKE_CROSSCOMPILING})
+if(NOT MBED_IS_NATIVE_BUILD)
     if(${MBED_C_LIB} STREQUAL "small")
         if(NOT "small" IN_LIST MBED_TARGET_SUPPORTED_C_LIBS)
             if("std" IN_LIST MBED_TARGET_SUPPORTED_C_LIBS)
@@ -85,29 +144,27 @@ if(${CMAKE_CROSSCOMPILING})
         )
     endif()
 
-    mbed_set_cpu_core_definitions(mbed-core)
+    mbed_set_cpu_core_definitions(mbed-core-flags)
     if(${MBED_TOOLCHAIN_FILE_USED})
-        mbed_set_profile_options(mbed-core ${MBED_TOOLCHAIN})
-        mbed_set_c_lib(mbed-core ${MBED_C_LIB})
-        mbed_set_printf_lib(mbed-core ${MBED_PRINTF_LIB})
+        mbed_set_profile_options(mbed-core-flags ${MBED_TOOLCHAIN})
+        mbed_set_c_lib(mbed-core-flags ${MBED_C_LIB})
+        mbed_set_printf_lib(mbed-core-flags ${MBED_PRINTF_LIB})
 
-        target_compile_features(mbed-core
+        target_compile_features(mbed-core-flags
             INTERFACE
                 c_std_11
                 cxx_std_14
         )
 
     endif()
 
-    target_compile_definitions(mbed-core
+    target_compile_definitions(mbed-core-flags
         INTERFACE
             TARGET_NAME=${MBED_TARGET}
-            ${MBED_TARGET_DEFINITIONS}
-            ${MBED_CONFIG_DEFINITIONS}
     )
 
     # Add MBED_TEST_MODE for backward compatibility with Greentea tests written for use with Mbed CLI 1
-    if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND BUILD_TESTING)
+    if(MBED_OS_ENABLE_TESTS)
         if(NOT BUILD_GREENTEA_TESTS)
             target_compile_definitions(${PROJECT_NAME}
                 INTERFACE
@@ -127,38 +184,53 @@ if(${CMAKE_CROSSCOMPILING})
     #
     # TODO: Remove this and find a more idiomatic way of passing compile definitions to CPP without
     # using response files or global properties.
-    mbed_generate_options_for_linker(mbed-core RESPONSE_FILE_PATH)
+    mbed_generate_options_for_linker(mbed-core-flags RESPONSE_FILE_PATH)
     set_property(GLOBAL PROPERTY COMPILE_DEFS_RESPONSE_FILE ${RESPONSE_FILE_PATH})
 
     # Add compile definitions for backward compatibility with the toolchain
     # supported. New source files should instead check for __GNUC__ and __clang__
     # for the GCC_ARM and ARM toolchains respectively.
     if(${MBED_TOOLCHAIN} STREQUAL "GCC_ARM")
-        target_compile_definitions(mbed-core
+        target_compile_definitions(mbed-core-flags
             INTERFACE
                 TOOLCHAIN_GCC_ARM
                 TOOLCHAIN_GCC
         )
     elseif(${MBED_TOOLCHAIN} STREQUAL "ARM")
-        target_compile_definitions(mbed-core
+        target_compile_definitions(mbed-core-flags
             INTERFACE
                 TOOLCHAIN_ARM
         )
     endif()
+
+
+    # Ensure the words that make up the Mbed target name are separated with a hyphen, lowercase, and with the `mbed-` prefix.
+    string(TOLOWER ${MBED_TARGET} MBED_TARGET_CMAKE_NAME)
+    string(REPLACE "_" "-" MBED_TARGET_CMAKE_NAME ${MBED_TARGET_CMAKE_NAME})
+    string(PREPEND MBED_TARGET_CMAKE_NAME "mbed-")
+
+endif()
+
+if(MBED_IS_NATIVE_BUILD)
+    # Fix issue on Windows with object files hitting a limit for number of sections
+    if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+        add_compile_options(-Wa,-mbig-obj)
+    endif()
+endif()
+
+# Generate target config header and include it in all files
+if(NOT MBED_IS_NATIVE_BUILD)
+    mbed_write_target_config_header(${CMAKE_CURRENT_BINARY_DIR}/mbed-target-config.h MBED_TARGET_DEFINITIONS MBED_CONFIG_DEFINITIONS)
+    target_compile_options(mbed-core-flags INTERFACE -include ${CMAKE_CURRENT_BINARY_DIR}/mbed-target-config.h)
 endif()
 
 # Include mbed.h and config from generate folder
-target_include_directories(mbed-core
+target_include_directories(mbed-core-flags
     INTERFACE
         ${CMAKE_CURRENT_SOURCE_DIR}
 )
 
-# These targets are made visible here so their source files which
-# are spread in different directories can be referenced and can be linked against
-# by libraries that depend on them.
-# TODO CMake: Should the source files be moved?
-add_library(mbed-device_key INTERFACE)
-add_library(mbed-rtos INTERFACE)
+# Recurse to subdirs  ----------------------------------------------------------------------------------
 
 # Include targets/ first, because any post-build hook needs to be defined
 # before other parts of Mbed OS can add greentea tests which require
@@ -181,14 +253,27 @@ add_subdirectory(features EXCLUDE_FROM_ALL)
 add_subdirectory(cmsis/CMSIS_5/CMSIS/RTOS2 EXCLUDE_FROM_ALL)
 add_subdirectory(cmsis/device/rtos EXCLUDE_FROM_ALL)
 
+# Create top-level targets  ----------------------------------------------------------------------------------
 
-if(${CMAKE_CROSSCOMPILING})
-    # Ensure the words that make up the Mbed target name are separated with a hyphen, lowercase, and with the `mbed-` prefix.
-    string(TOLOWER ${MBED_TARGET} MBED_TARGET_CONVERTED)
-    string(REPLACE "_" "-" MBED_TARGET_CONVERTED ${MBED_TARGET_CONVERTED})
-    string(PREPEND MBED_TARGET_CONVERTED "mbed-")
+if(NOT MBED_IS_NATIVE_BUILD)
+
+    # Create a distro for the microcontroller cmake target, ensuring its sources are only compiled once
+    mbed_create_distro(${MBED_TARGET_CMAKE_NAME}-obj ${MBED_TARGET_CMAKE_NAME} mbed-core-flags)
+
+    # Now make core flags depend on that distro, ensuring everything has access to the uC's flags and objects.
+    # Note that this enforces a hard restriction: none of the libraries folded into the mbed target cmake target can link to
+    # mbed-core-flags, because otherwise there would be a circular dependency.  I'm not sure if that limit will
+    # be a dealbreaker or not in the future.  If it is, we might need to also split each mbed target cmake target
+    # into a -flags and a -sources version.
+    target_link_libraries(mbed-core-flags INTERFACE ${MBED_TARGET_CMAKE_NAME}-obj)
 
-    target_link_libraries(mbed-core INTERFACE ${MBED_TARGET_CONVERTED})
+    # Core Mbed OS libraries
+    # mbed-baremetal contains baremetal sources + target sources + target compile flags.
+    # mbed-os will be a superset of mbed-baremetal, also containing the RTOS sources and RTOS flags.
+    # Note that many different source files will compile differently depending on if the RTOS is in use.
+    # So, it's needed to compile the core sources twice, once for RTOS and once for non-RTOS.
+    mbed_create_distro(mbed-baremetal mbed-core-flags mbed-core-sources)
+    mbed_create_distro(mbed-os mbed-core-flags mbed-core-sources mbed-rtos-flags mbed-rtos-sources)
 endif()
 
 # Ninja requires to be forced for response files
@@ -200,8 +285,4 @@ if ("${CMAKE_GENERATOR}" MATCHES "Ninja")
     if((CMAKE_HOST_SYSTEM_NAME MATCHES "Windows") AND ((${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.22.0") OR (NOT CMAKE_CXX_COMPILER_ID MATCHES "ARMClang")))
         set(CMAKE_NINJA_FORCE_RESPONSE_FILE 1 CACHE INTERNAL "")
     endif()
-endif()
-
-# TODO: Remove once all example applications have removed it
-function(mbed_configure_app_target target)
-endfunction()
+endif()

cmsis/CMSIS_5/CMSIS/RTOS2/CMakeLists.txt

@@ -3,12 +3,12 @@
 
 add_subdirectory(RTX)
 
-target_include_directories(mbed-rtos
+target_include_directories(mbed-rtos-flags
     INTERFACE
         Include
 )
 
-target_sources(mbed-rtos
+target_sources(mbed-rtos-sources
     INTERFACE
         Source/os_systick.c
         Source/os_tick_ptim.c

cmsis/CMSIS_5/CMSIS/RTOS2/RTX/CMakeLists.txt

@@ -22,7 +22,7 @@ if(${CMAKE_CROSSCOMPILING})
                 set(STARTUP_RTX_FILE TARGET_RTOS_M4_M7/irq_cm4f.S)
             endif()
 
-            target_sources(mbed-rtos
+            target_sources(mbed-rtos-sources
                 INTERFACE
                     Source/${toolchain_dir}/${STARTUP_RTX_FILE}
             )
@@ -42,15 +42,15 @@ if(${CMAKE_CROSSCOMPILING})
     _mbed_get_cortex_m_exception_handlers(TOOLCHAIN_GCC)
 endif()
 
-target_include_directories(mbed-rtos
+target_include_directories(mbed-rtos-flags
     INTERFACE
         Config
         Include
         Include1
         Source
 )
 
-target_sources(mbed-rtos
+target_sources(mbed-rtos-sources
     INTERFACE
         Config/RTX_Config.c
 

cmsis/CMakeLists.txt

@@ -4,6 +4,6 @@
 add_subdirectory(CMSIS_5)
 add_subdirectory(device)
 
-if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND BUILD_TESTING)
+if(MBED_OS_ENABLE_TESTS)
     add_subdirectory(tests/UNITTESTS/doubles)
 endif()