321 improved testing (#322)

* Expand tests to handle boundary/error cases (Closes #321).

CMakeLists.txt

@@ -238,6 +238,10 @@ set(MK404_SOURCES_3rdparty
 	3rdParty/arcball/Camera.cpp
 )
 
+set (MK404_TEST_SOURCES
+#	scripts/tests/Internal_tests.cpp
+)
+
 set(MK404_SOURCES ${MK404_SOURCES_base} ${MK404_SOURCES_3rdparty})
 
 if (ENABLE_TIDY)
@@ -335,6 +339,18 @@ if (ENABLE_GCOV)
 	target_link_libraries(MK404 -coverage -lgcov)
 	add_definitions(-DTEST_MODE)
 	set(CMAKE_CXX_OUTPUT_EXTENSION_REPLACE ON)
+
+	include(Catch)
+	#include(CatchAddTests)
+
+	add_executable(MK404_tests scripts/tests/Internal_tests.cpp ${MK404_SOURCES} ${H_FILES} ${MK404_TEST_SOURCES})
+	get_target_property(MK404_TEST_INCL MK404 INCLUDE_DIRECTORIES)
+	target_include_directories(MK404_tests PRIVATE ${MK404_TEST_INCL})
+	target_include_directories(MK404_tests PRIVATE "3rdParty/gsl/")
+	target_include_directories(MK404_tests PRIVATE "3rdParty/catch2/")
+	target_compile_options(MK404_tests PRIVATE -g -O0 -fprofile-arcs -ftest-coverage)
+	target_link_libraries(MK404_tests -coverage -lgcov pthread util m ${GLUT_LIBRARIES} OpenGL::GL OpenGL::GLU GLEW::GLEW ${PNG_LIBRARY} ${SDL_LIBRARY} tinyobjloader simavr ${LIBELF_LIBRARIES})
+	catch_discover_tests(MK404_tests)
 else()
 	set(CMAKE_CXX_OUTPUT_EXTENSION_REPLACE OFF)
 endif()
@@ -366,8 +382,8 @@ add_custom_target(Coverage.tar.gz
 	COMMAND lcov --remove coverage.info '/usr/*' -o coverage.info
 	COMMAND lcov --remove coverage.info '*/3rdParty/*' -o coverage.info
 	COMMAND genhtml coverage.info --output-directory Coverage
-	COMMAND tar -zcvf Coverage.tar.gz Coverage
-	COMMAND ${PROJECT_SOURCE_DIR}/3rdParty/gcovr -r ${PROJECT_SOURCE_DIR} -e 3rdParty | tee ${PROJECT_BINARY_DIR}/summary.txt
+#	COMMAND tar -zcvf Coverage.tar.gz Coverage
+#	COMMAND ${PROJECT_SOURCE_DIR}/3rdParty/gcovr -r ${PROJECT_SOURCE_DIR} -e 3rdParty | tee ${PROJECT_BINARY_DIR}/summary.txt
 	DEPENDS Test_Common
 )
 add_custom_target(Test_Parts
@@ -378,7 +394,7 @@ add_custom_target(Test_Parts
 	DEPENDS Test_Common Test_firmwares
 )
 add_custom_target(Test_ext1
-	COMMAND cd ${PROJECT_BINARY_DIR} && env CTEST_OUTPUT_ON_FAILURE=1 ctest -R 'core|ext1' --timeout 180 -j 1
+	COMMAND cd ${PROJECT_BINARY_DIR} && env CTEST_OUTPUT_ON_FAILURE=1 ctest -R 'core|ext1|Internal' --timeout 180 -j 1
 	COMMAND lcov --capture --gcov-tool="${GCOV_BINARY}" --directory ${PROJECT_BINARY_DIR} --output-file ${PROJECT_BINARY_DIR}/ext1.info
 	COMMAND lcov --remove ext1.info '/usr/*' -o ext1.info
 	COMMAND lcov --remove ext1.info '*/3rdParty/*' -o ext1.info

cmake/Catch.cmake

@@ -0,0 +1,206 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#[=======================================================================[.rst:
+Catch
+-----
+
+This module defines a function to help use the Catch test framework.
+
+The :command:`catch_discover_tests` discovers tests by asking the compiled test
+executable to enumerate its tests.  This does not require CMake to be re-run
+when tests change.  However, it may not work in a cross-compiling environment,
+and setting test properties is less convenient.
+
+This command is intended to replace use of :command:`add_test` to register
+tests, and will create a separate CTest test for each Catch test case.  Note
+that this is in some cases less efficient, as common set-up and tear-down logic
+cannot be shared by multiple test cases executing in the same instance.
+However, it provides more fine-grained pass/fail information to CTest, which is
+usually considered as more beneficial.  By default, the CTest test name is the
+same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
+
+.. command:: catch_discover_tests
+
+  Automatically add tests with CTest by querying the compiled test executable
+  for available tests::
+
+    catch_discover_tests(target
+                         [TEST_SPEC arg1...]
+                         [EXTRA_ARGS arg1...]
+                         [WORKING_DIRECTORY dir]
+                         [TEST_PREFIX prefix]
+                         [TEST_SUFFIX suffix]
+                         [PROPERTIES name1 value1...]
+                         [TEST_LIST var]
+                         [REPORTER reporter]
+                         [OUTPUT_DIR dir]
+                         [OUTPUT_PREFIX prefix}
+                         [OUTPUT_SUFFIX suffix]
+    )
+
+  ``catch_discover_tests`` sets up a post-build command on the test executable
+  that generates the list of tests by parsing the output from running the test
+  with the ``--list-test-names-only`` argument.  This ensures that the full
+  list of tests is obtained.  Since test discovery occurs at build time, it is
+  not necessary to re-run CMake when the list of tests changes.
+  However, it requires that :prop_tgt:`CROSSCOMPILING_EMULATOR` is properly set
+  in order to function in a cross-compiling environment.
+
+  Additionally, setting properties on tests is somewhat less convenient, since
+  the tests are not available at CMake time.  Additional test properties may be
+  assigned to the set of tests as a whole using the ``PROPERTIES`` option.  If
+  more fine-grained test control is needed, custom content may be provided
+  through an external CTest script using the :prop_dir:`TEST_INCLUDE_FILES`
+  directory property.  The set of discovered tests is made accessible to such a
+  script via the ``<target>_TESTS`` variable.
+
+  The options are:
+
+  ``target``
+    Specifies the Catch executable, which must be a known CMake executable
+    target.  CMake will substitute the location of the built executable when
+    running the test.
+
+  ``TEST_SPEC arg1...``
+    Specifies test cases, wildcarded test cases, tags and tag expressions to
+    pass to the Catch executable with the ``--list-test-names-only`` argument.
+
+  ``EXTRA_ARGS arg1...``
+    Any extra arguments to pass on the command line to each test case.
+
+  ``WORKING_DIRECTORY dir``
+    Specifies the directory in which to run the discovered test cases.  If this
+    option is not provided, the current binary directory is used.
+
+  ``TEST_PREFIX prefix``
+    Specifies a ``prefix`` to be prepended to the name of each discovered test
+    case.  This can be useful when the same test executable is being used in
+    multiple calls to ``catch_discover_tests()`` but with different
+    ``TEST_SPEC`` or ``EXTRA_ARGS``.
+
+  ``TEST_SUFFIX suffix``
+    Similar to ``TEST_PREFIX`` except the ``suffix`` is appended to the name of
+    every discovered test case.  Both ``TEST_PREFIX`` and ``TEST_SUFFIX`` may
+    be specified.
+
+  ``PROPERTIES name1 value1...``
+    Specifies additional properties to be set on all tests discovered by this
+    invocation of ``catch_discover_tests``.
+
+  ``TEST_LIST var``
+    Make the list of tests available in the variable ``var``, rather than the
+    default ``<target>_TESTS``.  This can be useful when the same test
+    executable is being used in multiple calls to ``catch_discover_tests()``.
+    Note that this variable is only available in CTest.
+
+  ``REPORTER reporter``
+    Use the specified reporter when running the test case. The reporter will
+    be passed to the Catch executable as ``--reporter reporter``.
+
+  ``OUTPUT_DIR dir``
+    If specified, the parameter is passed along as
+    ``--out dir/<test_name>`` to Catch executable. The actual file name is the
+    same as the test name. This should be used instead of
+    ``EXTRA_ARGS --out foo`` to avoid race conditions writing the result output
+    when using parallel test execution.
+
+  ``OUTPUT_PREFIX prefix``
+    May be used in conjunction with ``OUTPUT_DIR``.
+    If specified, ``prefix`` is added to each output file name, like so
+    ``--out dir/prefix<test_name>``.
+
+  ``OUTPUT_SUFFIX suffix``
+    May be used in conjunction with ``OUTPUT_DIR``.
+    If specified, ``suffix`` is added to each output file name, like so
+    ``--out dir/<test_name>suffix``. This can be used to add a file extension to
+    the output e.g. ".xml".
+
+#]=======================================================================]
+
+#------------------------------------------------------------------------------
+function(catch_discover_tests TARGET)
+  cmake_parse_arguments(
+    ""
+    ""
+    "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX"
+    "TEST_SPEC;EXTRA_ARGS;PROPERTIES"
+    ${ARGN}
+  )
+
+  if(NOT _WORKING_DIRECTORY)
+    set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+  endif()
+  if(NOT _TEST_LIST)
+    set(_TEST_LIST ${TARGET}_TESTS)
+  endif()
+
+  ## Generate a unique name based on the extra arguments
+  string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS} ${_REPORTER} ${_OUTPUT_DIR} ${_OUTPUT_PREFIX} ${_OUTPUT_SUFFIX}")
+  string(SUBSTRING ${args_hash} 0 7 args_hash)
+
+  # Define rule to generate test list for aforementioned test executable
+  set(ctest_include_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_include-${args_hash}.cmake")
+  set(ctest_tests_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_tests-${args_hash}.cmake")
+  get_property(crosscompiling_emulator
+    TARGET ${TARGET}
+    PROPERTY CROSSCOMPILING_EMULATOR
+  )
+  add_custom_command(
+    TARGET ${TARGET} POST_BUILD
+    BYPRODUCTS "${ctest_tests_file}"
+    COMMAND "${CMAKE_COMMAND}"
+            -D "TEST_TARGET=${TARGET}"
+            -D "TEST_EXECUTABLE=$<TARGET_FILE:${TARGET}>"
+            -D "TEST_EXECUTOR=${crosscompiling_emulator}"
+            -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}"
+            -D "TEST_SPEC=${_TEST_SPEC}"
+            -D "TEST_EXTRA_ARGS=${_EXTRA_ARGS}"
+            -D "TEST_PROPERTIES=${_PROPERTIES}"
+            -D "TEST_PREFIX=${_TEST_PREFIX}"
+            -D "TEST_SUFFIX=${_TEST_SUFFIX}"
+            -D "TEST_LIST=${_TEST_LIST}"
+            -D "TEST_REPORTER=${_REPORTER}"
+            -D "TEST_OUTPUT_DIR=${_OUTPUT_DIR}"
+            -D "TEST_OUTPUT_PREFIX=${_OUTPUT_PREFIX}"
+            -D "TEST_OUTPUT_SUFFIX=${_OUTPUT_SUFFIX}"
+            -D "CTEST_FILE=${ctest_tests_file}"
+            -P "${_CATCH_DISCOVER_TESTS_SCRIPT}"
+    VERBATIM
+  )
+
+  file(WRITE "${ctest_include_file}"
+    "if(EXISTS \"${ctest_tests_file}\")\n"
+    "  include(\"${ctest_tests_file}\")\n"
+    "else()\n"
+    "  add_test(${TARGET}_NOT_BUILT-${args_hash} ${TARGET}_NOT_BUILT-${args_hash})\n"
+    "endif()\n"
+  )
+
+  if(NOT ${CMAKE_VERSION} VERSION_LESS "3.10.0") 
+    # Add discovered tests to directory TEST_INCLUDE_FILES
+    set_property(DIRECTORY
+      APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
+    )
+  else()
+    # Add discovered tests as directory TEST_INCLUDE_FILE if possible
+    get_property(test_include_file_set DIRECTORY PROPERTY TEST_INCLUDE_FILE SET)
+    if (NOT ${test_include_file_set})
+      set_property(DIRECTORY
+        PROPERTY TEST_INCLUDE_FILE "${ctest_include_file}"
+      )
+    else()
+      message(FATAL_ERROR
+        "Cannot set more than one TEST_INCLUDE_FILE"
+      )
+    endif()
+  endif()
+
+endfunction()
+
+###############################################################################
+
+set(_CATCH_DISCOVER_TESTS_SCRIPT
+  ${CMAKE_CURRENT_LIST_DIR}/CatchAddTests.cmake
+  CACHE INTERNAL "Catch2 full path to CatchAddTests.cmake helper file"
+)

cmake/CatchAddTests.cmake

@@ -0,0 +1,135 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+set(prefix "${TEST_PREFIX}")
+set(suffix "${TEST_SUFFIX}")
+set(spec ${TEST_SPEC})
+set(extra_args ${TEST_EXTRA_ARGS})
+set(properties ${TEST_PROPERTIES})
+set(reporter ${TEST_REPORTER})
+set(output_dir ${TEST_OUTPUT_DIR})
+set(output_prefix ${TEST_OUTPUT_PREFIX})
+set(output_suffix ${TEST_OUTPUT_SUFFIX})
+set(script)
+set(suite)
+set(tests)
+
+function(add_command NAME)
+  set(_args "")
+  # use ARGV* instead of ARGN, because ARGN splits arrays into multiple arguments
+  math(EXPR _last_arg ${ARGC}-1)
+  foreach(_n RANGE 1 ${_last_arg})
+    set(_arg "${ARGV${_n}}")
+    if(_arg MATCHES "[^-./:a-zA-Z0-9_]")
+      set(_args "${_args} [==[${_arg}]==]") # form a bracket_argument
+    else()
+      set(_args "${_args} ${_arg}")
+    endif()
+  endforeach()
+  set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE)
+endfunction()
+
+# Run test executable to get list of available tests
+if(NOT EXISTS "${TEST_EXECUTABLE}")
+  message(FATAL_ERROR
+    "Specified test executable '${TEST_EXECUTABLE}' does not exist"
+  )
+endif()
+execute_process(
+  COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" ${spec} --list-test-names-only
+  OUTPUT_VARIABLE output
+  RESULT_VARIABLE result
+  WORKING_DIRECTORY "${TEST_WORKING_DIR}"
+)
+# Catch --list-test-names-only reports the number of tests, so 0 is... surprising
+if(${result} EQUAL 0)
+  message(WARNING
+    "Test executable '${TEST_EXECUTABLE}' contains no tests!\n"
+  )
+elseif(${result} LESS 0)
+  message(FATAL_ERROR
+    "Error running test executable '${TEST_EXECUTABLE}':\n"
+    "  Result: ${result}\n"
+    "  Output: ${output}\n"
+  )
+endif()
+
+string(REPLACE "\n" ";" output "${output}")
+
+# Run test executable to get list of available reporters
+execute_process(
+  COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" ${spec} --list-reporters
+  OUTPUT_VARIABLE reporters_output
+  RESULT_VARIABLE reporters_result
+  WORKING_DIRECTORY "${TEST_WORKING_DIR}"
+)
+if(${reporters_result} EQUAL 0)
+  message(WARNING
+    "Test executable '${TEST_EXECUTABLE}' contains no reporters!\n"
+  )
+elseif(${reporters_result} LESS 0)
+  message(FATAL_ERROR
+    "Error running test executable '${TEST_EXECUTABLE}':\n"
+    "  Result: ${reporters_result}\n"
+    "  Output: ${reporters_output}\n"
+  )
+endif()
+string(FIND "${reporters_output}" "${reporter}" reporter_is_valid)
+if(reporter AND ${reporter_is_valid} EQUAL -1)
+  message(FATAL_ERROR
+    "\"${reporter}\" is not a valid reporter!\n"
+  )
+endif()
+
+# Prepare reporter
+if(reporter)
+  set(reporter_arg "--reporter ${reporter}")
+endif()
+
+# Prepare output dir
+if(output_dir AND NOT IS_ABSOLUTE ${output_dir})
+  set(output_dir "${TEST_WORKING_DIR}/${output_dir}")
+  if(NOT EXISTS ${output_dir})
+    file(MAKE_DIRECTORY ${output_dir})
+  endif()
+endif()
+
+# Parse output
+foreach(line ${output})
+  set(test ${line})
+  # Escape characters in test case names that would be parsed by Catch2
+  set(test_name ${test})
+  foreach(char , [ ])
+    string(REPLACE ${char} "\\${char}" test_name ${test_name})
+  endforeach(char)
+  # ...add output dir
+  if(output_dir)
+    string(REGEX REPLACE "[^A-Za-z0-9_]" "_" test_name_clean ${test_name})
+    set(output_dir_arg "--out ${output_dir}/${output_prefix}${test_name_clean}${output_suffix}")
+  endif()
+
+  # ...and add to script
+  add_command(add_test
+    "${prefix}${test}${suffix}"
+    ${TEST_EXECUTOR}
+    "${TEST_EXECUTABLE}"
+    "${test_name}"
+    ${extra_args}
+    "${reporter_arg}"
+    "${output_dir_arg}"
+  )
+  add_command(set_tests_properties
+    "${prefix}${test}${suffix}"
+    PROPERTIES
+    WORKING_DIRECTORY "${TEST_WORKING_DIR}"
+    ${properties}
+  )
+  list(APPEND tests "${prefix}${test}${suffix}")
+endforeach()
+
+# Create a list of all discovered tests, which users may use to e.g. set
+# properties on the tests
+add_command(set ${TEST_LIST} ${tests})
+
+# Write CTest script
+file(WRITE "${CTEST_FILE}" "${script}")