diff --git a/.evergreen/compile.sh b/.evergreen/compile.sh index e833950895..89e7aa9ec4 100755 --- a/.evergreen/compile.sh +++ b/.evergreen/compile.sh @@ -72,15 +72,17 @@ if [[ -f "${mongoc_prefix:?}/.evergreen/scripts/find-ccache.sh" ]]; then find_ccache_and_export_vars "$(pwd)" || true fi + +build_targets=() cmake_build_opts=() case "${OSTYPE:?}" in cygwin) cmake_build_opts+=("/verbosity:minimal") - cmake_examples_target="examples/examples" + build_targets+=(--target ALL_BUILD --target examples/examples) ;; darwin* | linux*) - cmake_examples_target="examples" + build_targets+=(--target all --target examples) ;; *) @@ -88,7 +90,6 @@ darwin* | linux*) exit 1 ;; esac -: "${cmake_examples_target:?}" # Create a VERSION_CURRENT file in the build directory to include in the dist tarball. python ./etc/calc_release_version.py >./build/VERSION_CURRENT @@ -223,21 +224,23 @@ if [[ -n "${REQUIRED_CXX_STANDARD:-}" ]]; then cmake_flags+=("-DCMAKE_CXX_STANDARD_REQUIRED=ON") fi +if [[ "${COMPILE_MACRO_GUARD_TESTS:-"OFF"}" == "ON" ]]; then + cmake_flags+=("-DENABLE_MACRO_GUARD_TESTS=ON") +fi + echo "Configuring with CMake flags: ${cmake_flags[*]}" "${cmake_binary}" "${cmake_flags[@]}" .. if [[ "${COMPILE_MACRO_GUARD_TESTS:-"OFF"}" == "ON" ]]; then # We only need to compile the macro guard tests. - "${cmake_binary}" -DENABLE_MACRO_GUARD_TESTS=ON .. "${cmake_binary}" --build . --config "${build_type:?}" --target test_bsoncxx_macro_guards test_mongocxx_macro_guards -- "${cmake_build_opts[@]}" exit # Nothing else to be done. fi # Regular build and install routine. -"${cmake_binary}" --build . --config "${build_type:?}" -- "${cmake_build_opts[@]}" -"${cmake_binary}" --build . --config "${build_type:?}" --target install -- "${cmake_build_opts[@]}" -"${cmake_binary}" --build . --config "${build_type:?}" --target "${cmake_examples_target:?}" -- "${cmake_build_opts[@]}" +"${cmake_binary}" --build . --config "${build_type:?}" "${build_targets[@]:?}" -- "${cmake_build_opts[@]}" +"${cmake_binary}" --install . --config "${build_type:?}" if [[ "${_RUN_DISTCHECK:-}" ]]; then "${cmake_binary}" --build . --config "${build_type:?}" --target distcheck diff --git a/.evergreen/test.sh b/.evergreen/test.sh index ec99f0d385..ade6af0449 100755 --- a/.evergreen/test.sh +++ b/.evergreen/test.sh @@ -262,7 +262,12 @@ else echo "CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH:?}" fi - run_test() { "$@"; } + test_args=( + --reporter compact + --allow-running-no-tests + ) + + run_test() { "$@" "${test_args[@]:?}"; } if [[ "${TEST_WITH_ASAN:-}" == "ON" || "${TEST_WITH_UBSAN:-}" == "ON" ]]; then export ASAN_OPTIONS="detect_leaks=1" @@ -270,7 +275,7 @@ else export PATH="/usr/lib/llvm-3.8/bin:${PATH:-}" elif [[ "${TEST_WITH_VALGRIND:-}" == "ON" ]]; then run_test() { - valgrind --leak-check=full --track-origins=yes --num-callers=50 --error-exitcode=1 --error-limit=no --read-var-info=yes --suppressions=../etc/memcheck.suppressions "$@" + valgrind --leak-check=full --track-origins=yes --num-callers=50 --error-exitcode=1 --error-limit=no --read-var-info=yes --suppressions=../etc/memcheck.suppressions "$@" "${test_args[@]:?}" } fi @@ -286,11 +291,7 @@ else run_test ./src/mongocxx/test/test_logging run_test ./src/mongocxx/test/test_retryable_reads_specs run_test ./src/mongocxx/test/test_read_write_concern_specs - run_test ./src/mongocxx/test/test_unified_format_spec - - echo "Building examples..." - "${cmake_binary:?}" --build . --target examples - echo "Building examples... done." + run_test ./src/mongocxx/test/test_unified_format_specs # Only run examples if MONGODB_API_VERSION is unset. We do not append # API version to example clients, so examples will fail when requireApiVersion diff --git a/cmake/FetchCatch2.cmake b/cmake/FetchCatch2.cmake index 3ce6c40d2a..d33cf7c89a 100644 --- a/cmake/FetchCatch2.cmake +++ b/cmake/FetchCatch2.cmake @@ -2,8 +2,6 @@ include(FetchContent) -message(STATUS "Downloading Catch2...") - function(fetch_catch2) set(fetch_args "") if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.25.0") @@ -26,6 +24,8 @@ function(fetch_catch2) FetchContent_GetProperties(EP_Catch2) if(NOT ep_catch2_POPULATED) + message(STATUS "Downloading Catch2...") + # Avoid Catch2 compile warnings from being treated as errors. string(REPLACE " -Werror" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") @@ -38,9 +38,9 @@ function(fetch_catch2) # Catch2 config vars. set_property(CACHE CATCH_INSTALL_DOCS PROPERTY VALUE OFF) set_property(CACHE CATCH_INSTALL_EXTRAS PROPERTY VALUE OFF) + + message (STATUS "Downloading Catch2... done.") endif() endfunction() fetch_catch2() - -message (STATUS "Downloading Catch2... done.") diff --git a/cmake/make_dist/MakeDistFiles.cmake b/cmake/make_dist/MakeDistFiles.cmake index 6841933ecc..3f8bb1d8c1 100644 --- a/cmake/make_dist/MakeDistFiles.cmake +++ b/cmake/make_dist/MakeDistFiles.cmake @@ -27,6 +27,7 @@ function (EXECUTE_PROCESS_AND_CHECK_RESULT) ${VARS_COMMAND} WORKING_DIRECTORY ${VARS_WORKING_DIRECTORY} RESULT_VARIABLE RESULT + OUTPUT_QUIET ) if (NOT "${RESULT}" STREQUAL "0") message (FATAL_ERROR ${VARS_ERROR_MSG}) diff --git a/src/bsoncxx/test/CMakeLists.txt b/src/bsoncxx/test/CMakeLists.txt index 603c25e665..f8d6286190 100644 --- a/src/bsoncxx/test/CMakeLists.txt +++ b/src/bsoncxx/test/CMakeLists.txt @@ -70,7 +70,8 @@ target_link_libraries(bsoncxx_test_properties_with_main INTERFACE bsoncxx::test_ add_library(bsoncxx::test_properties_with_main ALIAS bsoncxx_test_properties_with_main) target_link_libraries (test_bson PRIVATE bsoncxx::test_properties_with_main) -add_test(NAME bson COMMAND test_bson) + +add_test(NAME bson COMMAND test_bson --reporter compact --allow-running-no-tests) # Generate test to ensure macro guards behave properly. if(ENABLE_MACRO_GUARD_TESTS) diff --git a/src/mongocxx/test/CMakeLists.txt b/src/mongocxx/test/CMakeLists.txt index 75ae1b47d8..4883715faa 100644 --- a/src/mongocxx/test/CMakeLists.txt +++ b/src/mongocxx/test/CMakeLists.txt @@ -135,8 +135,8 @@ target_link_libraries(test_read_write_concern_specs PRIVATE spec_test_common cli add_executable(test_mongohouse_specs spec/mongohouse.cpp) target_link_libraries(test_mongohouse_specs PRIVATE spec_test_common client_helpers) -add_executable(test_unified_format_spec spec/unified_tests/operations.cpp spec/unified_tests/runner.cpp) -target_link_libraries(test_unified_format_spec PRIVATE spec_test_common client_helpers) +add_executable(test_unified_format_specs spec/unified_tests/operations.cpp spec/unified_tests/runner.cpp) +target_link_libraries(test_unified_format_specs PRIVATE spec_test_common client_helpers) add_executable(test_versioned_api versioned_api.cpp) target_link_libraries(test_versioned_api PRIVATE spec_test_common client_helpers) @@ -183,7 +183,7 @@ set_property( test_read_write_concern_specs test_retryable_reads_specs test_transactions_specs - test_unified_format_spec + test_unified_format_specs test_versioned_api APPEND PROPERTY LINK_LIBRARIES mongocxx::test_properties_with_main @@ -206,23 +206,27 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) target_link_libraries(test_driver PRIVATE Threads::Threads) -add_test(NAME driver COMMAND test_driver) -add_test(NAME logging COMMAND test_logging) -add_test(NAME instance COMMAND test_instance) -add_test(NAME crud_specs COMMAND test_crud_specs) -add_test(NAME gridfs_specs COMMAND test_gridfs_specs) -add_test(NAME client_side_encryption_specs COMMAND test_client_side_encryption_specs) -add_test(NAME command_monitoring_specs COMMAND test_command_monitoring_specs) -add_test(NAME transactions_specs COMMAND test_transactions_specs) -add_test(NAME retryable_reads_spec COMMAND test_retryable_reads_specs) -add_test(NAME read_write_concern_specs COMMAND test_read_write_concern_specs) -add_test(NAME unified_format_spec COMMAND test_unified_format_spec) -add_test(NAME versioned_api COMMAND test_versioned_api) +foreach(test_name + client_side_encryption_specs + command_monitoring_specs + crud_specs + driver + gridfs_specs + instance + logging + read_write_concern_specs + retryable_reads_specs + transactions_specs + unified_format_specs + versioned_api +) + add_test(NAME ${test_name} COMMAND test_${test_name} --reporter compact --allow-running-no-tests) +endforeach() # Adding this as a test will run it as part of the RUN_TESTS command in MSVC. # Do not add, since we only test mongohouse on Linux. if(0) - add_test(mongohouse_specs NAME test_mongohouse_specs COMMAND) + add_test(NAME mongohouse_specs COMMAND test_mongohouse_specs ${test_args}) endif() set_property(TEST crud_specs APPEND PROPERTY ENVIRONMENT @@ -251,7 +255,7 @@ set_property(TEST transactions_specs APPEND PROPERTY ENVIRONMENT "WITH_TRANSACTION_TESTS_PATH=${DATA_SOURCE_DIR}/with_transaction" ) -set_property(TEST retryable_reads_spec APPEND PROPERTY ENVIRONMENT +set_property(TEST retryable_reads_specs APPEND PROPERTY ENVIRONMENT "RETRYABLE_READS_LEGACY_TESTS_PATH=${DATA_SOURCE_DIR}/retryable-reads/legacy" ) @@ -259,7 +263,7 @@ set_property(TEST read_write_concern_specs APPEND PROPERTY ENVIRONMENT "READ_WRITE_CONCERN_OPERATION_TESTS_PATH=${DATA_SOURCE_DIR}/read-write-concern/operation" ) -set_property(TEST unified_format_spec APPEND PROPERTY ENVIRONMENT +set_property(TEST unified_format_specs APPEND PROPERTY ENVIRONMENT "CHANGE_STREAMS_UNIFIED_TESTS_PATH=${DATA_SOURCE_DIR}/change-streams/unified" "CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS_PATH=${DATA_SOURCE_DIR}/client_side_encryption/unified" "COLLECTION_MANAGEMENT_TESTS_PATH=${DATA_SOURCE_DIR}/collection-management" diff --git a/src/mongocxx/test/change_streams.cpp b/src/mongocxx/test/change_streams.cpp index 76724dac64..7fa07b77f3 100644 --- a/src/mongocxx/test/change_streams.cpp +++ b/src/mongocxx/test/change_streams.cpp @@ -95,8 +95,7 @@ TEST_CASE("Change stream options") { client mongodb_client{uri{}, test_util::add_test_server_api()}; if (!test_util::is_replica_set(mongodb_client)) { - WARN("skip: change streams require replica set"); - return; + SKIP("change streams require replica set"); } SECTION("Error if both resumeAfter and startAfter are set") { @@ -118,8 +117,7 @@ TEST_CASE("Spec Prose Tests") { client client{uri{}, test_util::add_test_server_api()}; if (!test_util::is_replica_set(client)) { - WARN("skip: change streams require replica set"); - return; + SKIP("change streams require replica set"); } auto db = client["db"]; @@ -387,8 +385,7 @@ TEST_CASE("Create streams.events and assert we can read a single event", "[min36 instance::current(); client mongodb_client{uri{}, test_util::add_test_server_api()}; if (!test_util::is_replica_set(mongodb_client)) { - WARN("skip: change streams require replica set"); - return; + SKIP("change streams require replica set"); } collection events = mongodb_client["streams"]["events"]; @@ -408,8 +405,7 @@ TEST_CASE("Give an invalid pipeline", "[min36]") { instance::current(); client mongodb_client{uri{}, test_util::add_test_server_api()}; if (!test_util::is_replica_set(mongodb_client)) { - WARN("skip: change streams require replica set"); - return; + SKIP("change streams require replica set"); } options::change_stream options{}; @@ -439,8 +435,7 @@ TEST_CASE("Documentation Examples", "[min36]") { mongocxx::pool pool{uri{}, options::pool(test_util::add_test_server_api())}; auto mongodb_client = pool.acquire(); if (!test_util::is_replica_set(*mongodb_client)) { - WARN("skip: change streams require replica set"); - return; + SKIP("change streams require replica set"); } collection events = (*mongodb_client)["streams"]["events"]; @@ -546,8 +541,7 @@ TEST_CASE("Watch 2 collections", "[min36]") { instance::current(); client mongodb_client{uri{}, test_util::add_test_server_api()}; if (!test_util::is_replica_set(mongodb_client)) { - WARN("skip: change streams require replica set"); - return; + SKIP("change streams require replica set"); } options::change_stream options{}; @@ -594,8 +588,7 @@ TEST_CASE("Watch a Collection", "[min36]") { instance::current(); client mongodb_client{uri{}, test_util::add_test_server_api()}; if (!test_util::is_replica_set(mongodb_client)) { - WARN("skip: change streams require replica set"); - return; + SKIP("change streams require replica set"); } options::change_stream options{}; diff --git a/src/mongocxx/test/client_helpers.cpp b/src/mongocxx/test/client_helpers.cpp index bde7b55c18..ac68d973eb 100644 --- a/src/mongocxx/test/client_helpers.cpp +++ b/src/mongocxx/test/client_helpers.cpp @@ -492,7 +492,7 @@ void check_outcome_collection(mongocxx::collection* coll, bsoncxx::document::vie REQUIRE(begin(actual) == end(actual)); } -bool server_has_sessions(const client& conn) { +bool server_has_sessions_impl(const client& conn) { auto result = get_is_master(conn); auto result_view = result.view(); @@ -500,51 +500,45 @@ bool server_has_sessions(const client& conn) { return true; } - WARN("skip: server does not support sessions"); return false; } -bool should_run_client_side_encryption_test(void) { -#ifndef MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION - WARN("linked libmongoc does not support client side encryption - skipping tests"); - return false; -#endif - - std::vector vars{ - "MONGOCXX_TEST_AWS_SECRET_ACCESS_KEY", - "MONGOCXX_TEST_AWS_ACCESS_KEY_ID", - "MONGOCXX_TEST_AZURE_TENANT_ID", - "MONGOCXX_TEST_AZURE_CLIENT_ID", - "MONGOCXX_TEST_AZURE_CLIENT_SECRET", - "MONGOCXX_TEST_CSFLE_TLS_CA_FILE", - "MONGOCXX_TEST_CSFLE_TLS_CERTIFICATE_KEY_FILE", - "MONGOCXX_TEST_GCP_EMAIL", - "MONGOCXX_TEST_GCP_PRIVATEKEY", - }; - - std::ostringstream os; - os << "Please set environment variables to enable client side encryption tests:\n"; - std::copy(std::begin(vars), std::end(vars), std::ostream_iterator(os, "\n")); +#if defined(MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION) - if (std::none_of(std::begin(vars), std::end(vars), std::getenv)) { - os << "Skipping client side encryption tests.\n"; +cseeos_result client_side_encryption_enabled_or_skip_impl() { + static const cseeos_result result = [] { + std::vector vars{ + "MONGOCXX_TEST_AWS_SECRET_ACCESS_KEY", + "MONGOCXX_TEST_AWS_ACCESS_KEY_ID", + "MONGOCXX_TEST_AZURE_TENANT_ID", + "MONGOCXX_TEST_AZURE_CLIENT_ID", + "MONGOCXX_TEST_AZURE_CLIENT_SECRET", + "MONGOCXX_TEST_CSFLE_TLS_CA_FILE", + "MONGOCXX_TEST_CSFLE_TLS_CERTIFICATE_KEY_FILE", + "MONGOCXX_TEST_GCP_EMAIL", + "MONGOCXX_TEST_GCP_PRIVATEKEY", + }; - WARN(os.str()); + const auto is_set = [](const char* var) -> bool { return std::getenv(var) != nullptr; }; - return false; - } + const auto count = std::count_if(vars.begin(), vars.end(), is_set); - if (!std::all_of(std::begin(vars), std::end(vars), std::getenv)) { - os << "Failing client side encryption tests (some environment variables were not set).\n"; + if (count == 0) { + return cseeos_result::skip; + } - FAIL(os.str()); + if (static_cast(count) < vars.size()) { + return cseeos_result::fail; + } - return false; - } + return cseeos_result::enable; + }(); - return true; + return result; } +#endif // defined(MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION) + std::string getenv_or_fail(const std::string env_name) { auto val = std::getenv(env_name.c_str()); if (!val) { diff --git a/src/mongocxx/test/client_helpers.hh b/src/mongocxx/test/client_helpers.hh index e89a73b270..905362d313 100644 --- a/src/mongocxx/test/client_helpers.hh +++ b/src/mongocxx/test/client_helpers.hh @@ -210,9 +210,39 @@ auto size(Container c) -> decltype(std::distance(std::begin(c), std::end(c))) { // // @return Whether sessions are supported by the client's topology. // -bool server_has_sessions(const client& conn); - -bool should_run_client_side_encryption_test(void); +bool server_has_sessions_impl(const client& conn); + +#define SERVER_HAS_SESSIONS_OR_SKIP(conn) \ + if (!mongocxx::test_util::server_has_sessions_impl(conn)) { \ + SKIP("server does not support session"); \ + } else \ + ((void)0) + +#if defined(MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION) +enum struct cseeos_result { + enable, + skip, + fail, +}; + +cseeos_result client_side_encryption_enabled_or_skip_impl(); + +#define CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP() \ + if (1) { \ + switch (mongocxx::test_util::client_side_encryption_enabled_or_skip_impl()) { \ + case mongocxx::test_util::cseeos_result::enable: \ + break; \ + case mongocxx::test_util::cseeos_result::skip: \ + SKIP("CSE environemnt variables not set"); \ + case mongocxx::test_util::cseeos_result::fail: \ + FAIL("One or more CSE environment variable is missing"); \ + } \ + } else \ + ((void)0) +#else +#define CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP() \ + SKIP("linked libmongoc does not support client side encryption") +#endif // defined(MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION) std::string getenv_or_fail(const std::string env_name); diff --git a/src/mongocxx/test/client_session.cpp b/src/mongocxx/test/client_session.cpp index 9950f7503d..e2f959aa1a 100644 --- a/src/mongocxx/test/client_session.cpp +++ b/src/mongocxx/test/client_session.cpp @@ -37,16 +37,13 @@ using bsoncxx::document::value; using bsoncxx::types::b_timestamp; using namespace mongocxx; -using test_util::server_has_sessions; TEST_CASE("session options", "[session]") { instance::current(); client c{uri{}, test_util::add_test_server_api()}; - if (!server_has_sessions(c)) { - return; - } + SERVER_HAS_SESSIONS_OR_SKIP(c); SECTION("default") { // Make sure the defaults don't cause a client exception: @@ -184,9 +181,7 @@ TEST_CASE("session", "[session]") { client c{uri{}, test_util::add_test_server_api()}; - if (!server_has_sessions(c)) { - return; - } + SERVER_HAS_SESSIONS_OR_SKIP(c); auto s = c.start_session(); @@ -398,9 +393,7 @@ TEST_CASE("lsid", "[session]") { session_test test; - if (!server_has_sessions(test.client)) { - return; - } + SERVER_HAS_SESSIONS_OR_SKIP(test.client); auto s = test.client.start_session(); auto db = test.client["lsid"]; @@ -699,8 +692,7 @@ TEST_CASE("lsid", "[session]") { SECTION("collection::watch") { if (!test_util::is_replica_set(test.client)) { - WARN("skip: watch() requires replica set"); - return; + SKIP("watch() requires replica set"); } auto f = [&s, &collection](bool use_session) { @@ -848,9 +840,7 @@ TEST_CASE("with_transaction", "[session]") { session_test test; - if (!server_has_sessions(test.client)) { - return; - } + SERVER_HAS_SESSIONS_OR_SKIP(test.client); auto session = test.client.start_session(); @@ -859,8 +849,7 @@ TEST_CASE("with_transaction", "[session]") { SECTION("callback raises a custom error") { // Multi-document transactions require server 4.2+. if (compare_versions(get_server_version(test.client), "4.2") < 0) { - WARN("Skipping - MongoDB server 4.2 or newer required"); - return; + SKIP("MongoDB server 4.2 or newer required"); } // Test an operation_exception @@ -895,9 +884,7 @@ TEST_CASE("unacknowledged write in session", "[session]") { session_test test; - if (!server_has_sessions(test.client)) { - return; - } + SERVER_HAS_SESSIONS_OR_SKIP(test.client); auto s = test.client.start_session(); auto db = test.client["lsid"]; diff --git a/src/mongocxx/test/client_side_encryption.cpp b/src/mongocxx/test/client_side_encryption.cpp index 225a223e09..7cf9a1260d 100644 --- a/src/mongocxx/test/client_side_encryption.cpp +++ b/src/mongocxx/test/client_side_encryption.cpp @@ -343,6 +343,8 @@ void run_datakey_and_double_encryption(Callable create_data_key, TEST_CASE("Datakey and double encryption", "[client_side_encryption]") { instance::current(); + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); + // Setup // 1. Create a mongoclient without encryption options::client client_opts; @@ -351,14 +353,9 @@ TEST_CASE("Datakey and double encryption", "[client_side_encryption]") { mongocxx::client setup_client{uri{}, test_util::add_test_server_api(client_opts)}; - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } - if (test_util::get_max_wire_version(setup_client) < 8) { // Automatic encryption requires wire version 8. - WARN("Skipping - max wire version is < 8"); - return; + SKIP("max wire version is < 8"); } // 2. Drop keyvault.datakeys and db.coll @@ -584,9 +581,7 @@ void run_external_key_vault_test(bool with_external_key_vault) { TEST_CASE("External key vault", "[client_side_encryption]") { instance::current(); - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); mongocxx::client setup_client{ uri{}, @@ -595,8 +590,7 @@ TEST_CASE("External key vault", "[client_side_encryption]") { if (test_util::get_max_wire_version(setup_client) < 8) { // Automatic encryption requires wire version 8. - WARN("Skipping - max wire version is < 8"); - return; + SKIP("max wire version is < 8"); } run_external_key_vault_test(true); @@ -606,9 +600,7 @@ TEST_CASE("External key vault", "[client_side_encryption]") { TEST_CASE("BSON size limits and batch splitting", "[client_side_encryption]") { instance::current(); - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); // Create a MongoClient without encryption enabled (referred to as client). mongocxx::client client{ @@ -618,8 +610,7 @@ TEST_CASE("BSON size limits and batch splitting", "[client_side_encryption]") { if (test_util::get_max_wire_version(client) < 8) { // Automatic encryption requires wire version 8. - WARN("Skipping - max wire version is < 8"); - return; + SKIP("max wire version is < 8"); } // Load in json schema limits/limits-schema.json and limits/limits-key.json @@ -768,9 +759,7 @@ TEST_CASE("BSON size limits and batch splitting", "[client_side_encryption]") { TEST_CASE("Views are prohibited", "[client_side_encryption]") { instance::current(); - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); // Create a MongoClient without encryption enabled (referred to as client). mongocxx::client client{ @@ -780,8 +769,7 @@ TEST_CASE("Views are prohibited", "[client_side_encryption]") { if (test_util::get_max_wire_version(client) < 8) { // Automatic encryption requires wire version 8. - WARN("Skipping - max wire version is < 8"); - return; + SKIP("max wire version is < 8"); } // Using client, drop and create a view named db.view with an empty pipeline. @@ -1119,13 +1107,11 @@ void _run_corpus_test(bool use_schema_map) { TEST_CASE("Corpus", "[client_side_encryption]") { instance::current(); + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); + // Data keys created with AWS KMS may specify a custom endpoint to contact // (instead of the default endpoint derived from the AWS region). - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } - mongocxx::client setup_client{ uri{}, test_util::add_test_server_api(), @@ -1133,8 +1119,7 @@ TEST_CASE("Corpus", "[client_side_encryption]") { if (test_util::get_max_wire_version(setup_client) < 8) { // Automatic encryption requires wire version 8. - WARN("Skipping - max wire version is < 8"); - return; + SKIP("max wire version is < 8"); } _run_corpus_test(true); _run_corpus_test(false); @@ -1249,13 +1234,11 @@ void _run_endpoint_test(mongocxx::client* setup_client, TEST_CASE("Custom endpoint", "[client_side_encryption]") { instance::current(); + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); + // Data keys created with AWS KMS may specify a custom endpoint to contact // (instead of the default endpoint derived from the AWS region). - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } - mongocxx::client setup_client{ uri{}, test_util::add_test_server_api(), @@ -1263,8 +1246,7 @@ TEST_CASE("Custom endpoint", "[client_side_encryption]") { if (test_util::get_max_wire_version(setup_client) < 8) { // Automatic encryption requires wire version 8. - WARN("Skipping - max wire version is < 8"); - return; + SKIP("max wire version is < 8"); } // Call client_encryption.createDataKey() with "aws" as the provider and the following @@ -1605,9 +1587,7 @@ void bypass_mongocrypt_via_shared_library(const std::string& shared_lib_path, TEST_CASE("Bypass spawning mongocryptd", "[client_side_encryption]") { instance::current(); - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); mongocxx::client setup_client{ uri{}, @@ -1616,8 +1596,7 @@ TEST_CASE("Bypass spawning mongocryptd", "[client_side_encryption]") { if (test_util::get_max_wire_version(setup_client) < 8) { // Automatic encryption requires wire version 8. - WARN("Skipping - max wire version is < 8"); - return; + SKIP("max wire version is < 8"); } auto shared_lib_path = getenv("CRYPT_SHARED_LIB_PATH"); @@ -1744,9 +1723,7 @@ class kms_tls_expired_cert_matcher : public Catch::Matchers::MatcherBase _setup_explicit_encryp TEST_CASE("Explicit Encryption", "[client_side_encryption]") { instance::current(); - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); mongocxx::client conn{ mongocxx::uri{}, @@ -2212,13 +2174,11 @@ TEST_CASE("Explicit Encryption", "[client_side_encryption]") { }; if (!test_util::newer_than(conn, "7.0")) { - WARN("Skipping - MongoDB server 7.0 or newer required"); - return; + SKIP("MongoDB server 7.0 or newer required"); } if (test_util::get_topology(conn) == "single") { - WARN("Skipping - must not run against a standalone server"); - return; + SKIP("must not run against a standalone server"); } // Load the file key1-document.json as key1Document. @@ -2510,20 +2470,17 @@ TEST_CASE("Explicit Encryption", "[client_side_encryption]") { TEST_CASE("Create Encrypted Collection", "[client_side_encryption]") { instance::current(); - mongocxx::client conn{mongocxx::uri{}, test_util::add_test_server_api()}; - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); + + mongocxx::client conn{mongocxx::uri{}, test_util::add_test_server_api()}; if (!test_util::newer_than(conn, "7.0")) { - WARN("Explicit Encryption tests require MongoDB server 7.0+."); - return; + SKIP("Explicit Encryption tests require MongoDB server 7.0+."); } if (test_util::get_topology(conn) == "single") { - WARN("Explicit Encryption tests must not run against a standalone."); - return; + SKIP("Explicit Encryption tests must not run against a standalone."); } conn.database("keyvault").collection("datakeys").drop(); @@ -2649,13 +2606,10 @@ TEST_CASE("Create Encrypted Collection", "[client_side_encryption]") { TEST_CASE("Unique Index on keyAltNames", "[client_side_encryption]") { instance::current(); - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); if (!test_util::newer_than(uri{}, "4.2")) { - WARN("Skipping - requires MongoDB server 4.2+"); - return; + SKIP("requires MongoDB server 4.2+"); } // 1. Create a MongoClient object (referred to as client). @@ -2793,14 +2747,10 @@ TEST_CASE("Unique Index on keyAltNames", "[client_side_encryption]") { TEST_CASE("Custom Key Material Test", "[client_side_encryption]") { instance::current(); - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - WARN("Skipping - Client Side Encryption is required"); - return; - } + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); if (!test_util::newer_than(uri{}, "4.2")) { - WARN("Skipping - MongoDB server 4.2 or newer required"); - return; + SKIP("MongoDB server 4.2 or newer required"); } // 1. Create a MongoClient object (referred to as client). @@ -3139,9 +3089,7 @@ range_explicit_encryption_objects range_explicit_encryption_setup(const std::str TEST_CASE("Range Explicit Encryption", "[client_side_encryption]") { instance::current(); - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); // Tests for `DecimalNoPrecision` must only run against a replica set. auto is_replica_set = false; @@ -3150,20 +3098,15 @@ TEST_CASE("Range Explicit Encryption", "[client_side_encryption]") { auto client = mongocxx::client(mongocxx::uri(), test_util::add_test_server_api()); if (!test_util::newer_than(client, "7.0")) { - WARN("Skipping - MongoDB server 7.0 or newer required"); - return; + SKIP("MongoDB server 7.0 or newer required"); } if (test_util::newer_than(client, "8.0")) { - WARN( - "Skipping - test is skipped on MongoDB server 8.0 or newer pending updates for " - "DRIVERS-2776"); - return; + SKIP("skipped on MongoDB server 8.0 or newer pending updates for DRIVERS-2776"); } if (test_util::get_topology(client) == "single") { - WARN("Skipping - must not run against a standalone server"); - return; + SKIP("must not run against a standalone server"); } is_replica_set = test_util::get_topology(client) == "replicaset"; @@ -3184,8 +3127,7 @@ TEST_CASE("Range Explicit Encryption", "[client_side_encryption]") { DYNAMIC_SECTION("Field Type - " << type_str) { if (field_type == RangeFieldType::DecimalNoPrecision && !is_replica_set) { - WARN("Skipping - must only run against a replica set"); - continue; + SKIP(type_str << ": must only run against a replica set"); } auto test_objects = range_explicit_encryption_setup(type_str, field_type); @@ -3553,9 +3495,7 @@ TEST_CASE("16. Rewrap. Case 2: RewrapManyDataKeyOpts.provider is not optional", "[client_side_encryption]") { instance::current(); - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); auto keyvault_client = mongocxx::client(mongocxx::uri(), test_util::add_test_server_api()); auto ce_opts = mongocxx::options::client_encryption(); diff --git a/src/mongocxx/test/collection.cpp b/src/mongocxx/test/collection.cpp index d9bdd6b8db..95adeab561 100644 --- a/src/mongocxx/test/collection.cpp +++ b/src/mongocxx/test/collection.cpp @@ -50,7 +50,6 @@ using bsoncxx::builder::basic::make_array; using bsoncxx::builder::basic::make_document; using namespace mongocxx; -using test_util::server_has_sessions; TEST_CASE("A default constructed collection cannot perform operations", "[collection]") { instance::current(); @@ -218,8 +217,7 @@ TEST_CASE("CRUD functionality", "[driver::collection]") { SECTION("unacknowledged write concern returns disengaged optional", "[collection]") { if (test_util::get_max_wire_version(mongodb_client) > 13) { - WARN("Skipping - getLastError removed in SERVER-57390"); - return; + SKIP("getLastError removed in SERVER-57390"); } collection coll = db["insert_one_unack_write"]; coll.drop(); @@ -349,8 +347,7 @@ TEST_CASE("CRUD functionality", "[driver::collection]") { SECTION("unacknowledged write concern returns disengaged optional") { if (test_util::get_max_wire_version(mongodb_client) > 13) { - WARN("Skipping - getLastError removed in SERVER-57390"); - return; + SKIP("getLastError removed in SERVER-57390"); } collection coll = db["insert_many_unack_write"]; coll.drop(); @@ -542,8 +539,7 @@ TEST_CASE("CRUD functionality", "[driver::collection]") { SECTION("update_one can take a pipeline", "[collection]") { if (!test_util::newer_than(mongodb_client, "4.1.11")) { - WARN("skip: pipeline updates require 4.1.11"); - return; + SKIP("pipeline updates require 4.1.11"); } collection coll = db["update_one_pipeline"]; @@ -613,8 +609,7 @@ TEST_CASE("CRUD functionality", "[driver::collection]") { SECTION("unacknowledged write concern returns disengaged optional") { if (test_util::get_max_wire_version(mongodb_client) > 13) { - WARN("Skipping - getLastError removed in SERVER-57390"); - return; + SKIP("getLastError removed in SERVER-57390"); } collection coll = db["update_one_unack_write"]; coll.drop(); @@ -725,8 +720,7 @@ TEST_CASE("CRUD functionality", "[driver::collection]") { SECTION("unacknowledged write concern returns disengaged optional") { if (test_util::get_max_wire_version(mongodb_client) > 13) { - WARN("Skipping - getLastError removed in SERVER-57390"); - return; + SKIP("getLastError removed in SERVER-57390"); } collection coll = db["update_many_unack_write"]; coll.drop(); @@ -897,8 +891,7 @@ TEST_CASE("CRUD functionality", "[driver::collection]") { SECTION("unacknowledged write concern returns disengaged optional") { if (test_util::get_max_wire_version(mongodb_client) > 13) { - WARN("Skipping - getLastError removed in SERVER-57390"); - return; + SKIP("getLastError removed in SERVER-57390"); } collection coll = db["replace_one_unack_write"]; coll.drop(); @@ -1033,8 +1026,7 @@ TEST_CASE("CRUD functionality", "[driver::collection]") { SECTION("unacknowledged write concern returns disengaged optional") { if (test_util::get_max_wire_version(mongodb_client) > 13) { - WARN("Skipping - getLastError removed in SERVER-57390"); - return; + SKIP("getLastError removed in SERVER-57390"); } collection coll = db["delete_one_unack_write"]; coll.drop(); @@ -1134,8 +1126,7 @@ TEST_CASE("CRUD functionality", "[driver::collection]") { SECTION("unacknowledged write concern returns disengaged optional") { if (test_util::get_max_wire_version(mongodb_client) > 13) { - WARN("Skipping - getLastError removed in SERVER-57390"); - return; + SKIP("getLastError removed in SERVER-57390"); } collection coll = db["delete_many_unack_write"]; coll.drop(); @@ -2009,8 +2000,7 @@ TEST_CASE("CRUD functionality", "[driver::collection]") { SECTION("unacknowledged write concern returns disengaged optional", "[collection]") { if (test_util::get_max_wire_version(mongodb_client) > 13) { - WARN("Skipping - getLastError removed in SERVER-57390"); - return; + SKIP("getLastError removed in SERVER-57390"); } collection coll = db["bulk_write_unack_write"]; coll.drop(); @@ -2589,8 +2579,7 @@ TEST_CASE("Ensure that the WriteConcernError 'errInfo' object is propagated", "[ if (test_util::get_topology(mongodb_client) == "sharded" && test_util::compare_versions(test_util::get_server_version(mongodb_client), "4.1.0") < 0) { - WARN("Skipping - failCommand on mongos requires 4.1+"); - return; + SKIP("failCommand on mongos requires 4.1+"); } using bsoncxx::builder::basic::sub_document; @@ -2693,8 +2682,7 @@ TEST_CASE("expose writeErrors[].errInfo", "[collection]") { auto mongodb_client = mongocxx::client(uri{}, client_opts); if (!test_util::newer_than(mongodb_client, "5.0")) { - WARN("skip: test requires MongoDB server 5.0 or newer"); - return; + SKIP("test requires MongoDB server 5.0 or newer"); } database db = mongodb_client["prose_test_expose_details"]; diff --git a/src/mongocxx/test/database.cpp b/src/mongocxx/test/database.cpp index ca40903ea5..ca920a1efe 100644 --- a/src/mongocxx/test/database.cpp +++ b/src/mongocxx/test/database.cpp @@ -39,7 +39,6 @@ using namespace mongocxx; using bsoncxx::builder::basic::kvp; using bsoncxx::builder::basic::make_array; using bsoncxx::builder::basic::make_document; -using test_util::server_has_sessions; bool check_for_collections(cursor cursor, std::set expected_colls) { for (auto&& coll : cursor) { @@ -342,9 +341,7 @@ TEST_CASE("Database integration tests", "[database]") { }; SECTION("listLocalSessions") { - if (!server_has_sessions(mongo_client)) { - return; - } + SERVER_HAS_SESSIONS_OR_SKIP(mongo_client); // SERVER-79306: Ensure the database exists for consistent behavior with sharded // clusters. diff --git a/src/mongocxx/test/index_view.cpp b/src/mongocxx/test/index_view.cpp index 70bed848aa..8273adcb49 100644 --- a/src/mongocxx/test/index_view.cpp +++ b/src/mongocxx/test/index_view.cpp @@ -179,8 +179,7 @@ TEST_CASE("create_one", "[index_view]") { SECTION("commitQuorum option") { if (test_util::get_topology(mongodb_client) == "single") { - WARN("skip: commitQuorum option requires a replica set"); - return; + SKIP("commitQuorum option requires a replica set"); } collection coll = db["index_view_create_one_commit_quorum"]; diff --git a/src/mongocxx/test/search_index_view.cpp b/src/mongocxx/test/search_index_view.cpp index ffb1b40d93..33198ad6ee 100644 --- a/src/mongocxx/test/search_index_view.cpp +++ b/src/mongocxx/test/search_index_view.cpp @@ -80,8 +80,7 @@ TEST_CASE("atlas search indexes prose tests", "[atlas][search_indexes]") { auto uri_getenv = std::getenv("MONGODB_URI"); if (!uri_getenv) { - WARN("Skipping - Test requires the environment variable: MONGODB_URI"); - return; + SKIP("Test requires the environment variable: MONGODB_URI"); } client mongodb_client{uri{uri_getenv}}; @@ -403,8 +402,7 @@ TEST_CASE("atlas search indexes tests", "[atlas][search_indexes]") { auto uri_getenv = std::getenv("MONGODB_URI"); if (!uri_getenv) { - WARN("Skipping - Test requires the environment variable: MONGODB_URI"); - return; + SKIP("Test requires the environment variable: MONGODB_URI"); } client mongodb_client{uri{uri_getenv}}; diff --git a/src/mongocxx/test/spec/client_side_encryption.cpp b/src/mongocxx/test/spec/client_side_encryption.cpp index cabcae0660..79797f6a6f 100644 --- a/src/mongocxx/test/spec/client_side_encryption.cpp +++ b/src/mongocxx/test/spec/client_side_encryption.cpp @@ -211,10 +211,7 @@ void run_encryption_tests_in_file(const std::string& test_path) { auto tests = test_spec_view["tests"].get_array().value; /* we may not have a supported topology */ - if (should_skip_spec_test(client{uri{}, test_util::add_test_server_api()}, test_spec_view)) { - WARN("File skipped - " + test_path); - return; - } + CHECK_IF_SKIP_SPEC_TEST((client{uri{}, test_util::add_test_server_api()}), test_spec_view); mongocxx::client setup_client{ uri{}, @@ -227,11 +224,9 @@ void run_encryption_tests_in_file(const std::string& test_path) { for (auto&& test : tests) { const auto description = string::to_string(test["description"].get_string().value); - SECTION(description) { - if (should_skip_spec_test(client{uri{}, test_util::add_test_server_api()}, - test.get_document().value)) { - continue; - } + DYNAMIC_SECTION(description) { + CHECK_IF_SKIP_SPEC_TEST((client{uri{}, test_util::add_test_server_api()}), + test.get_document().value); options::client client_opts; @@ -325,16 +320,14 @@ void run_encryption_tests_in_file(const std::string& test_path) { TEST_CASE("Client side encryption spec automated tests", "[client_side_encryption_spec]") { instance::current(); + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); + std::set unsupported_tests = { "badQueries.json", "count.json", "unsupportedCommand.json"}; char* encryption_tests_path = std::getenv("CLIENT_SIDE_ENCRYPTION_LEGACY_TESTS_PATH"); REQUIRE(encryption_tests_path); - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - return; - } - std::string path{encryption_tests_path}; if (path.back() == '/') { path.pop_back(); @@ -345,11 +338,10 @@ TEST_CASE("Client side encryption spec automated tests", "[client_side_encryptio std::string test_file; while (std::getline(test_files, test_file)) { - SECTION(test_file) { + DYNAMIC_SECTION(test_file) { if (std::find(unsupported_tests.begin(), unsupported_tests.end(), test_file) != unsupported_tests.end()) { - WARN("skipping " << test_file); - continue; + SKIP("unsupported test file: " << test_file); } run_encryption_tests_in_file(path + "/" + test_file); diff --git a/src/mongocxx/test/spec/command_monitoring.cpp b/src/mongocxx/test/spec/command_monitoring.cpp index c72671e2c6..0c1b049e38 100644 --- a/src/mongocxx/test/spec/command_monitoring.cpp +++ b/src/mongocxx/test/spec/command_monitoring.cpp @@ -70,9 +70,8 @@ void run_command_monitoring_tests_in_file(std::string test_path) { // Use a separate client to check version info, so as not to interfere with APM client temp_client{uri{}, test_util::add_test_server_api()}; - if (spec::should_skip_spec_test(temp_client, test.get_document().value)) { - return; - } + + CHECK_IF_SKIP_SPEC_TEST(temp_client, test.get_document().value); // Used by the listeners auto events = expectations.begin(); diff --git a/src/mongocxx/test/spec/initial_dns_seedlist_discovery.cpp b/src/mongocxx/test/spec/initial_dns_seedlist_discovery.cpp index a79c234349..be3b4b3435 100644 --- a/src/mongocxx/test/spec/initial_dns_seedlist_discovery.cpp +++ b/src/mongocxx/test/spec/initial_dns_seedlist_discovery.cpp @@ -219,7 +219,7 @@ static void iterate_srv_max_hosts_tests(std::string dir, std::vectorview(); for (auto&& test : test_spec_view["tests"].get_array().value) { client client{get_uri(test.get_document().value), client_opts}; - if (should_skip_spec_test(client, test_spec_view)) { - return; - } + CHECK_IF_SKIP_SPEC_TEST(client, test_spec_view); INFO("Test description: " << test["description"].get_string().value); - if (should_skip_spec_test(client, test.get_document())) { - continue; - } + CHECK_IF_SKIP_SPEC_TEST(client, test.get_document()); auto get_value_or_default = [&](std::string key, std::string default_str) { if (test_spec_view[key]) { @@ -121,7 +117,7 @@ void run_retryable_reads_tests_in_file(std::string test_path) { } } -TEST_CASE("retryable reads spec tests", "[retryable_reads_spec]") { +TEST_CASE("retryable reads spec tests", "[retryable_reads_specs]") { instance::current(); std::set unsupported_tests{"gridfs-downloadByName.json", diff --git a/src/mongocxx/test/spec/unified_tests/runner.cpp b/src/mongocxx/test/spec/unified_tests/runner.cpp index 9534ef3e55..2c41907228 100644 --- a/src/mongocxx/test/spec/unified_tests/runner.cpp +++ b/src/mongocxx/test/spec/unified_tests/runner.cpp @@ -1141,27 +1141,24 @@ void run_tests(mongocxx::stdx::string_view test_description, document::view test for (const auto& ele : test["tests"].get_array().value) { const auto description = string::to_string(ele["description"].get_string().value); - SECTION(description) { + + DYNAMIC_SECTION(description) { { const auto iter = should_skip_test_cases.find({test_description, description}); if (iter != should_skip_test_cases.end()) { - WARN("test skipped: " << iter->second); - continue; + SKIP(test_description << ": " << description << ": unsupported test case"); } } if (!has_run_on_requirements(ele.get_document())) { - std::stringstream warning; - warning << "test skipped: " - << "none of the runOnRequirements were met" << std::endl - << to_json(ele["runOnRequirements"].get_array().value); - WARN(warning.str()); - continue; + SKIP(test_description << ": " << description + << ": none of the runOnRequirements were met: " + << to_json(ele["runOnRequirements"].get_array().value)); } if (ele["skipReason"]) { - WARN("Skip Reason: " + string::to_string(ele["skipReason"].get_string().value)); - continue; + SKIP(test_description << ": " << description << ": " + << string::to_string(ele["skipReason"].get_string().value)); } fail_point_guard_type fail_point_guard; @@ -1266,12 +1263,8 @@ void run_tests_in_file(const std::string& test_path) { } if (!has_run_on_requirements(test_spec_view)) { - std::stringstream warning; - warning << "file skipped: " << test_path << std::endl - << "none of the runOnRequirements were met" << std::endl - << to_json(test_spec_view["runOnRequirements"].get_array().value); - WARN(warning.str()); - return; + CAPTURE(to_json(test_spec_view["runOnRequirements"].get_array().value)); + SKIP(test_path << ": none of the runOnRequirements were met"); } const auto description = test_spec_view["description"].get_string().value; @@ -1284,7 +1277,7 @@ void run_tests_in_file(const std::string& test_path) { // Check the environment for the specified variable; if present, extract it // as a directory and run all the tests contained in the magic "test_files.txt" // file: -bool run_unified_format_tests_in_env_dir( +void run_unified_format_tests_in_env_dir( const std::string& env_path, const std::set& unsupported_tests = {}) { const char* p = std::getenv(env_path.c_str()); @@ -1304,72 +1297,67 @@ bool run_unified_format_tests_in_env_dir( instance::current(); for (std::string file; std::getline(files, file);) { - SECTION(file) { + DYNAMIC_SECTION(file) { if (unsupported_tests.find(file) != unsupported_tests.end()) { - WARN("Skipping unsupported test file: " << file); - } else { - run_tests_in_file(base_path + '/' + file); + SKIP("unsupported test file: " << file); } + + run_tests_in_file(base_path + '/' + file); } } - - return true; } -TEST_CASE("unified format spec automated tests", "[unified_format_spec]") { +TEST_CASE("unified format spec automated tests", "[unified_format_specs]") { const std::set unsupported_tests = { // Waiting on CDRIVER-3525 and CXX-2166. "valid-pass/entity-client-cmap-events.json", // Waiting on CDRIVER-3525 and CXX-2166. "valid-pass/assertNumberConnectionsCheckedOut.json"}; - CHECK(run_unified_format_tests_in_env_dir("UNIFIED_FORMAT_TESTS_PATH", unsupported_tests)); + run_unified_format_tests_in_env_dir("UNIFIED_FORMAT_TESTS_PATH", unsupported_tests); } -TEST_CASE("session unified format spec automated tests", "[unified_format_spec]") { - CHECK(run_unified_format_tests_in_env_dir("SESSION_UNIFIED_TESTS_PATH")); +TEST_CASE("session unified format spec automated tests", "[unified_format_specs]") { + run_unified_format_tests_in_env_dir("SESSION_UNIFIED_TESTS_PATH"); } -TEST_CASE("CRUD unified format spec automated tests", "[unified_format_spec]") { - CHECK(run_unified_format_tests_in_env_dir("CRUD_UNIFIED_TESTS_PATH")); +TEST_CASE("CRUD unified format spec automated tests", "[unified_format_specs]") { + run_unified_format_tests_in_env_dir("CRUD_UNIFIED_TESTS_PATH"); } -TEST_CASE("change streams unified format spec automated tests", "[unified_format_spec]") { - CHECK(run_unified_format_tests_in_env_dir("CHANGE_STREAMS_UNIFIED_TESTS_PATH")); +TEST_CASE("change streams unified format spec automated tests", "[unified_format_specs]") { + run_unified_format_tests_in_env_dir("CHANGE_STREAMS_UNIFIED_TESTS_PATH"); } -TEST_CASE("retryable reads unified format spec automated tests", "[unified_format_spec]") { - CHECK(run_unified_format_tests_in_env_dir("RETRYABLE_READS_UNIFIED_TESTS_PATH")); +TEST_CASE("retryable reads unified format spec automated tests", "[unified_format_specs]") { + run_unified_format_tests_in_env_dir("RETRYABLE_READS_UNIFIED_TESTS_PATH"); } -TEST_CASE("retryable writes unified format spec automated tests", "[unified_format_spec]") { - CHECK(run_unified_format_tests_in_env_dir("RETRYABLE_WRITES_UNIFIED_TESTS_PATH")); +TEST_CASE("retryable writes unified format spec automated tests", "[unified_format_specs]") { + run_unified_format_tests_in_env_dir("RETRYABLE_WRITES_UNIFIED_TESTS_PATH"); } -TEST_CASE("transactions unified format spec automated tests", "[unified_format_spec]") { - CHECK(run_unified_format_tests_in_env_dir("TRANSACTIONS_UNIFIED_TESTS_PATH")); +TEST_CASE("transactions unified format spec automated tests", "[unified_format_specs]") { + run_unified_format_tests_in_env_dir("TRANSACTIONS_UNIFIED_TESTS_PATH"); } -TEST_CASE("versioned API spec automated tests", "[unified_format_spec]") { - CHECK(run_unified_format_tests_in_env_dir("VERSIONED_API_TESTS_PATH")); +TEST_CASE("versioned API spec automated tests", "[unified_format_specs]") { + run_unified_format_tests_in_env_dir("VERSIONED_API_TESTS_PATH"); } -TEST_CASE("collection management spec automated tests", "[unified_format_spec]") { - CHECK(run_unified_format_tests_in_env_dir("COLLECTION_MANAGEMENT_TESTS_PATH")); +TEST_CASE("collection management spec automated tests", "[unified_format_specs]") { + run_unified_format_tests_in_env_dir("COLLECTION_MANAGEMENT_TESTS_PATH"); } -TEST_CASE("index management spec automated tests", "[unified_format_spec]") { - CHECK(run_unified_format_tests_in_env_dir("INDEX_MANAGEMENT_TESTS_PATH")); +TEST_CASE("index management spec automated tests", "[unified_format_specs]") { + run_unified_format_tests_in_env_dir("INDEX_MANAGEMENT_TESTS_PATH"); } // See: // https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/client-side-encryption.rst -TEST_CASE("client side encryption unified format spec automated tests", "[unified_format_spec]") { - if (!mongocxx::test_util::should_run_client_side_encryption_test()) { - WARN("Skipping - client side encryption unified tests"); - return; - } - CHECK(run_unified_format_tests_in_env_dir("CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS_PATH")); +TEST_CASE("client side encryption unified format spec automated tests", "[unified_format_specs]") { + CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP(); + run_unified_format_tests_in_env_dir("CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS_PATH"); } } // namespace diff --git a/src/mongocxx/test/spec/uri_options.cpp b/src/mongocxx/test/spec/uri_options.cpp index f2f8bc5105..9ac28805f9 100644 --- a/src/mongocxx/test/spec/uri_options.cpp +++ b/src/mongocxx/test/spec/uri_options.cpp @@ -101,7 +101,8 @@ TEST_CASE("uri_options::test_srv_options", "[uri_options]") { for (const auto& it : tests) { auto doc = it.get_document().value; auto test = URIOptionsTest::parse(doc); - SECTION(std::string(test.description)) { + + DYNAMIC_SECTION(test.description) { try { mongocxx::uri my_uri{test.uri}; REQUIRE(test.valid); diff --git a/src/mongocxx/test/spec/util.cpp b/src/mongocxx/test/spec/util.cpp index 4f029e7d87..79dd1cfeac 100644 --- a/src/mongocxx/test/spec/util.cpp +++ b/src/mongocxx/test/spec/util.cpp @@ -90,8 +90,8 @@ uint32_t error_code_from_name(string_view name) { } /* Called with the entire test file and individual tests. */ -bool should_skip_spec_test(const client& client, document::view test) { - std::set unsupported_tests = { +bool check_if_skip_spec_test_impl(const client& client, document::view test, std::string& reason) { + static const std::set unsupported_tests = { "CreateIndex and dropIndex omits default write concern", "MapReduce omits default write concern", "Deprecated count with empty collection", @@ -104,41 +104,55 @@ bool should_skip_spec_test(const client& client, document::view test) { "run command fails with explicit secondary read preference", }; - if (test["description"]) { - std::string description = std::string(test["description"].get_string().value); - if (unsupported_tests.find(description) != unsupported_tests.end()) { - UNSCOPED_INFO("Test skipped - " << description << "\n" - << "reason: unsupported in C++ driver"); + if (const auto description = test["description"]) { + const auto desc = std::string(description.get_string().value); + if (unsupported_tests.find(desc) != unsupported_tests.end()) { + reason.append(desc); + reason.append(": not supported by the C++ Driver"); return true; } } - if (test["skipReason"]) { - UNSCOPED_INFO("Test skipped - " << test["description"].get_string().value << "\n" - << "reason: " << test["skipReason"].get_string().value); + if (const auto skip_reason = test["skipReason"]) { + const auto desc = test["description"].get_string().value; + const auto str = skip_reason.get_string().value; + + reason.append(desc.data(), desc.size()); + reason.append(": "); + reason.append(str.data(), str.size()); + return true; } - auto run_mongohouse_tests = std::getenv("RUN_MONGOHOUSE_TESTS"); - if (run_mongohouse_tests && std::string(run_mongohouse_tests) == "ON") { - // mongohoused does not return `version` field in response to serverStatus. - // Exit early to run the test. - return false; + { + const auto run_mongohouse_tests = std::getenv("RUN_MONGOHOUSE_TESTS"); + + if (run_mongohouse_tests && std::string(run_mongohouse_tests) == "ON") { + // mongohoused does not return `version` field in response to serverStatus. + // Exit early to run the test. + return false; + } } - std::string server_version = test_util::get_server_version(client); + const auto server_version = test_util::get_server_version(client); - std::string topology = test_util::get_topology(client); + const auto topology = test_util::get_topology(client); if (test["ignore_if_server_version_greater_than"]) { - std::string max_server_version = bsoncxx::string::to_string( + const auto max_server_version = bsoncxx::string::to_string( test["ignore_if_server_version_greater_than"].get_string().value); + if (test_util::compare_versions(server_version, max_server_version) > 0) { + reason.append(server_version); + reason.append(" is greater than "); + reason.append(max_server_version); return true; } } auto should_skip = [&](document::view requirements) { + reason.clear(); + if (requirements["topology"]) { auto topologies = requirements["topology"].get_array().value; bool found = false; @@ -149,22 +163,30 @@ bool should_skip_spec_test(const client& client, document::view test) { } } if (!found) { + reason.append(topology); + reason.append(" is not present in the list of topologies"); return true; } } if (requirements["minServerVersion"]) { - auto min_server_version = + const auto min_server_version = string::to_string(requirements["minServerVersion"].get_string().value); if (test_util::compare_versions(server_version, min_server_version) < 0) { + reason.append(server_version); + reason.append(" is greater than minimum server version "); + reason.append(min_server_version); return true; } } if (requirements["maxServerVersion"]) { - auto max_server_version = + const auto max_server_version = string::to_string(requirements["maxServerVersion"].get_string().value); if (test_util::compare_versions(server_version, max_server_version) > 0) { + reason.append(server_version); + reason.append(" is greater than max server version "); + reason.append(max_server_version); return true; } } @@ -178,13 +200,14 @@ bool should_skip_spec_test(const client& client, document::view test) { return false; } } - } else if (!should_skip(test)) { - return false; + + reason.clear(); + reason.append("no runOn conditions are satisfied"); + + return true; } - UNSCOPED_INFO("Skipping - unsupported server version '" + server_version + "' with topology '" + - topology + "'"); - return true; + return should_skip(test); } void configure_fail_point(const client& client, document::view test) { @@ -564,7 +587,7 @@ void run_tests_in_suite(std::string ev, test_runner cb, std::set un std::string test_file; while (std::getline(test_files, test_file)) { - SECTION(test_file) { + DYNAMIC_SECTION(test_file) { if (unsupported_tests.find(test_file) != unsupported_tests.end()) { WARN("Skipping unsupported test file: " << test_file); } else { @@ -851,22 +874,17 @@ void run_transactions_tests_in_file(const std::string& test_path) { const auto tests = test_spec_view["tests"].get_array().value; /* we may not have a supported topology */ - if (should_skip_spec_test({uri{}, test_util::add_test_server_api()}, test_spec_view)) { - WARN("File skipped - " + test_path); - return; - } + CHECK_IF_SKIP_SPEC_TEST((client{uri{}, test_util::add_test_server_api()}), test_spec_view); for (auto&& test : tests) { const auto description = string::to_string(test["description"].get_string().value); - SECTION(description) { + DYNAMIC_SECTION(description) { client setup_client{get_uri(test.get_document().value), test_util::add_test_server_api()}; // Step 1: If the `skipReason` field is present, skip this test completely. - if (should_skip_spec_test(setup_client, test.get_document().value)) { - continue; - } + CHECK_IF_SKIP_SPEC_TEST(setup_client, test.get_document().value); // Steps 2-8. test_setup(test.get_document().value, test_spec_view); @@ -1018,16 +1036,13 @@ void run_crud_tests_in_file(const std::string& test_path, uri test_uri) { client client{std::move(test_uri), client_opts}; document::view test_spec_view = test_spec->view(); - if (should_skip_spec_test(client, test_spec_view)) { - return; - } + CHECK_IF_SKIP_SPEC_TEST(client, test_spec_view); for (auto&& test : test_spec_view["tests"].get_array().value) { auto description = test["description"].get_string().value; - SECTION(to_string(description)) { - if (should_skip_spec_test(client, test.get_document())) { - continue; - } + + DYNAMIC_SECTION(to_string(description)) { + CHECK_IF_SKIP_SPEC_TEST(client, test.get_document()); auto get_value_or_default = [&](std::string key, std::string default_str) { if (test_spec_view[key]) { diff --git a/src/mongocxx/test/spec/util.hh b/src/mongocxx/test/spec/util.hh index 5fbfaf236b..e2977997bc 100644 --- a/src/mongocxx/test/spec/util.hh +++ b/src/mongocxx/test/spec/util.hh @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -36,12 +37,16 @@ using namespace mongocxx; /// uint32_t error_code_from_name(stdx::string_view name); -/// -/// Returns true if this test should be skipped for any reason (for -/// example, if a skipReason is defined, or if the given topology is not -/// supported for this test. -/// -bool should_skip_spec_test(const client& client, document::view test); +bool check_if_skip_spec_test_impl(const client& client, document::view test, std::string& reason); + +#define CHECK_IF_SKIP_SPEC_TEST(client, test) \ + { \ + std::string reason; \ + if (mongocxx::spec::check_if_skip_spec_test_impl(client, test, reason)) { \ + SKIP(reason); \ + } \ + } \ + ((void)0) /// /// Configures the fail point described by test["failPoint"]. diff --git a/src/mongocxx/test/transactions.cpp b/src/mongocxx/test/transactions.cpp index 0adb4caa72..f07665b51c 100644 --- a/src/mongocxx/test/transactions.cpp +++ b/src/mongocxx/test/transactions.cpp @@ -36,11 +36,9 @@ TEST_CASE("Transaction tests", "[transactions]") { client mongodb_client{uri{}, test_util::add_test_server_api()}; if (!test_util::is_replica_set(mongodb_client)) { - WARN("Skipping: transactions tests require replica set"); - return; + SKIP("transactions tests require replica set"); } else if (test_util::get_max_wire_version(mongodb_client) < 7) { - WARN("Skipping - transactions tests require max wire version is >= 7"); - return; + SKIP("transactions tests require max wire version is >= 7"); } // The test run in first 3 SECTIONs below @@ -223,11 +221,9 @@ TEST_CASE("Transactions Documentation Examples", "[transactions]") { client client{uri{}, test_util::add_test_server_api()}; if (!test_util::is_replica_set(client)) { - WARN("Skipping: transactions tests require replica set"); - return; + SKIP("transactions tests require replica set"); } else if (test_util::get_max_wire_version(client) < 7) { - WARN("Skipping - transactions tests require max wire version is >= 7"); - return; + SKIP("transactions tests require max wire version is >= 7"); } /* Create necessary collections. */ @@ -443,13 +439,11 @@ TEST_CASE("Transactions Mongos Pinning Prose Tests", "[transactions]") { instance::current(); if (test_util::compare_versions(test_util::get_server_version(), "4.1.6") < 0) { - WARN("Skipping - requires server 4.1.6+"); - return; + SKIP("requires server 4.1.6+"); } if (test_util::get_topology() != "sharded") { - WARN("Skipping - requires sharded cluster topology"); - return; + SKIP("requires sharded cluster topology"); } // @require_mongos_count_at_least(2)