diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index ef989b70c2..84f2226a4b 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -461,6 +461,8 @@ jobs: --short_output_paths \ --gha_build \ ${additional_flags[*]} + echo "Key hash used for building:" + echo | keytool -list -v -alias androiddebugkey -keystore ~/.android/debug.keystore - name: Prepare results summary artifact if: ${{ !cancelled() }} shell: bash diff --git a/messaging/integration_test/src/integration_test.cc b/messaging/integration_test/src/integration_test.cc index 74b70c5698..1e53d27e7a 100644 --- a/messaging/integration_test/src/integration_test.cc +++ b/messaging/integration_test/src/integration_test.cc @@ -58,8 +58,9 @@ const char kRestEndpoint[] = "https://fcm.googleapis.com/fcm/send"; const char kNotificationLinkKey[] = "gcm.n.link"; const char kTestLink[] = "https://this-is-a-test-link/"; -// Give each operation approximately 120 seconds before failing. -const int kTimeoutSeconds = 120; +// Give each operation approximately 30 seconds before failing. +// Much longer than this and our FTL tests time out. +const int kTimeoutSeconds = 30; const char kTestingNotificationKey[] = "fcm_testing_notification"; using app_framework::LogDebug; @@ -83,6 +84,9 @@ class FirebaseMessagingTest : public FirebaseTest { void SetUp() override; void TearDown() override; + static bool InitializeMessaging(); + static void TerminateMessaging(); + // Create a request and heads for a test message (returning false if unable to // do so). send_to can be a FCM token or a topic subscription. bool CreateTestMessage( @@ -133,6 +137,15 @@ firebase::messaging::PollableListener* FirebaseMessagingTest::shared_listener_ = bool FirebaseMessagingTest::is_desktop_stub_; void FirebaseMessagingTest::SetUpTestSuite() { + ASSERT_TRUE(InitializeMessaging()); + + is_desktop_stub_ = false; +#if !defined(ANDROID) && !(defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) + is_desktop_stub_ = true; +#endif // !defined(ANDROID) && !(defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) +} + +bool FirebaseMessagingTest::InitializeMessaging() { LogDebug("Initialize Firebase App."); #if defined(__ANDROID__) @@ -173,18 +186,27 @@ void FirebaseMessagingTest::SetUpTestSuite() { WaitForCompletion(initializer.InitializeLastResult(), "Initialize"); - ASSERT_EQ(initializer.InitializeLastResult().error(), 0) + EXPECT_EQ(initializer.InitializeLastResult().error(), 0) << initializer.InitializeLastResult().error_message(); - LogDebug("Successfully initialized Firebase Cloud Messaging."); - is_desktop_stub_ = false; -#if !defined(ANDROID) && !(defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) - is_desktop_stub_ = true; -#endif // !defined(ANDROID) && !(defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) + if (initializer.InitializeLastResult().error() == 0) { + LogDebug("Successfully initialized Firebase Cloud Messaging."); + } + + return (initializer.InitializeLastResult().error() == 0); } void FirebaseMessagingTest::TearDownTestSuite() { LogDebug("All tests finished, cleaning up."); + + TerminateMessaging(); + + // On iOS/FTL, most or all of the tests are skipped, so add a delay so the app + // doesn't finish too quickly, as this makes test results flaky. + ProcessEvents(1000); +} + +void FirebaseMessagingTest::TerminateMessaging() { firebase::messaging::SetListener(nullptr); delete shared_listener_; shared_listener_ = nullptr; @@ -193,13 +215,10 @@ void FirebaseMessagingTest::TearDownTestSuite() { LogDebug("Shutdown Firebase Cloud Messaging."); firebase::messaging::Terminate(); + LogDebug("Shutdown Firebase App."); delete shared_app_; shared_app_ = nullptr; - - // On iOS/FTL, most or all of the tests are skipped, so add a delay so the app - // doesn't finish too quickly, as this makes test results flaky. - ProcessEvents(1000); } FirebaseMessagingTest::FirebaseMessagingTest() { @@ -362,15 +381,29 @@ TEST_F(FirebaseMessagingTest, TestReceiveToken) { SKIP_TEST_ON_ANDROID_EMULATOR; + FLAKY_TEST_SECTION_BEGIN(); + EXPECT_TRUE(RequestPermission()); EXPECT_TRUE(::firebase::messaging::IsTokenRegistrationOnInitEnabled()); - FLAKY_TEST_SECTION_BEGIN(); - EXPECT_TRUE(WaitForToken()); EXPECT_NE(*shared_token_, ""); + FLAKY_TEST_SECTION_RESET(); + + // This section will run after each failed flake attempt. If we failed to get + // a token, we might need to completely uninitialize messaging and + // reinitialize it. + LogInfo("Reinitializing FCM before retry..."); + TerminateMessaging(); + ProcessEvents(2000); // Pause a few seconds. + EXPECT_TRUE(InitializeMessaging()); + ProcessEvents(2000); // Pause a few seconds. + // Toggle SetTokenRegistrationOnInitEnabled. + firebase::messaging::SetTokenRegistrationOnInitEnabled(false); + firebase::messaging::SetTokenRegistrationOnInitEnabled(true); + FLAKY_TEST_SECTION_END(); } @@ -378,10 +411,8 @@ TEST_F(FirebaseMessagingTest, TestSubscribeAndUnsubscribe) { TEST_REQUIRES_USER_INTERACTION_ON_IOS; // TODO(b/196589796) Test fails on Android emulators and causes failures in - // our CI. Since we don't have a good way to deterine if the runtime is an - // emulator or real device, we should disable the test in CI until we find - // the cause of problem. - TEST_REQUIRES_USER_INTERACTION_ON_ANDROID; + // our CI. + SKIP_TEST_ON_ANDROID_EMULATOR; EXPECT_TRUE(RequestPermission()); EXPECT_TRUE(WaitForToken()); @@ -512,12 +543,16 @@ TEST_F(FirebaseMessagingTest, TestSendMessageToToken) { SKIP_TEST_ON_ANDROID_EMULATOR; - EXPECT_TRUE(RequestPermission()); - EXPECT_TRUE(WaitForToken()); - + // When deflaking this test, sometimes we get out of sync, so attempt #2 + // receives the message that was meant for attempt #1. By storing the previous + // unique ID, we can catch this situation and consume the extraneous message. + std::string previous_unique_id = "XXX"; + std::string unique_id; FLAKY_TEST_SECTION_BEGIN(); - std::string unique_id = GetUniqueMessageId(); + EXPECT_TRUE(RequestPermission()); + EXPECT_TRUE(WaitForToken()); + unique_id = GetUniqueMessageId(); const char kNotificationTitle[] = "Token Test"; const char kNotificationBody[] = "Token Test notification body"; SendTestMessage(shared_token()->c_str(), kNotificationTitle, @@ -527,7 +562,15 @@ TEST_F(FirebaseMessagingTest, TestSendMessageToToken) { {kNotificationLinkKey, kTestLink}}); LogDebug("Waiting for message."); firebase::messaging::Message message; + EXPECT_TRUE(WaitForMessage(&message)); + if (message.data["unique_id"] == previous_unique_id) { + // Flaky fix: We've received a leftover message from the previous attempt. + // Consume it and get another (with a short timeout), to see if it matches. + LogDebug( + "Message unique_id matches *previous* attempt, getting another..."); + EXPECT_TRUE(WaitForMessage(&message, 10)); + } EXPECT_EQ(message.data["unique_id"], unique_id); EXPECT_NE(message.notification, nullptr); if (message.notification) { @@ -536,6 +579,22 @@ TEST_F(FirebaseMessagingTest, TestSendMessageToToken) { } EXPECT_EQ(message.link, kTestLink); + FLAKY_TEST_SECTION_RESET(); + + previous_unique_id = unique_id; + + // This section will run after each failed flake attempt. If we failed to get + // a token, we might need to completely uninitialize messaging and + // reinitialize it. + LogInfo("Reinitializing FCM before retry..."); + TerminateMessaging(); + ProcessEvents(2000); // Pause a few seconds. + EXPECT_TRUE(InitializeMessaging()); + ProcessEvents(2000); // Pause a few seconds. + // Toggle SetTokenRegistrationOnInitEnabled. + firebase::messaging::SetTokenRegistrationOnInitEnabled(false); + firebase::messaging::SetTokenRegistrationOnInitEnabled(true); + FLAKY_TEST_SECTION_END(); } @@ -586,6 +645,16 @@ TEST_F(FirebaseMessagingTest, TestSendMessageToTopic) { // shouldn't have. EXPECT_FALSE(WaitForMessage(&message, 5)); + FLAKY_TEST_SECTION_RESET(); + + // This section will run after each failed flake attempt. If we failed to get + // a token, we might need to completely uninitialize messaging and + // reinitialize it. + LogInfo("Reinitializing FCM before retry..."); + TerminateMessaging(); + ProcessEvents(3000); // Pause a few seconds. + EXPECT_TRUE(InitializeMessaging()); + FLAKY_TEST_SECTION_END(); } diff --git a/testing/test_framework/src/firebase_test_framework.h b/testing/test_framework/src/firebase_test_framework.h index 8c2ce92d08..d5ca4bbc51 100644 --- a/testing/test_framework/src/firebase_test_framework.h +++ b/testing/test_framework/src/firebase_test_framework.h @@ -260,11 +260,18 @@ namespace firebase_test_framework { #define DEATHTEST_SIGABRT "" #endif +// clang-format off // Macros to surround a flaky section of your test. // If this section fails, it will retry several times until it succeeds. +// NOLINTNEXTLINE #define FLAKY_TEST_SECTION_BEGIN() RunFlakyTestSection([&]() { (void)0 -#define FLAKY_TEST_SECTION_END() \ - }) +// NOLINTNEXTLINE +#define FLAKY_TEST_SECTION_END() }) +// If you use FLAKY_TEST_SECTION_RESET, it will run the code in between this and +// FLAKY_TEST_SECTION_END after each failed flake attempt. +// NOLINTNEXTLINE +#define FLAKY_TEST_SECTION_RESET() }, [&]() { (void)0 +// clang-format on class FirebaseTest : public testing::Test { public: @@ -396,6 +403,25 @@ class FirebaseTest : public testing::Test { }); } + // This is the same as RunFlakyTestSection above, but it will call + // reset_function in between each flake attempt. + void RunFlakyTestSection(std::function flaky_test_section, + std::function reset_function) { + // Save the current state of test results. + auto saved_test_results = SaveTestPartResults(); + RunFlakyBlock([&]() { + RestoreTestPartResults(saved_test_results); + + flaky_test_section(); + + if (HasFailure()) { + reset_function(); + } + + return !HasFailure(); + }); + } + // Run an operation that returns a Future (via a callback), retrying with // exponential backoff if the operation fails. //