Add pendingTestCase command and use it.

Makefile

@@ -94,11 +94,16 @@ ci: c db-migrate
 .PHONY: sanitize-pr
 sanitize-pr:
 	./hack/bin/generate-local-nix-packages.sh
-	make formatf-all
-	make hlint-inplace-all
+	make formatf
+	make hlint-inplace-pr
 	make git-add-cassandra-schema
 	@git diff-files --quiet -- || ( echo "There are unstaged changes, please take a look, consider committing them, and try again."; exit 1 )
 	@git diff-index --quiet --cached HEAD -- || ( echo "There are staged changes, please take a look, consider committing them, and try again."; exit 1 )
+	make list-flaky-tests
+
+list-flaky-tests:
+	@echo -e "\n\nif you want to run these, set RUN_FLAKY_TESTS=1\n\n"
+	@git grep -Hn '\bflakyTestCase \"'
 
 .PHONY: cabal-fmt
 cabal-fmt:

libs/types-common/default.nix

@@ -102,6 +102,7 @@ mkDerivation {
     swagger2
     tagged
     tasty
+    tasty-hunit
     text
     time
     time-locale-compat

libs/types-common/src/Test/Tasty/Pending.hs

@@ -0,0 +1,30 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2023 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+-- | related: https://github.com/nomeata/tasty-expected-failure
+module Test.Tasty.Pending (flakyTestCase) where
+
+import Imports
+import System.IO.Unsafe (unsafePerformIO)
+import Test.Tasty
+import Test.Tasty.HUnit
+
+flakyTestCase :: TestName -> Assertion -> TestTree
+flakyTestCase name test = testCase name test'
+  where
+    test' = when (runthem == Just "1") test
+    runthem = unsafePerformIO $ lookupEnv "RUN_FLAKY_TESTS"

libs/types-common/types-common.cabal

@@ -31,6 +31,7 @@ library
     Data.SizedHashMap
     Data.Text.Ascii
     Data.UUID.Tagged
+    Test.Tasty.Pending
     Util.Attoparsec
     Util.Logging
     Util.Options
@@ -126,6 +127,7 @@ library
     , swagger2
     , tagged                 >=0.8
     , tasty                  >=0.11
+    , tasty-hunit
     , text                   >=0.11
     , time                   >=1.6
     , time-locale-compat     >=0.1

services/federator/test/unit/Test/Federator/Remote.hs

@@ -39,6 +39,7 @@ import Test.Federator.Options (defRunSettings)
 import Test.Federator.Util
 import Test.Tasty
 import Test.Tasty.HUnit
+import Test.Tasty.Pending (flakyTestCase)
 import Wire.API.Federation.Component
 import Wire.API.Federation.Error
 import Wire.Network.DNS.SRV (SrvTarget (SrvTarget))
@@ -102,11 +103,11 @@ testValidatesCertificateSuccess :: TestTree
 testValidatesCertificateSuccess =
   testGroup
     "can get response with valid certificate"
-    [ testCase "when hostname=localhost and certificate-for=localhost" $
+    [ flakyTestCase "when hostname=localhost and certificate-for=localhost" $
         withMockServer certForLocalhost $ \port -> do
           tlsSettings <- mkTLSSettingsOrThrow settings
           assertNoRemoteError (mkTestCall tlsSettings port),
-      testCase "when hostname=localhost. and certificate-for=localhost" $
+      flakyTestCase "when hostname=localhost. and certificate-for=localhost" $
         withMockServer certForLocalhost $ \port -> do
           tlsSettings <- mkTLSSettingsOrThrow settings
           assertNoRemoteError (mkTestCall tlsSettings port),