diff --git a/changelog.d/5-internal/xml-reports b/changelog.d/5-internal/xml-reports new file mode 100644 index 0000000000..5ab1e589de --- /dev/null +++ b/changelog.d/5-internal/xml-reports @@ -0,0 +1,9 @@ +All integration tests can generate XML reports. + +To generate the report in brig-integration, galley-integration, +cargohold-integration, gundeck-integration, stern-integration and the new +integration suite pass `--xml=` to generate the XML file. + +For spar-integration and federator-integration pass `-f junit` and set +`JUNIT_OUTPUT_DIRECTORY` and `JUNIT_SUITE_NAME` environment variables. The XML +report will be generated at `$JUNIT_OUTPUT_DIRECTORY/junit.xml`. diff --git a/integration/default.nix b/integration/default.nix index 1aa88b39fd..0ef73927c7 100644 --- a/integration/default.nix +++ b/integration/default.nix @@ -42,6 +42,8 @@ , proto-lens , random , raw-strings-qq +, regex-base +, regex-tdfa , retry , scientific , split @@ -59,6 +61,7 @@ , vector , websockets , wire-message-proto-lens +, xml , yaml }: mkDerivation { @@ -105,6 +108,8 @@ mkDerivation { proto-lens random raw-strings-qq + regex-base + regex-tdfa retry scientific split @@ -122,6 +127,7 @@ mkDerivation { vector websockets wire-message-proto-lens + xml yaml ]; license = lib.licenses.agpl3Only; diff --git a/integration/integration.cabal b/integration/integration.cabal index 0041bca361..25d5557e2a 100644 --- a/integration/integration.cabal +++ b/integration/integration.cabal @@ -130,6 +130,7 @@ library Testlib.Run Testlib.RunServices Testlib.Types + Testlib.XML build-depends: , aeson @@ -168,6 +169,8 @@ library , proto-lens , random , raw-strings-qq + , regex-base + , regex-tdfa , retry , scientific , split @@ -185,4 +188,5 @@ library , vector , websockets , wire-message-proto-lens + , xml , yaml diff --git a/integration/test/Testlib/Options.hs b/integration/test/Testlib/Options.hs index 2ba8fdafd9..094c595154 100644 --- a/integration/test/Testlib/Options.hs +++ b/integration/test/Testlib/Options.hs @@ -9,6 +9,7 @@ data TestOptions = TestOptions { includeTests :: [String], excludeTests :: [String], listTests :: Bool, + xmlReport :: Maybe FilePath, configFile :: String } @@ -32,6 +33,13 @@ parser = ) ) <*> switch (long "list" <> short 'l' <> help "Only list tests.") + <*> optional + ( strOption + ( long "xml" + <> metavar "FILE" + <> help "Generate XML report for the tests" + ) + ) <*> strOption ( long "config" <> short 'c' @@ -53,12 +61,16 @@ getOptions :: IO TestOptions getOptions = do defaultsInclude <- maybe [] (splitOn ",") <$> lookupEnv "TEST_INCLUDE" defaultsExclude <- maybe [] (splitOn ",") <$> lookupEnv "TEST_EXCLUDE" + defaultsXMLReport <- lookupEnv "TEST_XML" opts <- execParser optInfo pure opts { includeTests = includeTests opts `orFromEnv` defaultsInclude, - excludeTests = excludeTests opts `orFromEnv` defaultsExclude + excludeTests = excludeTests opts `orFromEnv` defaultsExclude, + xmlReport = xmlReport opts `orFromEnv` defaultsXMLReport } where - orFromEnv [] fromEnv = fromEnv - orFromEnv patterns _ = patterns + orFromEnv fromArgs fromEnv = + if null fromArgs + then fromEnv + else fromArgs diff --git a/integration/test/Testlib/Run.hs b/integration/test/Testlib/Run.hs index 777ad6ebcc..82c1b1eaaa 100644 --- a/integration/test/Testlib/Run.hs +++ b/integration/test/Testlib/Run.hs @@ -24,22 +24,11 @@ import Testlib.JSON import Testlib.Options import Testlib.Printing import Testlib.Types +import Testlib.XML import Text.Printf import UnliftIO.Async import Prelude -data TestReport = TestReport - { count :: Int, - failures :: [String] - } - deriving (Eq, Show) - -instance Semigroup TestReport where - TestReport s1 f1 <> TestReport s2 f2 = TestReport (s1 + s2) (f1 <> f2) - -instance Monoid TestReport where - mempty = TestReport 0 mempty - runTest :: GlobalEnv -> App a -> IO (Either String a) runTest ge action = lowerCodensity $ do env <- mkEnv ge @@ -55,16 +44,18 @@ pluralise :: Int -> String -> String pluralise 1 x = x pluralise _ x = x <> "s" -printReport :: TestReport -> IO () +printReport :: TestSuiteReport -> IO () printReport report = do - unless (null report.failures) $ putStrLn $ "----------" - putStrLn $ show report.count <> " " <> pluralise report.count "test" <> " run." - unless (null report.failures) $ do + let numTests = length report.cases + failures = filter (\testCase -> testCase.result /= TestSuccess) report.cases + numFailures = length failures + when (numFailures > 0) $ putStrLn $ "----------" + putStrLn $ show numTests <> " " <> pluralise numTests "test" <> " run." + when (numFailures > 0) $ do putStrLn "" - let numFailures = length report.failures putStrLn $ colored red (show numFailures <> " failed " <> pluralise numFailures "test" <> ": ") - for_ report.failures $ \name -> - putStrLn $ " - " <> name + for_ failures $ \testCase -> + putStrLn $ " - " <> testCase.name testFilter :: TestOptions -> String -> Bool testFilter opts n = included n && not (excluded n) @@ -105,7 +96,7 @@ main = do qualifiedName = module0 <> "." <> name in (qualifiedName, summary, full, action) - if opts.listTests then doListTests tests else runTests tests cfg + if opts.listTests then doListTests tests else runTests tests opts.xmlReport cfg createGlobalEnv :: FilePath -> IO GlobalEnv createGlobalEnv cfg = do @@ -123,8 +114,8 @@ createGlobalEnv cfg = do Just dir -> dir "galley" relPath pure genv0 {gRemovalKeyPath = path} -runTests :: [(String, x, y, App ())] -> FilePath -> IO () -runTests tests cfg = do +runTests :: [(String, x, y, App ())] -> Maybe FilePath -> FilePath -> IO () +runTests tests mXMLOutput cfg = do output <- newChan let displayOutput = readChan output >>= \case @@ -149,14 +140,15 @@ runTests tests cfg = do <> ") -----\n" <> err <> "\n" - pure (TestReport 1 [qname]) + pure (TestSuiteReport [TestCaseReport qname (TestFailure err) tm]) Right _ -> do writeOutput $ qname <> colored green " OK" <> " (" <> printTime tm <> ")" <> "\n" - pure (TestReport 1 []) + pure (TestSuiteReport [TestCaseReport qname TestSuccess tm]) writeChan output Nothing wait displayThread printReport report - unless (null report.failures) $ + mapM_ (saveXMLReport report) mXMLOutput + when (any (\testCase -> testCase.result /= TestSuccess) report.cases) $ exitFailure doListTests :: [(String, String, String, x)] -> IO () diff --git a/integration/test/Testlib/Types.hs b/integration/test/Testlib/Types.hs index 847b8eaa10..e247f58f98 100644 --- a/integration/test/Testlib/Types.hs +++ b/integration/test/Testlib/Types.hs @@ -26,6 +26,7 @@ import Data.Set qualified as Set import Data.String import Data.Text qualified as T import Data.Text.Encoding qualified as T +import Data.Time import Data.Word import GHC.Generics (Generic) import GHC.Records @@ -440,3 +441,17 @@ data BackendName allServices :: [Service] allServices = [minBound .. maxBound] + +newtype TestSuiteReport = TestSuiteReport {cases :: [TestCaseReport]} + deriving (Eq, Show) + deriving newtype (Semigroup, Monoid) + +data TestCaseReport = TestCaseReport + { name :: String, + result :: TestResult, + time :: NominalDiffTime + } + deriving (Eq, Show) + +data TestResult = TestSuccess | TestFailure String + deriving (Eq, Show) diff --git a/integration/test/Testlib/XML.hs b/integration/test/Testlib/XML.hs new file mode 100644 index 0000000000..35235dec4d --- /dev/null +++ b/integration/test/Testlib/XML.hs @@ -0,0 +1,60 @@ +module Testlib.XML where + +import Data.Array qualified as Array +import Data.Fixed +import Data.Time +import Testlib.Types +import Text.Regex.Base qualified as Regex +import Text.Regex.TDFA.String qualified as Regex +import Text.XML.Light +import Prelude + +saveXMLReport :: TestSuiteReport -> FilePath -> IO () +saveXMLReport report output = + writeFile output $ showTopElement $ xmlReport report + +xmlReport :: TestSuiteReport -> Element +xmlReport report = + unode + "testsuites" + ( Attr (unqual "name") "wire-server", + testSuiteElements + ) + where + testSuiteElements = + unode + "testsuite" + ( attrs, + map encodeTestCase report.cases + ) + attrs = + [ Attr (unqual "name") "integration", + Attr (unqual "tests") $ show $ length report.cases, + Attr (unqual "failures") $ show $ length $ filter (\testCase -> testCase.result /= TestSuccess) report.cases, + Attr (unqual "time") $ showFixed True $ nominalDiffTimeToSeconds $ sum $ map (.time) report.cases + ] + +encodeTestCase :: TestCaseReport -> Element +encodeTestCase TestCaseReport {..} = + unode "testcase" (attrs, content) + where + attrs = + [ Attr (unqual "name") name, + Attr (unqual "time") (showFixed True (nominalDiffTimeToSeconds time)) + ] + content = case result of + TestSuccess -> [] + TestFailure msg -> [failure msg] + failure msg = unode "failure" (blank_cdata {cdData = dropConsoleFormatting msg}) + + -- Drops ANSI control characters which might be used to set colors. + -- Including these breaks XML, there is not much point encoding them. + dropConsoleFormatting input = + let regex = Regex.makeRegex "\x1b\\[[0-9;]*[mGKHF]" :: Regex.Regex + matches = Regex.matchAll regex input + dropMatch (offset, len) input' = + let (begining, rest) = splitAt offset input' + (_, end) = splitAt len rest + in begining <> end + matchTuples = map (Array.! 0) matches + in foldr dropMatch input matchTuples diff --git a/nix/haskell-pins.nix b/nix/haskell-pins.nix index 291ea06526..f486cd7657 100644 --- a/nix/haskell-pins.nix +++ b/nix/haskell-pins.nix @@ -204,6 +204,22 @@ let sha256 = "sha256-htEIJY+LmIMACVZrflU60+X42/g14NxUyFM7VJs4E6w="; }; }; + # PR: https://github.com/ocharles/tasty-ant-xml/pull/32 + tasty-ant-xml = { + src = fetchgit { + url = "https://github.com/akshaymankar/tasty-ant-xml"; + rev = "34ff294d805e62e73678dccc0be9d3da13540fbe"; + sha256 = "sha256-+rHcS+BwEFsXqPAHX/KZDIgv9zfk1dZl0LlZJ57Com4="; + }; + }; + # PR: https://github.com/freckle/hspec-junit-formatter/pull/24 + hspec-junit-formatter = { + src = fetchgit { + url = "https://github.com/akshaymankar/hspec-junit-formatter"; + rev = "acec31822cc4f90489d9940bad23b3fd6d1d7c75"; + sha256 = "sha256-4xGW3KHQKbTL+6+Q/gzfaMBP+J0npUe7tP5ZCQCB5+s="; + }; + }; }; hackagePins = { # Major re-write upstream, we should get rid of this dependency rather than diff --git a/nix/manual-overrides.nix b/nix/manual-overrides.nix index 74c0da5615..3b4b8b9879 100644 --- a/nix/manual-overrides.nix +++ b/nix/manual-overrides.nix @@ -27,6 +27,9 @@ hself: hsuper: { wai-middleware-prometheus = hlib.doJailbreak hsuper.wai-middleware-prometheus; wai-predicates = hlib.markUnbroken hsuper.wai-predicates; + # PR with fix: https://github.com/freckle/hspec-junit-formatter/pull/23 + hspec-junit-formatter = hlib.markUnbroken (hlib.dontCheck hsuper.hspec-junit-formatter); + # Some test seems to be broken hsaml2 = hlib.dontCheck hsuper.hsaml2; saml2-web-sso = hlib.dontCheck hsuper.saml2-web-sso; diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal index b5024f2e86..8c16a8c817 100644 --- a/services/brig/brig.cabal +++ b/services/brig/brig.cabal @@ -595,6 +595,7 @@ executable brig-integration , spar , streaming-commons , tasty >=1.0 + , tasty-ant-xml , tasty-cannon >=0.3.4 , tasty-hunit >=0.2 , temporary >=1.2.1 diff --git a/services/brig/default.nix b/services/brig/default.nix index e5859de928..19022e4a0f 100644 --- a/services/brig/default.nix +++ b/services/brig/default.nix @@ -122,6 +122,7 @@ , streaming-commons , swagger2 , tasty +, tasty-ant-xml , tasty-cannon , tasty-hunit , tasty-quickcheck @@ -354,6 +355,7 @@ mkDerivation { spar streaming-commons tasty + tasty-ant-xml tasty-cannon tasty-hunit temporary diff --git a/services/brig/test/integration/Main.hs b/services/brig/test/integration/Main.hs index dcdef16cd5..13cd48c748 100644 --- a/services/brig/test/integration/Main.hs +++ b/services/brig/test/integration/Main.hs @@ -64,6 +64,9 @@ import System.Environment (withArgs) import System.Logger qualified as Logger import Test.Tasty import Test.Tasty.HUnit +import Test.Tasty.Ingredients +import Test.Tasty.Runners +import Test.Tasty.Runners.AntXML import Util import Util.Options import Util.Test @@ -165,7 +168,7 @@ runTests iConf brigOpts otherArgs = do mlsApi = MLS.tests mg b brigOpts oauthAPI = API.OAuth.tests mg db b n brigOpts - withArgs otherArgs . defaultMain + withArgs otherArgs . defaultMainWithIngredients (listingTests : (composeReporters antXMLRunner consoleTestReporter) : defaultIngredients) $ testGroup "Brig API Integration" $ [ testCase "sitemap" $ diff --git a/services/cargohold/cargohold.cabal b/services/cargohold/cargohold.cabal index 725699a95c..cbe5965a74 100644 --- a/services/cargohold/cargohold.cabal +++ b/services/cargohold/cargohold.cabal @@ -289,6 +289,7 @@ executable cargohold-integration , servant-client , tagged >=0.8 , tasty >=1.0 + , tasty-ant-xml , tasty-hunit >=0.9 , text >=1.1 , time >=1.5 diff --git a/services/cargohold/default.nix b/services/cargohold/default.nix index 38aed9f075..144a55f194 100644 --- a/services/cargohold/default.nix +++ b/services/cargohold/default.nix @@ -50,6 +50,7 @@ , servant-server , tagged , tasty +, tasty-ant-xml , tasty-hunit , text , time @@ -154,6 +155,7 @@ mkDerivation { servant-client tagged tasty + tasty-ant-xml tasty-hunit text time diff --git a/services/cargohold/test/integration/Main.hs b/services/cargohold/test/integration/Main.hs index 86da6b2644..4615fa52bf 100644 --- a/services/cargohold/test/integration/Main.hs +++ b/services/cargohold/test/integration/Main.hs @@ -30,7 +30,10 @@ import Imports hiding (local) import qualified Metrics import Options.Applicative import Test.Tasty +import Test.Tasty.Ingredients import Test.Tasty.Options +import Test.Tasty.Runners +import Test.Tasty.Runners.AntXML import TestSetup import Util.Test @@ -75,4 +78,6 @@ main = do [ Option (Proxy :: Proxy ServiceConfigFile), Option (Proxy :: Proxy IntegrationConfigFile) ] + : listingTests + : composeReporters antXMLRunner consoleTestReporter : defaultIngredients diff --git a/services/federator/default.nix b/services/federator/default.nix index 77517559a7..7aafe2dc58 100644 --- a/services/federator/default.nix +++ b/services/federator/default.nix @@ -23,6 +23,8 @@ , hinotify , HsOpenSSL , hspec +, hspec-core +, hspec-junit-formatter , http-client , http-client-tls , http-media @@ -125,6 +127,7 @@ mkDerivation { ]; executableHaskellDepends = [ aeson + async base bilge binary @@ -136,6 +139,8 @@ mkDerivation { exceptions HsOpenSSL hspec + hspec-core + hspec-junit-formatter http-client-tls http-types http2-manager diff --git a/services/federator/federator.cabal b/services/federator/federator.cabal index 53895622a4..4a5228e30c 100644 --- a/services/federator/federator.cabal +++ b/services/federator/federator.cabal @@ -275,6 +275,7 @@ executable federator-integration build-depends: aeson + , async , base , bilge , binary @@ -287,6 +288,8 @@ executable federator-integration , federator , HsOpenSSL , hspec + , hspec-core + , hspec-junit-formatter , http-client-tls , http-types , http2-manager diff --git a/services/federator/test/integration/Main.hs b/services/federator/test/integration/Main.hs index fb1183a592..d63572adf7 100644 --- a/services/federator/test/integration/Main.hs +++ b/services/federator/test/integration/Main.hs @@ -20,6 +20,7 @@ module Main ) where +import Control.Concurrent.Async import Imports import OpenSSL (withOpenSSL) import System.Environment (withArgs) @@ -27,6 +28,10 @@ import Test.Federator.IngressSpec qualified import Test.Federator.InwardSpec qualified import Test.Federator.Util (TestEnv, mkEnvFromOptions) import Test.Hspec +import Test.Hspec.Core.Format +import Test.Hspec.JUnit +import Test.Hspec.JUnit.Config.Env +import Test.Hspec.Runner main :: IO () main = withOpenSSL $ do @@ -34,7 +39,26 @@ main = withOpenSSL $ do env <- withArgs wireArgs mkEnvFromOptions -- withArgs hspecArgs . hspec $ do -- beforeAll (pure env) . afterAll destroyEnv $ Hspec.mkspec - withArgs hspecArgs . hspec $ mkspec env + cfg <- hspecConfig + withArgs hspecArgs . hspecWith cfg $ mkspec env + +hspecConfig :: IO Config +hspecConfig = do + junitConfig <- envJUnitConfig + pure $ + defaultConfig + { configAvailableFormatters = + ("junit", checksAndJUnitFormatter junitConfig) + : configAvailableFormatters defaultConfig + } + where + checksAndJUnitFormatter :: JUnitConfig -> FormatConfig -> IO Format + checksAndJUnitFormatter junitConfig config = do + junit <- junitFormat junitConfig config + let checksFormatter = fromJust (lookup "checks" $ configAvailableFormatters defaultConfig) + checks <- checksFormatter config + pure $ \event -> do + concurrently_ (junit event) (checks event) partitionArgs :: [String] -> ([String], [String]) partitionArgs = go [] [] diff --git a/services/galley/default.nix b/services/galley/default.nix index 6f114b7648..cc56a5b353 100644 --- a/services/galley/default.nix +++ b/services/galley/default.nix @@ -90,6 +90,7 @@ , streaming-commons , tagged , tasty +, tasty-ant-xml , tasty-cannon , tasty-hunit , tasty-quickcheck @@ -282,6 +283,7 @@ mkDerivation { streaming-commons tagged tasty + tasty-ant-xml tasty-cannon tasty-hunit temporary diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal index 91fd95fbc7..4aa9a1edf1 100644 --- a/services/galley/galley.cabal +++ b/services/galley/galley.cabal @@ -489,6 +489,7 @@ executable galley-integration , streaming-commons , tagged , tasty >=0.8 + , tasty-ant-xml , tasty-cannon >=0.3.2 , tasty-hunit >=0.9 , temporary diff --git a/services/galley/test/integration/Main.hs b/services/galley/test/integration/Main.hs index 6d465089cb..86617dfba5 100644 --- a/services/galley/test/integration/Main.hs +++ b/services/galley/test/integration/Main.hs @@ -49,7 +49,10 @@ import Options.Applicative import System.Logger.Class qualified as Logger import Test.Tasty import Test.Tasty.HUnit +import Test.Tasty.Ingredients +import Test.Tasty.Ingredients.Basic import Test.Tasty.Options +import Test.Tasty.Runners.AntXML import TestHelpers (test) import TestSetup import Util.Options @@ -84,6 +87,8 @@ runTests run = defaultMainWithIngredients ings $ [ Option (Proxy :: Proxy ServiceConfigFile), Option (Proxy :: Proxy IntegrationConfigFile) ] + : listingTests + : composeReporters antXMLRunner consoleTestReporter : defaultIngredients main :: IO () diff --git a/services/gundeck/default.nix b/services/gundeck/default.nix index 624a143c7e..18410fe998 100644 --- a/services/gundeck/default.nix +++ b/services/gundeck/default.nix @@ -59,6 +59,7 @@ , servant-server , tagged , tasty +, tasty-ant-xml , tasty-hunit , tasty-quickcheck , text @@ -171,6 +172,7 @@ mkDerivation { safe tagged tasty + tasty-ant-xml tasty-hunit text tinylog diff --git a/services/gundeck/gundeck.cabal b/services/gundeck/gundeck.cabal index 1327735f99..7a116433a4 100644 --- a/services/gundeck/gundeck.cabal +++ b/services/gundeck/gundeck.cabal @@ -303,6 +303,7 @@ executable gundeck-integration , safe , tagged , tasty >=1.0 + , tasty-ant-xml , tasty-hunit >=0.9 , text , tinylog diff --git a/services/gundeck/test/integration/Main.hs b/services/gundeck/test/integration/Main.hs index 6a9502c9c9..9ab372ede3 100644 --- a/services/gundeck/test/integration/Main.hs +++ b/services/gundeck/test/integration/Main.hs @@ -39,7 +39,10 @@ import OpenSSL (withOpenSSL) import Options.Applicative import System.Logger qualified as Logger import Test.Tasty +import Test.Tasty.Ingredients import Test.Tasty.Options +import Test.Tasty.Runners +import Test.Tasty.Runners.AntXML import TestSetup import Util.Options import Util.Test @@ -83,6 +86,8 @@ runTests run = defaultMainWithIngredients ings $ [ Option (Proxy :: Proxy ServiceConfigFile), Option (Proxy :: Proxy IntegrationConfigFile) ] + : listingTests + : composeReporters antXMLRunner consoleTestReporter : defaultIngredients main :: IO () diff --git a/services/spar/default.nix b/services/spar/default.nix index 4bd791dfcf..3f7e41e8d1 100644 --- a/services/spar/default.nix +++ b/services/spar/default.nix @@ -5,6 +5,7 @@ { mkDerivation , aeson , aeson-qq +, async , base , base64-bytestring , bilge @@ -27,7 +28,9 @@ , hscim , HsOpenSSL , hspec +, hspec-core , hspec-discover +, hspec-junit-formatter , hspec-wai , http-api-data , http-client @@ -136,6 +139,7 @@ mkDerivation { executableHaskellDepends = [ aeson aeson-qq + async base base64-bytestring bilge @@ -156,6 +160,8 @@ mkDerivation { hscim HsOpenSSL hspec + hspec-core + hspec-junit-formatter hspec-wai http-api-data http-client diff --git a/services/spar/spar.cabal b/services/spar/spar.cabal index 568034cedd..7cad30f4ec 100644 --- a/services/spar/spar.cabal +++ b/services/spar/spar.cabal @@ -313,6 +313,7 @@ executable spar-integration build-depends: aeson , aeson-qq + , async , base , base64-bytestring , bilge @@ -331,6 +332,8 @@ executable spar-integration , hscim , HsOpenSSL , hspec + , hspec-core + , hspec-junit-formatter , hspec-wai , http-api-data , http-client diff --git a/services/spar/test-integration/Main.hs b/services/spar/test-integration/Main.hs index b42bb8c66b..3eefa283be 100644 --- a/services/spar/test-integration/Main.hs +++ b/services/spar/test-integration/Main.hs @@ -29,6 +29,7 @@ -- the solution: https://github.com/hspec/hspec/pull/397. module Main where +import Control.Concurrent.Async import Control.Lens ((.~), (^.)) import Data.Text (pack) import Imports @@ -37,6 +38,10 @@ import Spar.Run (mkApp) import System.Environment (withArgs) import System.Random (randomRIO) import Test.Hspec +import Test.Hspec.Core.Format +import Test.Hspec.Core.Runner +import Test.Hspec.JUnit +import Test.Hspec.JUnit.Config.Env import qualified Test.LoggingSpec import qualified Test.MetricsSpec import qualified Test.Spar.APISpec @@ -53,7 +58,8 @@ main :: IO () main = do (wireArgs, hspecArgs) <- partitionArgs <$> getArgs let env = withArgs wireArgs mkEnvFromOptions - withArgs hspecArgs . hspec $ do + cfg <- hspecConfig + withArgs hspecArgs . hspecWith cfg $ do for_ [minBound ..] $ \idpApiVersion -> do describe (show idpApiVersion) . beforeAll (env <&> teWireIdPAPIVersion .~ idpApiVersion) . afterAll destroyEnv $ do mkspecMisc @@ -61,6 +67,24 @@ main = do mkspecScim mkspecHscimAcceptance env destroyEnv +hspecConfig :: IO Config +hspecConfig = do + junitConfig <- envJUnitConfig + pure $ + defaultConfig + { configAvailableFormatters = + ("junit", checksAndJUnitFormatter junitConfig) + : configAvailableFormatters defaultConfig + } + where + checksAndJUnitFormatter :: JUnitConfig -> FormatConfig -> IO Format + checksAndJUnitFormatter junitConfig config = do + junit <- junitFormat junitConfig config + let checksFormatter = fromJust (lookup "checks" $ configAvailableFormatters defaultConfig) + checks <- checksFormatter config + pure $ \event -> do + concurrently_ (junit event) (checks event) + partitionArgs :: [String] -> ([String], [String]) partitionArgs = go [] [] where diff --git a/tools/stern/default.nix b/tools/stern/default.nix index c8c64c0d78..db69557460 100644 --- a/tools/stern/default.nix +++ b/tools/stern/default.nix @@ -40,6 +40,7 @@ , swagger2 , tagged , tasty +, tasty-ant-xml , tasty-hunit , text , tinylog @@ -119,6 +120,7 @@ mkDerivation { schema-profunctor tagged tasty + tasty-ant-xml tasty-hunit text tinylog diff --git a/tools/stern/stern.cabal b/tools/stern/stern.cabal index 0a4be042c5..96a8738ff2 100644 --- a/tools/stern/stern.cabal +++ b/tools/stern/stern.cabal @@ -270,6 +270,7 @@ executable stern-integration , stern , tagged , tasty >=0.8 + , tasty-ant-xml , tasty-hunit >=0.9 , text , tinylog diff --git a/tools/stern/test/integration/Main.hs b/tools/stern/test/integration/Main.hs index 6c95115b87..3acef76603 100644 --- a/tools/stern/test/integration/Main.hs +++ b/tools/stern/test/integration/Main.hs @@ -34,7 +34,10 @@ import OpenSSL (withOpenSSL) import Options.Applicative import System.Logger qualified as Logger import Test.Tasty +import Test.Tasty.Ingredients import Test.Tasty.Options +import Test.Tasty.Runners +import Test.Tasty.Runners.AntXML import TestSetup import Util.Options (Endpoint (Endpoint)) import Util.Test @@ -74,6 +77,8 @@ runTests run = defaultMainWithIngredients ings $ [ Option (Proxy :: Proxy ServiceConfigFile), Option (Proxy :: Proxy IntegrationConfigFile) ] + : listingTests + : composeReporters antXMLRunner consoleTestReporter : defaultIngredients main :: IO ()