diff --git a/Expecto.FsCheck3/Expecto.FsCheck3.fsproj b/Expecto.FsCheck3/Expecto.FsCheck3.fsproj new file mode 100644 index 00000000..806d9917 --- /dev/null +++ b/Expecto.FsCheck3/Expecto.FsCheck3.fsproj @@ -0,0 +1,16 @@ + + + Expecto.FsCheck3 + Property testing for Expecto, powered by FsCheck3 + net6.0 + + + + + + + + + + + \ No newline at end of file diff --git a/Expecto.FsCheck3/FsCheck3.fs b/Expecto.FsCheck3/FsCheck3.fs new file mode 100644 index 00000000..eebe56e6 --- /dev/null +++ b/Expecto.FsCheck3/FsCheck3.fs @@ -0,0 +1,196 @@ +namespace Expecto + +open System +open FsCheck +open Expecto + +[] +module ExpectoFsCheck = + + let private propertyTest methodName focusState configs name property = + + /// This running plugs into FsCheck and gives us the ability to get called + /// by the framework on start/finish of fixtures. + let runner (config: FsCheckConfig) = + { new IRunner with + /// Called before a group of properties on a type are checked. + member __.OnStartFixture _ = + () + + /// Called whenever arguments are generated and after the test is run. + member __.OnArguments (testNumber, args, formatOnEvery) = + config.receivedArgs config name testNumber args + |> Async.RunSynchronously + + /// Called on a succesful shrink. + member __.OnShrink (values, formatValues) = + config.successfulShrink config name values + |> Async.RunSynchronously + + /// Called whenever all tests are done, either True, False or Exhausted. + member __.OnFinished (fsCheckTestName, testResult) = + config.finishedTest config fsCheckTestName + |> Async.RunSynchronously + + let numTests i = if i = 1 then "1 test" else sprintf "%i tests" i + + let stampsToString s = + let entry (p,xs) = sprintf "%A%s %s" p "%" (String.concat ", " xs) + match Seq.map entry s |> Seq.toList with + | [] -> "" + | [x] -> sprintf " (%s)\n" x + | xs -> sprintf "%s\n" (String.concat "\n" xs) + + match testResult with + | TestResult.Passed (_testData,_b) -> () + + | TestResult.Failed (_,_,_, Outcome.Failed (:? IgnoreException as e),_,_,_) -> + raise e + + | TestResult.Failed (data, original, shrunk, outcome,originalSeed,_finalSeed,size) -> + let parameters = + original + |> List.map (sprintf "%A") + |> String.concat " " + |> sprintf "Parameters:\n\t%s" + + let shrunk = + if data.NumberOfShrinks > 0 then + shrunk + |> List.map (sprintf "%A") + |> String.concat " " + |> sprintf "\nShrunk %i times to:\n\t%s" data.NumberOfShrinks + else "" + + let labels = + match data.Labels.Count with + | 0 -> String.Empty + | 1 -> sprintf "Label of failing property: %s\n" + (Set.toSeq data.Labels |> Seq.head) + | _ -> sprintf "Labels of failing property (one or more is failing): %s\n" + (String.concat " " data.Labels) + + let focus = + sprintf "Focus on error:\n\t%s (%i, %i) \"%s\"" methodName originalSeed.Seed originalSeed.Gamma name + + sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s" + (numTests data.NumberOfTests) parameters shrunk + outcome labels (stampsToString data.Stamps) focus + |> FailedException + |> raise + + | TestResult.Exhausted data -> + sprintf "Exhausted after %s%s" + (numTests data.NumberOfTests) (stampsToString data.Stamps) + |> FailedException + |> raise + } + + let test (config: FsCheckConfig) = + + let config = + Config.Default + .WithMaxTest(config.maxTest) + .WithReplay(Option.map (fun (seed,gamma) -> {Rnd = Rnd(uint64 seed, uint64 gamma); Size = None}) config.replay) + .WithName(name) + .WithStartSize(config.startSize) + .WithEndSize(config.endSize) + .WithQuietOnSuccess(true) + .WithArbitrary(config.arbitrary) + .WithRunner(runner config) + + + Check.One(config, property) + |> async.Return + + let testCode = + match configs with + | None -> + AsyncFsCheck (None, None, test) + | Some (testConfig, stressConfig) -> + AsyncFsCheck (Some testConfig, Some stressConfig, test) + + TestLabel(name, TestCase (testCode, focusState), focusState) + + /// Builds a test property with config + let testPropertyWithConfigs testConfig stressConfig name = + propertyTest "etestPropertyWithConfigs" Normal + (Some(testConfig, stressConfig)) name + + /// Builds an ignored test property with an explicit config. + let ptestPropertyWithConfigs testConfig stressConfig name = + propertyTest "etestPropertyWithConfigs" Pending + (Some (testConfig,stressConfig)) name + + /// Builds an ignored test property with an explicit config. + let ftestPropertyWithConfigs testConfig stressConfig name = + propertyTest "etestPropertyWithConfigs" Focused + (Some (testConfig,stressConfig)) name + + /// Builds a test property with config that will make Expecto to + /// ignore other unfocused tests and use an error stdGen. + let etestPropertyWithConfigs stdGen testConfig stressConfig name = + let testConfig = { testConfig with replay = Some stdGen } + let stressConfig = { stressConfig with replay = Some stdGen } + propertyTest "etestPropertyWithConfigs" Focused + (Some(testConfig,stressConfig)) name + + let testPropertyWithConfigsStdGen stdGen testConfig stressConfig name = + let testConfig = { testConfig with replay = Some stdGen } + let stressConfig = { stressConfig with replay = Some stdGen } + propertyTest "testPropertyWithConfigsStdGen" Normal (Some (testConfig,stressConfig)) name + + /// Builds a test property with config + let testPropertyWithConfig config name = + propertyTest "etestPropertyWithConfig" Normal (Some (config,config)) name + + /// Builds a test property with config that will be ignored by Expecto. + let ptestPropertyWithConfig config name = + propertyTest "etestPropertyWithConfig" Pending (Some(config,config)) name + + /// Builds a test property with config that will make Expecto + /// ignore other unfocused tests + let ftestPropertyWithConfig config name = + propertyTest "etestPropertyWithConfig" Focused (Some(config,config)) name + + /// Builds a test property with config that will make Expecto + /// ignore other unfocused tests and use an error stdGen. + let etestPropertyWithConfig stdGen config name = + let config = { config with replay = Some stdGen } + propertyTest "etestPropertyWithConfig" Focused (Some(config,config)) name + + /// Builds a test property with a config and a random seed number to ensure FsCheck runs identically every time. + let testPropertyWithConfigStdGen stdGen config name = + let config = { config with replay = Some stdGen } + propertyTest "testPropertyWithConfigStdGen" Normal (Some(config,config)) name + + /// Builds a test property. + let testProperty name = + propertyTest "etestProperty" Normal None name + + /// Builds a test property that will be ignored by Expecto. + let ptestProperty name = + propertyTest "etestProperty" Pending None name + + /// Builds a test property that will make Expecto to ignore other unfocused tests. + let ftestProperty name = + propertyTest "etestProperty" Focused None name + + /// Builds a test property that will make Expecto focus on this test use an error stdGen. + let etestProperty stdGen name = + let config = { FsCheckConfig.defaultConfig with replay = Some stdGen } + propertyTest "etestProperty" Focused (Some(config,config)) name + + +type FsCheck = + static member Property(name, property: Func<_,bool>) = + testProperty name property.Invoke + + static member Property(name, property: Func<_,_,bool>) = + testProperty name (fun a b -> property.Invoke(a,b)) + + static member Property(name, property: Func<_,_,_,bool>) = + testProperty name (fun a b c -> property.Invoke(a,b,c)) + + static member Property(name, property: Func<_,_,_,_,bool>) = + testProperty name (fun a b c d -> property.Invoke(a,b,c,d)) \ No newline at end of file diff --git a/Expecto.FsCheck3/paket.references b/Expecto.FsCheck3/paket.references new file mode 100644 index 00000000..481becee --- /dev/null +++ b/Expecto.FsCheck3/paket.references @@ -0,0 +1,2 @@ +group FsCheck3 +FsCheck \ No newline at end of file diff --git a/Expecto.Tests.FsCheck3/Expecto.Tests.FsCheck3.fsproj b/Expecto.Tests.FsCheck3/Expecto.Tests.FsCheck3.fsproj new file mode 100644 index 00000000..7a349cab --- /dev/null +++ b/Expecto.Tests.FsCheck3/Expecto.Tests.FsCheck3.fsproj @@ -0,0 +1,15 @@ + + + Expecto.Tests + Exe + net6.0 + + + + + + + + + + \ No newline at end of file diff --git a/Expecto.Tests.FsCheck3/FsCheck3Tests.fs b/Expecto.Tests.FsCheck3/FsCheck3Tests.fs new file mode 100644 index 00000000..0c234271 --- /dev/null +++ b/Expecto.Tests.FsCheck3/FsCheck3Tests.fs @@ -0,0 +1,147 @@ +module Expecto.FsCheckTests + +open Expecto +open Expecto.Impl +let properties = + + testList "FsCheck" [ + testProperty "Addition is commutative" <| + fun a b -> + a + b = b + a + + testProperty "Deliberately failing test" <| + fun a b c -> + // wrong on purpose to test failures + a * (b + c) = a * a + a * c + + testProperty "ignored" <| fun _ -> + skiptest "Because I feel like it." + + ptestProperty "ignored2" <| ignore + + ] + +[] +let runFsCheckTests = + testCaseAsync "run" <| async { + let! results = Impl.evalTestsSilent properties + Expect.equal results.Length 4 "results length" + + let getResult name = + results + |> Seq.filter (fun (r,_) -> r.name=name) + |> Seq.head + |> snd + + Expect.equal (getResult ["FsCheck";"Addition is commutative"]).result + TestResult.Passed "passed count" + + match (getResult ["FsCheck";"Deliberately failing test"]).result with + | TestResult.Failed _ -> + () + | x -> + failtestf "Expected Failed, actual %A" x + + match (getResult ["FsCheck";"ignored"]).result with + | TestResult.Ignored e -> + Expect.equal "Because I feel like it." e "It should fail with the right message." + | x -> + failtestf "Expected Ignored, actual %A" x + + match (getResult ["FsCheck";"ignored2"]).result with + | TestResult.Ignored _ -> + () + | x -> + failtestf "Expected Ignored, actual %A" x + + } + +let focused = + testList "FsCheck focused" [ + testCase "ignore me" <| ignore + + etestProperty (1,3) "Deliberately failing test" <| + fun a b c -> + // wrong on purpose to test failures + a * (b + c) = a * a + a * c + ] + +[] +let runFsCheckFocusedTests = + testCaseAsync "focused" <| async { + let! results = Impl.evalTestsSilent focused + Expect.equal results.Length 2 "results length" + + let getResult name = + results + |> Seq.filter (fun (r,_) -> r.name=name) + |> Seq.head + |> snd + + match (getResult ["FsCheck focused";"ignore me"]).result with + | TestResult.Ignored _ -> + () + | x -> + failtestf "Expected Ignored, actual %A" x + + match (getResult ["FsCheck focused";"Deliberately failing test"]).result with + | TestResult.Failed actual -> + let expected = " +Failed after 3 tests. Parameters: + -1 1 2 +Shrunk 2 times to: + -1 0 0 +Result: + Failed System.Exception: Expected true, got false. +Focus on error: + etestProperty (1, 3) \"Deliberately failing test\"" + Expect.equal actual expected "It should fail with the right message" + | x -> + failtestf "Expected Failed, actual was: %A" x + } + +let config = + testList "FsCheck config" [ + testCase "ignore me" ignore + + etestPropertyWithConfig (1,3) FsCheckConfig.defaultConfig + "Deliberately failing test" <| + fun a b c -> + // wrong on purpose to test failures + a * (b + c) = a * a + a * c + ] + +[] +let runFsCheckConfigTests = + testCaseAsync "config" <| async { + let! results = Impl.evalTestsSilent config + Expect.equal results.Length 2 "results length" + + let getResult name = + results + |> Seq.filter (fun (r,_) -> r.name=name) + |> Seq.head + |> snd + + match (getResult ["FsCheck config";"ignore me"]).result with + | TestResult.Ignored _ -> + () + | x -> + failtestf "Expected Ignored, actual %A" x + + match (getResult ["FsCheck config";"Deliberately failing test"]).result with + | TestResult.Failed actual -> + let expected = " +Failed after 3 tests. Parameters: + -1 1 2 +Shrunk 2 times to: + -1 0 0 +Result: + Failed System.Exception: Expected true, got false. +Focus on error: + etestPropertyWithConfig (1, 3) \"Deliberately failing test\"" + Expect.equal actual expected "It should fail with the right message." + + | x -> + failtestf "Expected Failed, actual %A" x + } \ No newline at end of file diff --git a/Expecto.Tests.FsCheck3/Main.fs b/Expecto.Tests.FsCheck3/Main.fs new file mode 100644 index 00000000..fae29267 --- /dev/null +++ b/Expecto.Tests.FsCheck3/Main.fs @@ -0,0 +1,14 @@ +module Main + +open Expecto +open Expecto.Logging + +let logger = Log.create "Expecto.Tests" + +[] +let main args = + let test = + Impl.testFromThisAssembly() + |> Option.orDefault (TestList ([], Normal)) + |> Test.shuffle "." + runTestsWithCLIArgs [NUnit_Summary "bin/Expecto.Tests.TestResults.xml"] args test diff --git a/Expecto.Tests.FsCheck3/paket.references b/Expecto.Tests.FsCheck3/paket.references new file mode 100644 index 00000000..481becee --- /dev/null +++ b/Expecto.Tests.FsCheck3/paket.references @@ -0,0 +1,2 @@ +group FsCheck3 +FsCheck \ No newline at end of file diff --git a/Expecto.sln b/Expecto.sln index 538cf1b7..bbf584c2 100644 --- a/Expecto.sln +++ b/Expecto.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.27130.2010 +# Visual Studio Version 17 +VisualStudioVersion = 17.4.33027.239 MinimumVisualStudioVersion = 10.0.40219.1 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Expecto", "Expecto\Expecto.fsproj", "{F82A6D5C-3B7C-4204-8A96-7D0AB8A67721}" EndProject @@ -21,15 +21,19 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Expecto.Hopac", "Expecto.Ho EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Expecto.Hopac.Tests", "Expecto.Hopac.Tests\Expecto.Hopac.Tests.fsproj", "{DF3DAA62-1273-47AB-ADEB-659492FFEE94}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Expecto.Diff", "Expecto.Diff\Expecto.Diff.fsproj", "{499DEA86-F089-4983-BB60-D54C22B6643D}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Expecto.Diff", "Expecto.Diff\Expecto.Diff.fsproj", "{499DEA86-F089-4983-BB60-D54C22B6643D}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{9B913138-FBD2-4216-95EC-BDEE468E94BA}" -ProjectSection(SolutionItems) = preProject - build.fsx = build.fsx - paket.dependencies = paket.dependencies - paket.lock = paket.lock - NuGet.props = NuGet.props -EndProjectSection + ProjectSection(SolutionItems) = preProject + build.fsx = build.fsx + NuGet.props = NuGet.props + paket.dependencies = paket.dependencies + paket.lock = paket.lock + EndProjectSection +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Expecto.FsCheck3", "Expecto.FsCheck3\Expecto.FsCheck3.fsproj", "{854FC486-3741-4910-A719-54D1C355B07D}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Expecto.Tests.FsCheck3", "Expecto.Tests.FsCheck3\Expecto.Tests.FsCheck3.fsproj", "{C82C1A73-8341-49E4-BBAF-7C39316963A6}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -77,6 +81,14 @@ Global {499DEA86-F089-4983-BB60-D54C22B6643D}.Debug|Any CPU.Build.0 = Debug|Any CPU {499DEA86-F089-4983-BB60-D54C22B6643D}.Release|Any CPU.ActiveCfg = Release|Any CPU {499DEA86-F089-4983-BB60-D54C22B6643D}.Release|Any CPU.Build.0 = Release|Any CPU + {854FC486-3741-4910-A719-54D1C355B07D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {854FC486-3741-4910-A719-54D1C355B07D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {854FC486-3741-4910-A719-54D1C355B07D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {854FC486-3741-4910-A719-54D1C355B07D}.Release|Any CPU.Build.0 = Release|Any CPU + {C82C1A73-8341-49E4-BBAF-7C39316963A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C82C1A73-8341-49E4-BBAF-7C39316963A6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C82C1A73-8341-49E4-BBAF-7C39316963A6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C82C1A73-8341-49E4-BBAF-7C39316963A6}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/build.fsx b/build.fsx index 3a905123..6cf08e42 100644 --- a/build.fsx +++ b/build.fsx @@ -38,6 +38,7 @@ BuildServer.install [ let libProjects = !! "Expecto/*.fsproj" ++ "Expecto.FsCheck/*.fsproj" + ++ "Expecto.FsCheck3/*.fsproj" ++ "Expecto.Diff/*.fsproj" ++ "Expecto.BenchmarkDotNet/*.fsproj" ++ "Expecto.Hopac/*.fsproj" @@ -47,6 +48,7 @@ let testProjects = ++ "Expecto.Hopac.Tests/*.Tests.fsproj" ++ "Expecto.Tests.CSharp/*.Tests.CSharp.csproj" ++ "Expecto.Focused.Tests/*.Tests.fsproj" + ++ "Expecto.Tests.FsCheck3/*.Tests.FsCheck3.fsproj" let benchmarkProjects = !! "Expecto.BenchmarkDotNet/*.fsproj" @@ -107,6 +109,7 @@ Target.create "RunTest" <| fun _ -> runTest "Expecto.Hopac.Tests" runTest "Expecto.Focused.Tests" + runTest "Expecto.Tests.FsCheck3" Target.create "Pack" <| fun _ -> let args = diff --git a/paket.dependencies b/paket.dependencies index b396a708..9f0c03af 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -10,6 +10,11 @@ nuget DiffPlex ~> 1.5 nuget Mono.Cecil ~> 0.11 nuget BenchmarkDotNet ~> 0.13.5 +group FsCheck3 + source https://api.nuget.org/v3/index.json + + nuget FsCheck ~> 3.0.0-beta2 + group Build source https://api.nuget.org/v3/index.json storage none diff --git a/paket.lock b/paket.lock index 192670c0..19c055cf 100644 --- a/paket.lock +++ b/paket.lock @@ -371,3 +371,10 @@ NUGET System.Runtime.CompilerServices.Unsafe (>= 4.5.3) - restriction: || (&& (== net6.0) (>= net461)) (&& (== net6.0) (< netcoreapp2.1)) (&& (== net6.0) (< netstandard1.0)) (&& (== net6.0) (< netstandard2.0)) (&& (== net6.0) (>= wp8)) (== netstandard2.1) System.Windows.Extensions (7.0) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) System.Drawing.Common (>= 7.0) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + +GROUP FsCheck3 +NUGET + remote: https://api.nuget.org/v3/index.json + FsCheck (3.0.0-beta2) + FSharp.Core (>= 4.5) - restriction: >= netstandard2.0 + FSharp.Core (7.0.200) - restriction: >= netstandard2.0