diff --git a/CHANGES.md b/CHANGES.md index c4d2d94f3..5b69c03ab 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ - Add support for adding language tags and metadata labels in `mli` files. (#339, #357, @julow, @Leonidas-from-XIV) +- Add support for running non-deterministic tests in `dune` MDX 0.2 stanza by + setting the `MDX_RUN_NON_DETERMINISTIC` environment variable. (#365, + #366, @Leonidas-from-XIV) #### Changed diff --git a/bin/dune_gen.ml b/bin/dune_gen.ml index 37a38668f..eac4249da 100644 --- a/bin/dune_gen.ml +++ b/bin/dune_gen.ml @@ -32,8 +32,13 @@ let run (`Setup ()) (`Prelude prelude) (`Directories dirs) = line " ]"; line " in"; line " let predicates = Predicate.[ byte; toploop ] in"; + line " let non_deterministic ="; + line " match Sys.getenv_opt \"MDX_RUN_NON_DETERMINISTIC\" with"; + line " | Some _ -> true"; + line " | None -> false"; + line " in"; line " run_exn ~packages ~predicates ~prelude_str:[]"; - line " ~non_deterministic:false"; + line " ~non_deterministic"; line " ~silent_eval:false ~record_backtrace:false"; line " ~syntax:None ~silent:false"; line " ~verbose_findlib:false ~section:None"; diff --git a/test/bin/mdx-dune-gen/misc/basic/dune.gen.expected b/test/bin/mdx-dune-gen/misc/basic/dune.gen.expected index fe37309a6..90973e682 100644 --- a/test/bin/mdx-dune-gen/misc/basic/dune.gen.expected +++ b/test/bin/mdx-dune-gen/misc/basic/dune.gen.expected @@ -9,8 +9,13 @@ let run_exn_defaults = ] in let predicates = Predicate.[ byte; toploop ] in + let non_deterministic = + match Sys.getenv_opt "MDX_RUN_NON_DETERMINISTIC" with + | Some _ -> true + | None -> false + in run_exn ~packages ~predicates ~prelude_str:[] - ~non_deterministic:false + ~non_deterministic ~silent_eval:false ~record_backtrace:false ~syntax:None ~silent:false ~verbose_findlib:false ~section:None diff --git a/test/bin/mdx-dune-gen/misc/non-deterministic/different.ml b/test/bin/mdx-dune-gen/misc/non-deterministic/different.ml new file mode 100644 index 000000000..ca6b2ed00 --- /dev/null +++ b/test/bin/mdx-dune-gen/misc/non-deterministic/different.ml @@ -0,0 +1,42 @@ +(* small helper to determine whether two files differ *) + +type comparison = Same | Different + +let rec compare first second = + match input_line first with + | first_line -> ( + match input_line second with + | second_line -> + match String.equal first_line second_line with + | true -> compare first second + | false -> + (* we found a difference between the lines *) + Different + | exception End_of_file -> + (* the second file ended before the first *) + Different) + | exception End_of_file -> + (* the first file ended first *) + match input_line second with + | _ -> + (* the second file continues: a difference *) + Different + | exception End_of_file -> + (* the second file ended too *) + Same + +let main () = + let first = Sys.argv.(1) |> open_in in + let second = Sys.argv.(2) |> open_in in + let comparison = compare first second in + close_in first; + close_in second; + match comparison with + | Same -> + prerr_endline "The files appear to be identical"; + (* we didn't find a difference, exit with a failure code *) + exit 1 + | Different -> () + +let () = + main () diff --git a/test/bin/mdx-dune-gen/misc/non-deterministic/dune b/test/bin/mdx-dune-gen/misc/non-deterministic/dune new file mode 100644 index 000000000..c697b7358 --- /dev/null +++ b/test/bin/mdx-dune-gen/misc/non-deterministic/dune @@ -0,0 +1,41 @@ +(rule + (with-stdout-to + dune_gen.ml + (run ocaml-mdx dune-gen))) + +(executable + (name dune_gen) + (modules dune_gen) + (modes byte) + (libraries mdx.test)) + +(rule + (with-stdout-to + dune-mdx-nondeterministic.deterministic + (run ./dune_gen.exe %{dep:dune-mdx-nondeterministic}))) + +(rule + (setenv + MDX_RUN_NON_DETERMINISTIC + 1 + (with-stdout-to + dune-mdx-nondeterministic.nondeterministic + (run ./dune_gen.exe %{dep:dune-mdx-nondeterministic})))) + +(rule + (alias runtest) + (action + (diff dune-mdx-nondeterministic.expected + dune-mdx-nondeterministic.deterministic))) + +;; make sure the non-deterministic is different from the deterministic + +(executable + (name different) + (modules different)) + +(rule + (alias runtest) + (action + (run ./different.exe %{dep:dune-mdx-nondeterministic.expected} + %{dep:dune-mdx-nondeterministic.nondeterministic}))) diff --git a/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic b/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic new file mode 100644 index 000000000..76dcef695 --- /dev/null +++ b/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic @@ -0,0 +1,18 @@ +This test checks whether the non-deterministic mode works with the `dune` `mdx` +stanza. + +Deterministic stanzas should get run and corrected, as for 1 plus one is not 3: + +```ocaml +# 1 + 1;; +- : int = 42 +``` + +Non-deterministic ones should not be updated, since whatever `Random` outputs +should be random: + + +```ocaml +# Random.int 1000;; +- : int = 42 +``` diff --git a/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic.expected b/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic.expected new file mode 100644 index 000000000..37c997232 --- /dev/null +++ b/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic.expected @@ -0,0 +1,18 @@ +This test checks whether the non-deterministic mode works with the `dune` `mdx` +stanza. + +Deterministic stanzas should get run and corrected, as for 1 plus one is not 3: + +```ocaml +# 1 + 1;; +- : int = 2 +``` + +Non-deterministic ones should not be updated, since whatever `Random` outputs +should be random: + + +```ocaml +# Random.int 1000;; +- : int = 42 +```