Allow running non-deterministic MDX stanzas using dune-gen

CHANGES.md

@@ -4,6 +4,9 @@
 
 - Add support for adding language tags and metadata labels in `mli` files.
   (#339, #357, @julow, @Leonidas-from-XIV)
+- Add support for running non-deterministic tests in `dune` MDX 0.2 stanza by
+  setting the `MDX_RUN_NON_DETERMINISTIC` environment variable. (#365,
+  #366, @Leonidas-from-XIV)
 
 #### Changed
 

bin/dune_gen.ml

@@ -32,8 +32,13 @@ let run (`Setup ()) (`Prelude prelude) (`Directories dirs) =
   line "    ]";
   line "  in";
   line "  let predicates = Predicate.[ byte; toploop ] in";
+  line "  let non_deterministic =";
+  line "    match Sys.getenv_opt \"MDX_RUN_NON_DETERMINISTIC\" with";
+  line "    | Some _ -> true";
+  line "    | None -> false";
+  line "  in";
   line "  run_exn ~packages ~predicates ~prelude_str:[]";
-  line "    ~non_deterministic:false";
+  line "    ~non_deterministic";
   line "    ~silent_eval:false ~record_backtrace:false";
   line "    ~syntax:None ~silent:false";
   line "    ~verbose_findlib:false ~section:None";

test/bin/mdx-dune-gen/misc/basic/dune.gen.expected

@@ -9,8 +9,13 @@ let run_exn_defaults =
     ]
   in
   let predicates = Predicate.[ byte; toploop ] in
+  let non_deterministic =
+    match Sys.getenv_opt "MDX_RUN_NON_DETERMINISTIC" with
+    | Some _ -> true
+    | None -> false
+  in
   run_exn ~packages ~predicates ~prelude_str:[]
-    ~non_deterministic:false
+    ~non_deterministic
     ~silent_eval:false ~record_backtrace:false
     ~syntax:None ~silent:false
     ~verbose_findlib:false ~section:None

test/bin/mdx-dune-gen/misc/non-deterministic/different.ml

@@ -0,0 +1,42 @@
+(* small helper to determine whether two files differ *)
+
+type comparison = Same | Different
+
+let rec compare first second =
+  match input_line first with
+  | first_line -> (
+      match input_line second with
+      | second_line ->
+          match String.equal first_line second_line with
+          | true -> compare first second
+          | false ->
+              (* we found a difference between the lines *)
+              Different
+      | exception End_of_file ->
+          (* the second file ended before the first *)
+          Different)
+  | exception End_of_file ->
+      (* the first file ended first *)
+      match input_line second with
+      | _ ->
+          (* the second file continues: a difference *)
+          Different
+      | exception End_of_file ->
+          (* the second file ended too *)
+          Same
+
+let main () =
+  let first = Sys.argv.(1) |> open_in in
+  let second = Sys.argv.(2) |> open_in in
+  let comparison = compare first second in
+  close_in first;
+  close_in second;
+  match comparison with
+  | Same ->
+    prerr_endline "The files appear to be identical";
+    (* we didn't find a difference, exit with a failure code *)
+    exit 1
+  | Different -> ()
+
+let () =
+  main ()

test/bin/mdx-dune-gen/misc/non-deterministic/dune

@@ -0,0 +1,41 @@
+(rule
+ (with-stdout-to
+  dune_gen.ml
+  (run ocaml-mdx dune-gen)))
+
+(executable
+ (name dune_gen)
+ (modules dune_gen)
+ (modes byte)
+ (libraries mdx.test))
+
+(rule
+ (with-stdout-to
+  dune-mdx-nondeterministic.deterministic
+  (run ./dune_gen.exe %{dep:dune-mdx-nondeterministic})))
+
+(rule
+ (setenv
+  MDX_RUN_NON_DETERMINISTIC
+  1
+  (with-stdout-to
+   dune-mdx-nondeterministic.nondeterministic
+   (run ./dune_gen.exe %{dep:dune-mdx-nondeterministic}))))
+
+(rule
+ (alias runtest)
+ (action
+  (diff dune-mdx-nondeterministic.expected
+    dune-mdx-nondeterministic.deterministic)))
+
+;; make sure the non-deterministic is different from the deterministic
+
+(executable
+ (name different)
+ (modules different))
+
+(rule
+ (alias runtest)
+ (action
+  (run ./different.exe %{dep:dune-mdx-nondeterministic.expected}
+    %{dep:dune-mdx-nondeterministic.nondeterministic})))

test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic

@@ -0,0 +1,18 @@
+This test checks whether the non-deterministic mode works with the `dune` `mdx`
+stanza.
+
+Deterministic stanzas should get run and corrected, as for 1 plus one is not 3:
+
+```ocaml
+# 1 + 1;;
+- : int = 42
+```
+
+Non-deterministic ones should not be updated, since whatever `Random` outputs
+should be random:
+
+<!-- $MDX non-deterministic=command -->
+```ocaml
+# Random.int 1000;;
+- : int = 42
+```

test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic.expected

@@ -0,0 +1,18 @@
+This test checks whether the non-deterministic mode works with the `dune` `mdx`
+stanza.
+
+Deterministic stanzas should get run and corrected, as for 1 plus one is not 3:
+
+```ocaml
+# 1 + 1;;
+- : int = 2
+```
+
+Non-deterministic ones should not be updated, since whatever `Random` outputs
+should be random:
+
+<!-- $MDX non-deterministic=command -->
+```ocaml
+# Random.int 1000;;
+- : int = 42
+```