Allow running non-deterministic MDX stanzas using dune-gen

CHANGES.md

@@ -4,6 +4,9 @@
 
 - Add support for adding language tags and metadata labels in `mli` files.
   (#339, #357, @julow, @Leonidas-from-XIV)
+- Add support for running non-deterministic tests in `dune` MDX 0.2 stanza by
+  setting the `MDX_RUN_NON_DETERMINISTIC` environment variable. (#365,
+  #366, @Leonidas-from-XIV)
 
 #### Changed
 

bin/dune_gen.ml

@@ -32,8 +32,13 @@ let run (`Setup ()) (`Prelude prelude) (`Directories dirs) =
   line "    ]";
   line "  in";
   line "  let predicates = Predicate.[ byte; toploop ] in";
+  line "  let non_deterministic =";
+  line "    match Sys.getenv_opt \"MDX_RUN_NON_DETERMINISTIC\" with";
+  line "    | Some _ -> true";
+  line "    | None -> false";
+  line "  in";
   line "  run_exn ~packages ~predicates ~prelude_str:[]";
-  line "    ~non_deterministic:false";
+  line "    ~non_deterministic";
   line "    ~silent_eval:false ~record_backtrace:false";
   line "    ~syntax:None ~silent:false";
   line "    ~verbose_findlib:false ~section:None";

test/bin/mdx-dune-gen/misc/basic/dune.gen.expected

@@ -9,8 +9,13 @@ let run_exn_defaults =
     ]
   in
   let predicates = Predicate.[ byte; toploop ] in
+  let non_deterministic =
+    match Sys.getenv_opt "MDX_RUN_NON_DETERMINISTIC" with
+    | Some _ -> true
+    | None -> false
+  in
   run_exn ~packages ~predicates ~prelude_str:[]
-    ~non_deterministic:false
+    ~non_deterministic
     ~silent_eval:false ~record_backtrace:false
     ~syntax:None ~silent:false
     ~verbose_findlib:false ~section:None

test/bin/mdx-dune-gen/misc/non-deterministic/dune

@@ -0,0 +1,40 @@
+(rule
+ (with-stdout-to
+  dune_gen.ml
+  (run ocaml-mdx dune-gen)))
+
+(executable
+ (name dune_gen)
+ (modules dune_gen)
+ (modes byte)
+ (libraries mdx.test))
+
+(rule
+ (with-stdout-to
+  dune-mdx-nondeterministic.deterministic
+  (run ./dune_gen.exe %{dep:dune-mdx-nondeterministic})))
+
+(rule
+ (setenv
+  MDX_RUN_NON_DETERMINISTIC
+  1
+  (with-stdout-to
+   dune-mdx-nondeterministic.nondeterministic
+   (run ./dune_gen.exe %{dep:dune-mdx-nondeterministic}))))
+
+(rule
+ (alias runtest)
+ (action
+  (diff dune-mdx-nondeterministic.expected
+    dune-mdx-nondeterministic.deterministic)))
+
+;; make sure the non-deterministic is different from the deterministic
+
+(rule
+ (alias runtest)
+ (action
+  (with-accepted-exit-codes
+   1
+   (ignore-stdout
+    (run diff %{dep:dune-mdx-nondeterministic.expected}
+      %{dep:dune-mdx-nondeterministic.nondeterministic})))))

test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic

@@ -0,0 +1,18 @@
+This test checks whether the non-deterministic mode works with the `dune` `mdx`
+stanza.
+
+Deterministic stanzas should get run and corrected, as for 1 plus one is not 3:
+
+```ocaml
+# 1 + 1;;
+- : int = 42
+```
+
+Non-deterministic ones should not be updated, since whatever `Random` outputs
+should be random:
+
+<!-- $MDX non-deterministic=command -->
+```ocaml
+# Random.int 1000;;
+- : int = 42
+```

test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic.expected

@@ -0,0 +1,18 @@
+This test checks whether the non-deterministic mode works with the `dune` `mdx`
+stanza.
+
+Deterministic stanzas should get run and corrected, as for 1 plus one is not 3:
+
+```ocaml
+# 1 + 1;;
+- : int = 2
+```
+
+Non-deterministic ones should not be updated, since whatever `Random` outputs
+should be random:
+
+<!-- $MDX non-deterministic=command -->
+```ocaml
+# Random.int 1000;;
+- : int = 42
+```