Skip to content

Adjust avg and stddev computation and printing #326

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Mar 6, 2025
Merged

Adjust avg and stddev computation and printing #326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
ae31ea5
Add stddev summands in increasing order
jmid Mar 3, 2025
5d80161
Compute avg by summing posive and negative entries separately, by ord…
jmid Mar 3, 2025
0c718d3
Update expect test output
jmid Mar 3, 2025
cf59969
Switch to scientific notation for avg and stddev beyond +/-1e7
jmid Mar 3, 2025
584ead1
Update expect test outputs
jmid Mar 3, 2025
b7de376
Also trigger dune runtest on Windows
jmid Mar 3, 2025
e194b21
Workaround for Windows printing of float exponents
jmid Mar 3, 2025
947e464
Update 32-bit expect test outputs
jmid Mar 5, 2025
d7ac098
Add CHANGELOG entry
jmid Mar 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/workflows/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,3 @@ jobs:
- run: opam install . --deps-only --with-test
- run: opam exec -- dune build
- run: opam exec -- dune runtest
if: ${{ matrix.os != 'windows-latest'}}
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
## NEXT RELEASE

- Restore `Test.make`'s `max_fail` parameter which was accidentally broken in 0.18
- Adjust `stats` computation of average and standard deviation to
limit precision loss, print both using scientific notation, and
workaround MinGW float printing to also pass expect tests

## 0.24

Expand Down
2 changes: 1 addition & 1 deletion example/QCheck_runner_test.expected.ocaml4.32
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ stats dist:
+++ Stats for stat_display_test_7 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 100000, avg: 336840.90, stddev: 619301756.02, median 895228, min -1073728193, max 1073739280
num: 100000, avg: 336840.90, stddev: 6.193e+8, median 895228, min -1073728193, max 1073739280
-1073728193.. -966354820: ##################################################### 5009
-966354819.. -858981446: #################################################### 5004
-858981445.. -751608072: #################################################### 4917
Expand Down
2 changes: 1 addition & 1 deletion example/QCheck_runner_test.expected.ocaml4.64
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ stats dist:
+++ Stats for stat_display_test_7 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 100000, avg: 2541076923587387.50, stddev: 2660730801206827008.00, median 158655268318060, min -4611522359435274428, max 4611540922436307689
num: 100000, avg: 2.541e+15, stddev: 2.661e+18, median 158655268318060, min -4611522359435274428, max 4611540922436307689
-4611522359435274428..-4150369195341695293: ##################################################### 4976
-4150369195341695292..-3689216031248116157: ##################################################### 4963
-3689216031248116156..-3228062867154537021: ###################################################### 5038
Expand Down
2 changes: 1 addition & 1 deletion example/QCheck_runner_test.expected.ocaml5.32
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ stats dist:
+++ Stats for stat_display_test_7 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 100000, avg: -2481754.31, stddev: 618398387.27, median -5669677, min -1073719962, max 1073717275
num: 100000, avg: -2481754.31, stddev: 6.184e+8, median -5669677, min -1073719962, max 1073717275
-1073719962.. -966348101: ##################################################### 4978
-966348100.. -858976239: ##################################################### 5008
-858976238.. -751604377: #################################################### 4907
Expand Down
2 changes: 1 addition & 1 deletion example/QCheck_runner_test.expected.ocaml5.64
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ stats dist:
+++ Stats for stat_display_test_7 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 100000, avg: -7215552342607541.00, stddev: 2666234426234218496.00, median -16620417636667326, min -4611371852367564818, max 4611613630315464842
num: 100000, avg: -7.216e+15, stddev: 2.666e+18, median -16620417636667326, min -4611371852367564818, max 4611613630315464842
-4611371852367564818..-4150222578233413331: ##################################################### 5003
-4150222578233413330..-3689073304099261843: ####################################################### 5106
-3689073304099261842..-3227924029965110355: ###################################################### 5052
Expand Down
30 changes: 23 additions & 7 deletions src/core/QCheck2.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2031,26 +2031,35 @@ module Test = struct
let stat_max_lines = 20 (* maximum number of lines for a histogram *)

let print_stat ((name,_), tbl) =
let avg = ref 0. in
let neg_avg_summands = ref [] in
let pos_avg_summands = ref [] in
let num = ref 0 in
let min_idx, max_idx =
Hashtbl.fold
(fun i res (m1,m2) ->
avg := !avg +. float_of_int (i * res);
let avg_summand = float_of_int (i * res) in
if avg_summand < 0.
then neg_avg_summands := avg_summand::!neg_avg_summands
else pos_avg_summands := avg_summand::!pos_avg_summands;
num := !num + res;
min i m1, max i m2)
tbl (max_int,min_int)
in
(* compute average *)
(* compute average, summing positive/negative separately by order of magnitude *)
let neg_avg_sums = List.sort Float.compare !neg_avg_summands |> fun xs -> List.fold_right (+.) xs 0. in
let pos_avg_sums = List.sort Float.compare !pos_avg_summands |> List.fold_left (+.) 0. in
let avg = ref (neg_avg_sums +. pos_avg_sums) in
if !num > 0 then (
avg := !avg /. float_of_int !num
);
(* compute std-dev: sqroot of sum of squared distance-to-average
https://en.wikipedia.org/wiki/Standard_deviation *)
let stddev =
Hashtbl.fold
(fun i res m -> m +. (float_of_int i -. !avg) ** 2. *. float_of_int res)
tbl 0.
(fun i res acc -> float_of_int res *. ((float_of_int i -. !avg) ** 2.) :: acc)
tbl []
|> List.sort Float.compare (* add summands in increasing order to preserve precision *)
|> List.fold_left (+.) 0.
|> (fun s -> if !num>0 then s /. float_of_int !num else s)
|> sqrt
in
Expand Down Expand Up @@ -2091,10 +2100,17 @@ module Test = struct
max_val := max !max_val new_count) tbl;
(* print entries of the table, sorted by increasing index *)
let out = Buffer.create 128 in
(* Windows workaround to avoid annoying exponent zero such as "1.859e+018" *)
let cut_exp_zero s =
match String.split_on_char '+' s with
| [signif;exponent] -> Printf.sprintf "%s+%i" signif (int_of_string exponent)
| _ -> failwith "cut_exp_zero failed to parse scientific notation " ^ s in
let fmt_float f =
if f > 1e7 || f < -1e7 then cut_exp_zero (Printf.sprintf "%.3e" f) else Printf.sprintf "%.2f" f in
Printf.bprintf out "stats %s:\n" name;
Printf.bprintf out
" num: %d, avg: %.2f, stddev: %.2f, median %d, min %d, max %d\n"
!num !avg stddev !median min_idx max_idx;
" num: %d, avg: %s, stddev: %s, median %d, min %d, max %d\n"
!num (fmt_float !avg) (fmt_float stddev) !median min_idx max_idx;
let indwidth =
let str_width i = String.length (Printf.sprintf "%d" i) in
List.map str_width [min_idx; max_idx; min_idx + bucket_size * hist_size] |> List.fold_left max min_int in
Expand Down
6 changes: 3 additions & 3 deletions test/core/QCheck2_expect_test.expected.ocaml4.32
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1434,7 +1434,7 @@ stats dist:
+++ Stats for int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 100000, avg: -298652.90, stddev: 619096154.85, median 419404, min -1073741519, max 1073728237
num: 100000, avg: -298652.90, stddev: 6.191e+8, median 419404, min -1073741519, max 1073728237
-1073741519.. -966368032: ##################################################### 4984
-966368031.. -858994544: ###################################################### 5025
-858994543.. -751621056: ###################################################### 5035
Expand All @@ -1459,7 +1459,7 @@ stats dist:
+++ Stats for oneof int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 1073741.63, stddev: 676575790.92, median 0, min -1073741824, max 1073741823
num: 1000, avg: 1073741.63, stddev: 6.766e+8, median 0, min -1073741824, max 1073741823
-1073741824.. -966367642: ################## 208
-966367641.. -858993459: 0
-858993458.. -751619276: 0
Expand Down Expand Up @@ -1688,7 +1688,7 @@ random seed: 153870556
+++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 2229293.51, stddev: 427568354.78, median 9, min -1072726813, max 1073741823
num: 1000, avg: 2229293.51, stddev: 4.276e+8, median 9, min -1072726813, max 1073741823
-1072726813.. -965403382: ## 22
-965403381.. -858079950: ## 28
-858079949.. -750756518: # 17
Expand Down
6 changes: 3 additions & 3 deletions test/core/QCheck2_expect_test.expected.ocaml4.64
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1496,7 +1496,7 @@ stats dist:
+++ Stats for int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 100000, avg: 6739840024355437.00, stddev: 2654793546877646336.00, median 2435838602111153, min -4611682300221562449, max 4611492907363159042
num: 100000, avg: 6.740e+15, stddev: 2.655e+18, median 2435838602111153, min -4611682300221562449, max 4611492907363159042
-4611682300221562449..-4150523539842326354: #################################################### 4927
-4150523539842326353..-3689364779463090258: #################################################### 4923
-3689364779463090257..-3228206019083854162: #################################################### 4923
Expand All @@ -1521,7 +1521,7 @@ stats dist:
+++ Stats for oneof int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 4611686018427388.00, stddev: 2905870896563567616.00, median 0, min -4611686018427387904, max 4611686018427387903
num: 1000, avg: 4.612e+15, stddev: 2.906e+18, median 0, min -4611686018427387904, max 4611686018427387903
-4611686018427387904..-4150517416584649089: ################## 208
-4150517416584649088..-3689348814741910273: 0
-3689348814741910272..-3228180212899171457: 0
Expand Down Expand Up @@ -1750,7 +1750,7 @@ random seed: 153870556
+++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 48985084121559400.00, stddev: 1793541561270566400.00, median 9, min -4580864984167113344, max 4611686018427387903
num: 1000, avg: 4.899e+16, stddev: 1.794e+18, median 9, min -4580864984167113344, max 4611686018427387903
-4580864984167113344..-4121237434037388289: ## 23
-4121237434037388288..-3661609883907663233: # 19
-3661609883907663232..-3201982333777938177: ## 28
Expand Down
6 changes: 3 additions & 3 deletions test/core/QCheck2_expect_test.expected.ocaml5.32
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1417,7 +1417,7 @@ stats dist:
+++ Stats for int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 100000, avg: 1375300.54, stddev: 620110315.04, median -3852464, min -1073736753, max 1073733862
num: 100000, avg: 1375300.54, stddev: 6.201e+8, median -3852464, min -1073736753, max 1073733862
-1073736753.. -966363223: #################################################### 4972
-966363222.. -858989692: ##################################################### 5032
-858989691.. -751616161: #################################################### 4928
Expand All @@ -1442,7 +1442,7 @@ stats dist:
+++ Stats for oneof int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 1073741.60, stddev: 673131652.31, median 0, min -1073741824, max 1073741823
num: 1000, avg: 1073741.60, stddev: 6.731e+8, median 0, min -1073741824, max 1073741823
-1073741824.. -966367642: ################# 198
-966367641.. -858993459: 0
-858993458.. -751619276: 0
Expand Down Expand Up @@ -1672,7 +1672,7 @@ random seed: 153870556
+++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 12189159.05, stddev: 451294853.72, median 10, min -1073230792, max 1073741823
num: 1000, avg: 1.219e+7, stddev: 4.513e+8, median 10, min -1073230792, max 1073741823
-1073230792.. -965882162: ## 19
-965882161.. -858533531: ### 30
-858533530.. -751184900: ### 29
Expand Down
6 changes: 3 additions & 3 deletions test/core/QCheck2_expect_test.expected.ocaml5.64
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1479,7 +1479,7 @@ stats dist:
+++ Stats for int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 100000, avg: -16187567683715200.00, stddev: 2661643267167753216.00, median -19064865724378010, min -4611580124327955972, max 4611682453232767028
num: 100000, avg: -1.619e+16, stddev: 2.662e+18, median -19064865724378010, min -4611580124327955972, max 4611682453232767028
-4611580124327955972..-4150416995449919813: #################################################### 4939
-4150416995449919812..-3689253866571883653: ###################################################### 5114
-3689253866571883652..-3228090737693847493: ##################################################### 5019
Expand All @@ -1504,7 +1504,7 @@ stats dist:
+++ Stats for oneof int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 4611686018427388.00, stddev: 2891078433912002560.00, median 0, min -4611686018427387904, max 4611686018427387903
num: 1000, avg: 4.612e+15, stddev: 2.891e+18, median 0, min -4611686018427387904, max 4611686018427387903
-4611686018427387904..-4150517416584649089: ################# 198
-4150517416584649088..-3689348814741910273: 0
-3689348814741910272..-3228180212899171457: 0
Expand Down Expand Up @@ -1734,7 +1734,7 @@ random seed: 153870556
+++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 96259266356736624.00, stddev: 1858774986626273280.00, median 10, min -4602121864537469332, max 4611686018427387903
num: 1000, avg: 9.626e+16, stddev: 1.859e+18, median 10, min -4602121864537469332, max 4611686018427387903
-4602121864537469332..-4141431470389226517: # 17
-4141431470389226516..-3680741076240983701: ## 19
-3680741076240983700..-3220050682092740885: ## 19
Expand Down
6 changes: 3 additions & 3 deletions test/core/QCheck_expect_test.expected.ocaml4.32
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1411,7 +1411,7 @@ stats dist:
+++ Stats for int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 100000, avg: 336840.90, stddev: 619301756.02, median 895228, min -1073728193, max 1073739280
num: 100000, avg: 336840.90, stddev: 6.193e+8, median 895228, min -1073728193, max 1073739280
-1073728193.. -966354820: ##################################################### 5009
-966354819.. -858981446: #################################################### 5004
-858981445.. -751608072: #################################################### 4917
Expand All @@ -1436,7 +1436,7 @@ stats dist:
+++ Stats for oneof int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 1073741.63, stddev: 676575790.92, median 0, min -1073741824, max 1073741823
num: 1000, avg: 1073741.63, stddev: 6.766e+8, median 0, min -1073741824, max 1073741823
-1073741824.. -966367642: ################## 208
-966367641.. -858993459: 0
-858993458.. -751619276: 0
Expand Down Expand Up @@ -1665,7 +1665,7 @@ random seed: 153870556
+++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 10351291.88, stddev: 432212939.52, median 9, min -1066972208, max 1073741823
num: 1000, avg: 1.035e+7, stddev: 4.322e+8, median 9, min -1066972208, max 1073741823
-1066972208.. -959936507: ## 27
-959936506.. -852900805: ## 22
-852900804.. -745865103: ## 22
Expand Down
6 changes: 3 additions & 3 deletions test/core/QCheck_expect_test.expected.ocaml4.64
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1443,7 +1443,7 @@ stats dist:
+++ Stats for int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 100000, avg: 2541076923587387.50, stddev: 2660730801206827008.00, median 158655268318060, min -4611522359435274428, max 4611540922436307689
num: 100000, avg: 2.541e+15, stddev: 2.661e+18, median 158655268318060, min -4611522359435274428, max 4611540922436307689
-4611522359435274428..-4150369195341695293: ##################################################### 4976
-4150369195341695292..-3689216031248116157: ##################################################### 4963
-3689216031248116156..-3228062867154537021: ###################################################### 5038
Expand All @@ -1468,7 +1468,7 @@ stats dist:
+++ Stats for oneof int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 4611686018427388.00, stddev: 2905870896563567616.00, median 0, min -4611686018427387904, max 4611686018427387903
num: 1000, avg: 4.612e+15, stddev: 2.906e+18, median 0, min -4611686018427387904, max 4611686018427387903
-4611686018427387904..-4150517416584649089: ################## 208
-4150517416584649088..-3689348814741910273: 0
-3689348814741910272..-3228180212899171457: 0
Expand Down Expand Up @@ -1697,7 +1697,7 @@ random seed: 153870556
+++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: -55083208105414400.00, stddev: 1847115855773139200.00, median 9, min -4590718933436425025, max 4611686018427387903
num: 1000, avg: -5.508e+16, stddev: 1.847e+18, median 9, min -4590718933436425025, max 4611686018427387903
-4590718933436425025..-4130598685843234370: ## 26
-4130598685843234369..-3670478438250043714: # 13
-3670478438250043713..-3210358190656853058: ### 37
Expand Down
6 changes: 3 additions & 3 deletions test/core/QCheck_expect_test.expected.ocaml5.32
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1421,7 +1421,7 @@ stats dist:
+++ Stats for int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 100000, avg: -2481754.31, stddev: 618398387.27, median -5669677, min -1073719962, max 1073717275
num: 100000, avg: -2481754.31, stddev: 6.184e+8, median -5669677, min -1073719962, max 1073717275
-1073719962.. -966348101: ##################################################### 4978
-966348100.. -858976239: ##################################################### 5008
-858976238.. -751604377: #################################################### 4907
Expand All @@ -1446,7 +1446,7 @@ stats dist:
+++ Stats for oneof int dist ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: 1073741.60, stddev: 673131652.31, median 0, min -1073741824, max 1073741823
num: 1000, avg: 1073741.60, stddev: 6.731e+8, median 0, min -1073741824, max 1073741823
-1073741824.. -966367642: ################# 198
-966367641.. -858993459: 0
-858993458.. -751619276: 0
Expand Down Expand Up @@ -1676,7 +1676,7 @@ random seed: 153870556
+++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

stats dist:
num: 1000, avg: -20587725.59, stddev: 427231078.01, median 8, min -1072292884, max 1073741823
num: 1000, avg: -2.059e+7, stddev: 4.272e+8, median 8, min -1072292884, max 1073741823
-1072292884.. -964991149: ## 26
-964991148.. -857689413: ### 31
-857689412.. -750387677: ## 22
Expand Down
Loading