From 660848ce34b7a0790015f9e77c44274914527473 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 22 Nov 2017 10:05:07 +0100 Subject: [PATCH 1/2] hook up stdlib to the standard test running system --- test/Makefile | 4 +++- test/choosetests.jl | 29 +++++++++++++++++++++++++++++ test/runtests.jl | 41 +++++++++++++++++++++++++++++------------ test/stdlib.jl | 14 -------------- test/testdefs.jl | 4 ++-- 5 files changed, 63 insertions(+), 29 deletions(-) delete mode 100644 test/stdlib.jl diff --git a/test/Makefile b/test/Makefile index 099b71bacdb64..8e03d2e5a3f50 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,11 +1,13 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) JULIAHOME := $(abspath $(SRCDIR)/..) BUILDDIR := . +STDLIBDIR := $(abspath $(JULIAHOME)/stdlib) include $(JULIAHOME)/Make.inc # TODO: this Makefile ignores BUILDDIR, except for computing JULIA_EXECUTABLE TESTGROUPS = linalg sparse unicode strings dates -TESTS = all $(TESTGROUPS) \ +TESTS = all stdlib $(TESTGROUPS) \ + $(patsubst $(STDLIBDIR)/%/,%,$(dir $(wildcard $(STDLIBDIR)/*/.))) \ $(filter-out TestHelpers runtests testdefs, \ $(patsubst $(SRCDIR)/%.jl,%,$(wildcard $(SRCDIR)/*.jl))) \ $(foreach group,$(TESTGROUPS), \ diff --git a/test/choosetests.jl b/test/choosetests.jl index cfb0a3d08a6f0..19203c4896417 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -52,6 +52,8 @@ function choosetests(choices = []) "reinterpretarray", "syntax" ] + stdlibs = readdir(joinpath(@__DIR__, "..", "stdlib")) + if isdir(joinpath(JULIA_HOME, Base.DOCDIR, "examples")) push!(testnames, "examples") end @@ -176,6 +178,33 @@ function choosetests(choices = []) filter!(x -> !(x in net_required_for), tests) end + if "stdlib" in skip_tests + filter!(x -> x != "stdlib", tests) + elseif "stdlib" in tests + filter!(x -> x != "stdlib", tests) + prepend!(tests, stdlibs) + end + + if startswith(string(Sys.ARCH), "arm") + # Remove profile from default tests on ARM since it currently segfaults + # Allow explicitly adding it for testing + warn("Skipping Profile tests") + filter!(x -> (x != "Profile"), tests) + end + + stdlib_dir = joinpath(@__DIR__, "..", "stdlib") + for stdlib in filter(x -> x in stdlibs, tests) + splice!(tests, findfirst(equalto(stdlib), tests)) + dir = readdir(stdlib_dir) + test_file = joinpath(stdlib_dir, stdlib, "test", "runtests") + if isfile(test_file * ".jl") + unshift!(tests, test_file) + else + warn("Standard library $stdlib did not provide a `test/runtests.jl` file") + end + end + + filter!(x -> !(x in skip_tests), tests) tests, net_on, exit_on_error, seed diff --git a/test/runtests.jl b/test/runtests.jl index 77dd16397474b..08814e82e2917 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,15 +13,29 @@ else typemax(Csize_t) end + +test_names = copy(tests) +for (i, test) in enumerate(test_names) + if contains(test, "stdlib") && endswith(test, "runtests") + test_names[i] = split(test, "/")[end-2] + end +end + const node1_tests = String[] +const node1_test_names = String[] function move_to_node1(t) - if t in tests - splice!(tests, findfirst(equalto(t), tests)) - push!(node1_tests, t) + if t in test_names + i = findfirst(equalto(t), test_names) + push!(node1_test_names, test_names[i]) + push!(node1_tests, tests[i]) + splice!(test_names, i) + splice!(tests, i) end end # Base.compile only works from node 1, so compile test is handled specially move_to_node1("compile") +move_to_node1("SharedArrays") + # In a constrained memory environment, run the "distributed" test after all other tests # since it starts a lot of workers and can easily exceed the maximum memory max_worker_rss != typemax(Csize_t) && move_to_node1("distributed") @@ -38,7 +52,7 @@ cd(dirname(@__FILE__)) do @everywhere include("testdefs.jl") #pretty print the information about gc and mem usage - name_align = maximum([length("Test (Worker)"); map(x -> length(x) + 3 + ndigits(nworkers()), tests)]) + name_align = maximum([length("Test (Worker)"); map(x -> length(x) + 3 + ndigits(nworkers()), test_names)]) elapsed_align = length("Time (s)") gc_align = length("GC (s)") percent_align = length("GC %") @@ -47,23 +61,24 @@ cd(dirname(@__FILE__)) do print_with_color(:white, rpad("Test (Worker)",name_align," "), " | ") print_with_color(:white, "Time (s) | GC (s) | GC % | Alloc (MB) | RSS (MB)\n") results=[] + test_data = collect(zip(tests, test_names)) @sync begin for p in workers() @async begin - while length(tests) > 0 - test = shift!(tests) + while length(test_data) > 0 + test, test_name = shift!(test_data) local resp wrkr = p try - resp = remotecall_fetch(runtests, wrkr, test; seed=seed) + resp = remotecall_fetch(runtests, wrkr, test_name, test; seed=seed) catch e resp = [e] end push!(results, (test, resp)) if resp[1] isa Exception if exit_on_error - skipped = length(tests) - empty!(tests) + skipped = length(test_data) + empty!(test_data) end elseif resp[end] > max_worker_rss if n > 1 @@ -75,7 +90,7 @@ cd(dirname(@__FILE__)) do end end if !isa(resp[1], Exception) - print_with_color(:white, rpad(test*" ($wrkr)", name_align, " "), " | ") + print_with_color(:white, rpad(test_name*" ($wrkr)", name_align, " "), " | ") time_str = @sprintf("%7.2f",resp[2]) print_with_color(:white, rpad(time_str,elapsed_align," "), " | ") gc_str = @sprintf("%5.2f",resp[5].total_time/10^9) @@ -98,16 +113,18 @@ cd(dirname(@__FILE__)) do end end end - for t in node1_tests + for (t, t_name) in zip(node1_tests, node1_test_names) # As above, try to run each test # which must run on node 1. If # the test fails, catch the error, # and either way, append the results # to the overall aggregator n > 1 && print("\tFrom worker 1:\t") + isolate = true + t_name == "SharedArrays" && (isolate = false) local resp try - resp = eval(Expr(:call, () -> runtests(t, seed=seed))) # runtests is defined by the include above + resp = eval(Expr(:call, () -> runtests(t_name, t, isolate, seed=seed))) # runtests is defined by the include above catch e resp = [e] end diff --git a/test/stdlib.jl b/test/stdlib.jl deleted file mode 100644 index 0c16c90220c0e..0000000000000 --- a/test/stdlib.jl +++ /dev/null @@ -1,14 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -# test default packages - -cd(joinpath(JULIA_HOME,"..","share","julia","site","v$(VERSION.major).$(VERSION.minor)")) do - pkgs = readdir() - if startswith(string(Sys.ARCH), "arm") - # Remove profile from default tests on ARM since it currently segfaults - # Allow explicitly adding it for testing - warn("Skipping Profile tests") - filter!(x -> (x != "Profile"), pkgs) - end - Pkg.Entry.test(convert(Vector{AbstractString}, pkgs)) -end diff --git a/test/testdefs.jl b/test/testdefs.jl index ce13d1a54ee57..3191d553cb193 100644 --- a/test/testdefs.jl +++ b/test/testdefs.jl @@ -2,7 +2,7 @@ using Test -function runtests(name, isolate=true; seed=nothing) +function runtests(name, path, isolate=true; seed=nothing) old_print_setting = Test.TESTSET_PRINT_ENABLE[] Test.TESTSET_PRINT_ENABLE[] = false try @@ -19,7 +19,7 @@ function runtests(name, isolate=true; seed=nothing) @timed @testset $"$name" begin # srand(nothing) will fail $seed != nothing && srand($seed) - include($"$name.jl") + include($"$path.jl") end end res_and_time_data = eval(m, ex) From 8f197bd4200e05275b73fab05a4ccb4b9116a23e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 22 Nov 2017 11:44:22 +0100 Subject: [PATCH 2/2] fixups from review --- test/choosetests.jl | 24 ++++++----------------- test/runtests.jl | 46 ++++++++++++++++++++++----------------------- 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/test/choosetests.jl b/test/choosetests.jl index 19203c4896417..5586c461cbb80 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -1,5 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +const STDLIB_DIR = joinpath(JULIA_HOME, "..", "share", "julia", "site", "v$(VERSION.major).$(VERSION.minor)") +const STDLIBS = readdir(STDLIB_DIR) + @doc """ `tests, net_on, exit_on_error, seed = choosetests(choices)` selects a set of tests to be @@ -52,8 +55,6 @@ function choosetests(choices = []) "reinterpretarray", "syntax" ] - stdlibs = readdir(joinpath(@__DIR__, "..", "stdlib")) - if isdir(joinpath(JULIA_HOME, Base.DOCDIR, "examples")) push!(testnames, "examples") end @@ -179,10 +180,10 @@ function choosetests(choices = []) end if "stdlib" in skip_tests - filter!(x -> x != "stdlib", tests) + filter!(x -> (x != "stdlib" && !(x in STDLIBS)) , tests) elseif "stdlib" in tests - filter!(x -> x != "stdlib", tests) - prepend!(tests, stdlibs) + filter!(x -> (x != "stdlib" && !(x in STDLIBS)) , tests) + prepend!(tests, STDLIBS) end if startswith(string(Sys.ARCH), "arm") @@ -192,19 +193,6 @@ function choosetests(choices = []) filter!(x -> (x != "Profile"), tests) end - stdlib_dir = joinpath(@__DIR__, "..", "stdlib") - for stdlib in filter(x -> x in stdlibs, tests) - splice!(tests, findfirst(equalto(stdlib), tests)) - dir = readdir(stdlib_dir) - test_file = joinpath(stdlib_dir, stdlib, "test", "runtests") - if isfile(test_file * ".jl") - unshift!(tests, test_file) - else - warn("Standard library $stdlib did not provide a `test/runtests.jl` file") - end - end - - filter!(x -> !(x in skip_tests), tests) tests, net_on, exit_on_error, seed diff --git a/test/runtests.jl b/test/runtests.jl index 08814e82e2917..6370c366d757e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,25 +13,26 @@ else typemax(Csize_t) end - -test_names = copy(tests) -for (i, test) in enumerate(test_names) - if contains(test, "stdlib") && endswith(test, "runtests") - test_names[i] = split(test, "/")[end-2] +function test_path(test) + if test in STDLIBS + test_file = joinpath(STDLIB_DIR, test, "test", "runtests") + if !isfile(test_file * ".jl") + error("Standard library $test did not provide a `test/runtests.jl` file") + end + return test_file + else + return test end end const node1_tests = String[] -const node1_test_names = String[] function move_to_node1(t) - if t in test_names - i = findfirst(equalto(t), test_names) - push!(node1_test_names, test_names[i]) - push!(node1_tests, tests[i]) - splice!(test_names, i) - splice!(tests, i) + if t in tests + splice!(tests, findfirst(equalto(t), tests)) + push!(node1_tests, t) end end + # Base.compile only works from node 1, so compile test is handled specially move_to_node1("compile") move_to_node1("SharedArrays") @@ -52,7 +53,7 @@ cd(dirname(@__FILE__)) do @everywhere include("testdefs.jl") #pretty print the information about gc and mem usage - name_align = maximum([length("Test (Worker)"); map(x -> length(x) + 3 + ndigits(nworkers()), test_names)]) + name_align = maximum([length("Test (Worker)"); map(x -> length(x) + 3 + ndigits(nworkers()), tests)]) elapsed_align = length("Time (s)") gc_align = length("GC (s)") percent_align = length("GC %") @@ -61,24 +62,23 @@ cd(dirname(@__FILE__)) do print_with_color(:white, rpad("Test (Worker)",name_align," "), " | ") print_with_color(:white, "Time (s) | GC (s) | GC % | Alloc (MB) | RSS (MB)\n") results=[] - test_data = collect(zip(tests, test_names)) @sync begin for p in workers() @async begin - while length(test_data) > 0 - test, test_name = shift!(test_data) + while length(tests) > 0 + test = shift!(tests) local resp wrkr = p try - resp = remotecall_fetch(runtests, wrkr, test_name, test; seed=seed) + resp = remotecall_fetch(runtests, wrkr, test, test_path(test); seed=seed) catch e resp = [e] end push!(results, (test, resp)) if resp[1] isa Exception if exit_on_error - skipped = length(test_data) - empty!(test_data) + skipped = length(tests) + empty!(tests) end elseif resp[end] > max_worker_rss if n > 1 @@ -90,7 +90,7 @@ cd(dirname(@__FILE__)) do end end if !isa(resp[1], Exception) - print_with_color(:white, rpad(test_name*" ($wrkr)", name_align, " "), " | ") + print_with_color(:white, rpad(test*" ($wrkr)", name_align, " "), " | ") time_str = @sprintf("%7.2f",resp[2]) print_with_color(:white, rpad(time_str,elapsed_align," "), " | ") gc_str = @sprintf("%5.2f",resp[5].total_time/10^9) @@ -113,7 +113,7 @@ cd(dirname(@__FILE__)) do end end end - for (t, t_name) in zip(node1_tests, node1_test_names) + for t in node1_tests # As above, try to run each test # which must run on node 1. If # the test fails, catch the error, @@ -121,10 +121,10 @@ cd(dirname(@__FILE__)) do # to the overall aggregator n > 1 && print("\tFrom worker 1:\t") isolate = true - t_name == "SharedArrays" && (isolate = false) + t == "SharedArrays" && (isolate = false) local resp try - resp = eval(Expr(:call, () -> runtests(t_name, t, isolate, seed=seed))) # runtests is defined by the include above + resp = eval(Expr(:call, () -> runtests(t, test_path(t), isolate, seed=seed))) # runtests is defined by the include above catch e resp = [e] end