[WIP] Help wanted: switch Base.Test to use testsets everywhere

base/test.jl

@@ -266,7 +266,7 @@ function do_test(result::ExecutionResult, orig_expr)
         value = result.value
         testres = if isa(value, Bool)
             # a true value Passes
-            value ? Pass(:test, orig_expr, result.data, value) :
+            value ? Pass(:test, nothing, nothing, value) :
                     Fail(:test, orig_expr, result.data, value)
         else
             # If the result is non-Boolean, this counts as an Error
@@ -319,7 +319,7 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype)
     if isa(result, Threw)
         # Check that the right type of exception was thrown
         if isa(result.exception, extype)
-            testres = Pass(:test_throws, orig_expr, extype, result.exception)
+            testres = Pass(:test_throws, nothing, nothing, result.exception)
         else
             testres = Fail(:test_throws_wrong, orig_expr, extype, result.exception)
         end
@@ -368,6 +368,7 @@ type TestSetException <: Exception
     fail::Int
     error::Int
     broken::Int
+    errors_and_fails
 end
 
 function Base.show(io::IO, ex::TestSetException)
@@ -421,32 +422,36 @@ record(ts::DefaultTestSet, t::Union{Pass,Broken}) = (push!(ts.results, t); t)
 # For the other result types, immediately print the error message
 # but do not terminate. Print a backtrace.
 function record(ts::DefaultTestSet, t::Union{Fail, Error})
-    print_with_color(:white, ts.description, ": ")
-    print(t)
-    # don't print the backtrace for Errors because it gets printed in the show
-    # method
-    isa(t, Error) || Base.show_backtrace(STDOUT, backtrace())
-    println()
+    if myid() == 1
+        print_with_color(:white, ts.description, ": ")
+        print(t)
+        # don't print the backtrace for Errors because it gets printed in the show
+        # method
+        isa(t, Error) || Base.show_backtrace(STDOUT, backtrace())
+        println()
+    end
     push!(ts.results, t)
-    t
+    t, isa(t, Error) || backtrace()
 end
 
 # When a DefaultTestSet finishes, it records itself to its parent
 # testset, if there is one. This allows for recursive printing of
 # the results at the end of the tests
 record(ts::DefaultTestSet, t::AbstractTestSet) = push!(ts.results, t)
 
-# Called at the end of a @testset, behaviour depends on whether
-# this is a child of another testset, or the "root" testset
-function finish(ts::DefaultTestSet)
-    # If we are a nested test set, do not print a full summary
-    # now - let the parent test set do the printing
-    if get_testset_depth() != 0
-        # Attach this test set to the parent test set
-        parent_ts = get_testset()
-        record(parent_ts, ts)
-        return
+function print_test_errors(ts::DefaultTestSet)
+    for t in ts.results
+        if (isa(t, Error) || isa(t, Fail)) && myid() == 1
+            println("Error in testset $(ts.description):")
+            Base.show(STDOUT,t)
+            println()
+        elseif isa(t, DefaultTestSet)
+            print_test_errors(t)
+        end
     end
+end
+
+function print_test_results(ts::DefaultTestSet, depth_pad=0)
     # Calculate the overall number for each type so each of
     # the test result types are aligned
     passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken = get_test_counts(ts)
@@ -489,10 +494,31 @@ function finish(ts::DefaultTestSet)
     end
     println()
     # Recursively print a summary at every level
-    print_counts(ts, 0, align, pass_width, fail_width, error_width, broken_width, total_width)
+    print_counts(ts, depth_pad, align, pass_width, fail_width, error_width, broken_width, total_width)
+end
+
+# Called at the end of a @testset, behaviour depends on whether
+# this is a child of another testset, or the "root" testset
+function finish(ts::DefaultTestSet)
+    # If we are a nested test set, do not print a full summary
+    # now - let the parent test set do the printing
+    if get_testset_depth() != 0
+        # Attach this test set to the parent test set
+        parent_ts = get_testset()
+        record(parent_ts, ts)
+        return ts
+    end
+    passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken = get_test_counts(ts)
+    total_pass   = passes + c_passes
+    total_fail   = fails  + c_fails
+    total_error  = errors + c_errors
+    total_broken = broken + c_broken
+    total = total_pass + total_fail + total_error + total_broken
     # Finally throw an error as we are the outermost test set
-    if total != total_pass
-        throw(TestSetException(total_pass,total_fail,total_error, total_broken))
+    if total != total_pass + total_broken
+        # Get all the error/failures and bring them along for the ride
+        efs = filter_errors(ts)
+        throw(TestSetException(total_pass,total_fail,total_error, total_broken, efs))
     end
 
     # return the testset so it is returned from the @testset macro
@@ -517,6 +543,20 @@ function get_alignment(ts::DefaultTestSet, depth::Int)
 end
 get_alignment(ts, depth::Int) = 0
 
+# Recursive function that fetches backtraces for any and all errors
+# or failures the testset and its children encountered
+function filter_errors(ts::DefaultTestSet)
+    efs = []
+    for t in ts.results
+        if isa(t, DefaultTestSet)
+            append!(efs, filter_errors(t))
+        elseif isa(t, Union{Fail, Error})
+            append!(efs, [t])
+        end
+    end
+    efs
+end
+
 # Recursive function that counts the number of test results of each
 # type directly in the testset, and totals across the child testsets
 function get_test_counts(ts::DefaultTestSet)
@@ -547,7 +587,6 @@ function print_counts(ts::DefaultTestSet, depth, align,
     # through any child test sets
     passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken = get_test_counts(ts)
     subtotal = passes + fails + errors + broken + c_passes + c_fails + c_errors + c_broken
-
     # Print test set header, with an alignment that ensures all
     # the test results appear above each other
     print(rpad(string(lpad("  ",depth), ts.description), align, " "), " | ")

test/Makefile

@@ -4,7 +4,7 @@ BUILDDIR := .
 include $(JULIAHOME)/Make.inc
 # TODO: this Makefile ignores BUILDDIR, except for computing JULIA_EXECUTABLE
 
-TESTS = all linalg $(filter-out TestHelpers runtests testdefs,$(patsubst $(SRCDIR)/%.jl,%,$(wildcard $(SRCDIR)/*.jl $(SRCDIR)/linalg/*.jl)))
+TESTS = all linalg sparse unicode strings dates $(filter-out TestHelpers runtests testdefs,$(patsubst $(SRCDIR)/%.jl,%,$(wildcard $(SRCDIR)/*.jl $(SRCDIR)/linalg/*.jl)))
 
 default: all
 

test/base64.jl

@@ -1,5 +1,5 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
-
+@testset "base64" begin
 const inputText = "Man is distinguished, not only by his reason, but by this singular passion from other animals, which is a lust of the mind, that by a perseverance of delight in the continued and indefatigable generation of knowledge, exceeds the short vehemence of any carnal pleasure."
 const encodedMaxLine76 =
 """TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0aGlz
@@ -41,3 +41,5 @@ ipipe = Base64DecodePipe(IOBuffer(string(encodedMaxLine76[1:end-2],"==")))
 # Test incorrect format
 ipipe = Base64DecodePipe(IOBuffer(encodedMaxLine76[1:end-3]))
 @test_throws ArgumentError readstring(ipipe)
+
+end

test/bigint.jl

@@ -316,7 +316,6 @@ end
 @test big(5)^true == big(5)
 @test big(5)^false == one(BigInt)
 
-
 # operations that when applied to Int64 give Float64, should give BigFloat
 @test typeof(exp(a)) == BigFloat
 @test typeof(exp2(a)) == BigFloat

test/bitarray.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
-using Base.Test
-
 tc{N}(r1::NTuple{N}, r2::NTuple{N}) = all(x->tc(x...), [zip(r1,r2)...])
 tc{N}(r1::BitArray{N}, r2::Union{BitArray{N},Array{Bool,N}}) = true
 tc{T}(r1::T, r2::T) = true

test/blas.jl

@@ -299,3 +299,4 @@ for elty in [Float32, Float64, Complex64, Complex128]
         @test_throws DimensionMismatch BLAS.syrk!('L','N',one(elty),eye(elty,5),one(elty),eye(elty,6))
     end
 end
+

test/broadcast.jl

@@ -1,7 +1,6 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
 module TestBroadcastInternals
-
 using Base.Broadcast: broadcast_indices, check_broadcast_indices,
                       check_broadcast_shape, newindex, _bcs, _bcsm
 using Base: Test, OneTo

test/char.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
-#tests for /base/char.jl
-
 @test typemin(Char) == Char(0)
 @test ndims(Char) == 0
 @test getindex('a', 1) == 'a'

test/checked.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
-# Checked integer arithmetic
-
 import Base: checked_abs, checked_neg, checked_add, checked_sub, checked_mul,
              checked_div, checked_rem, checked_fld, checked_mod, checked_cld
 

test/choosetests.jl

@@ -16,7 +16,7 @@ Upon return, `tests` is a vector of fully-expanded test names, and
 function choosetests(choices = [])
     testnames = [
         "linalg", "subarray", "core", "inference", "keywordargs", "numbers",
-        "printf", "char", "string", "triplequote", "unicode",
+        "printf", "char", "strings", "triplequote", "unicode",
         "dates", "dict", "hashing", "iobuffer", "staged", "offsetarray",
         "arrayops", "tuple", "reduce", "reducedim", "random", "abstractarray",
         "intfuncs", "simdloop", "vecelement", "blas", "sparse",
@@ -60,6 +60,57 @@ function choosetests(choices = [])
         tests = testnames
     end
 
+    datestests = ["dates/accessors", "dates/adjusters", "dates/query",
+                  "dates/periods", "dates/ranges", "dates/rounding", "dates/types",
+                  "dates/io", "dates/arithmetic", "dates/conversions"]
+    if "dates" in skip_tests
+        filter!(x -> (x != "dates" && !(x in datestests)), tests)
+    elseif "dates" in tests
+        # specifically selected case
+        filter!(x -> x != "dates", tests)
+        prepend!(tests, datestests)
+    end
+
+    unicodetests = ["unicode/UnicodeError", "unicode/utf8proc", "unicode/utf8"]
+    if "unicode" in skip_tests
+        filter!(x -> (x != "unicode" && !(x in unicodetests)), tests)
+    elseif "unicode" in tests
+        # specifically selected case
+        filter!(x -> x != "unicode", tests)
+        prepend!(tests, unicodetests)
+    end
+
+    stringtests = ["strings/basic", "strings/search", "strings/util",
+                   "strings/io", "strings/types"]
+    if "strings" in skip_tests
+        filter!(x -> (x != "strings" && !(x in stringtests)), tests)
+    elseif "strings" in tests
+        # specifically selected case
+        filter!(x -> x != "strings", tests)
+        prepend!(tests, stringtests)
+    end
+
+
+    sparsetests = ["sparse/sparse", "sparse/sparsevector"]
+    if Base.USE_GPL_LIBS
+        append!(sparsetests, ["sparse/umfpack", "sparse/cholmod", "sparse/spqr"])
+    end
+    if "sparse" in skip_tests
+        filter!(x -> (x != "sparse" && !(x in sparsetests)), tests)
+    elseif "sparse" in tests
+        # specifically selected case
+        filter!(x -> x != "sparse", tests)
+        prepend!(tests, sparsetests)
+    end
+
+    #do subarray before sparse but after linalg
+    if "subarray" in skip_tests
+        filter!(x -> x != "subarray", tests)
+    elseif "subarray" in tests
+        filter!(x -> x != "subarray", tests)
+        prepend!(tests, ["subarray"])
+    end
+
     linalgtests = ["linalg/triangular", "linalg/qr", "linalg/dense",
                    "linalg/matmul", "linalg/schur", "linalg/special",
                    "linalg/eigen", "linalg/bunchkaufman", "linalg/svd",
@@ -83,7 +134,7 @@ function choosetests(choices = [])
     net_required_for = ["socket", "parallel", "libgit2"]
     net_on = true
     try
-        getipaddr()
+        ipa = getipaddr()
     catch
         warn("Networking unavailable: Skipping tests [" * join(net_required_for, ", ") * "]")
         net_on = false

test/cmdlineargs.jl

@@ -1,5 +1,7 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
+@testset "cmdlineargs" begin
+
 catcmd = `cat`
 if is_windows()
     try # use busybox-w32 on windows
@@ -339,3 +341,5 @@ for precomp in ("yes", "no")
     @test length(lno.captures) == 1
     @test parse(Int, lno.captures[1]) > 0
 end
+
+end

test/core.jl

@@ -1,7 +1,6 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
 # test core language features
-
 const Bottom = Union{}
 
 macro testintersect(args...)
@@ -1108,7 +1107,7 @@ let
     @test a==1 && b==2
 end
 
-# issue #1876
+@testset "issue #1876" begin
 let
     tst = 1
     m1(i) = (tst+=1;i-1)
@@ -1126,6 +1125,7 @@ let
     X[r...] *= 2
     @test X == [1,4,6,4]
 end
+end
 
 # issue #1632
 let
@@ -2301,24 +2301,26 @@ g9535() = (f9535(),f9535())
 
 # weak references
 type Obj; x; end
-@noinline function mk_wr(r, wr)
-    x = Obj(1)
-    push!(r, x)
-    push!(wr, WeakRef(x))
-end
-test_wr(r,wr) = @test r[1] == wr[1].value
-function test_wr()
-    ref = []
-    wref = []
-    mk_wr(ref, wref)
-    test_wr(ref, wref)
-    gc()
-    test_wr(ref, wref)
-    pop!(ref)
-    gc()
-    @test wref[1].value === nothing
+@testset "weak references" begin
+    @noinline function mk_wr(r, wr)
+        x = Obj(1)
+        push!(r, x)
+        push!(wr, WeakRef(x))
+    end
+    test_wr(r,wr) = @test r[1] == wr[1].value
+    function test_wr()
+        ref = []
+        wref = []
+        mk_wr(ref, wref)
+        test_wr(ref, wref)
+        gc()
+        test_wr(ref, wref)
+        pop!(ref)
+        gc()
+        @test wref[1].value == nothing
+    end
+    test_wr()
 end
-test_wr()
 
 # issue #9947
 function f9947()