Merge pull request #10396 from JuliaLang/cb/publish

Faster Pkg.Publish()

base/pkg/entry.jl

@@ -314,9 +314,7 @@ function publish(branch::AbstractString)
     ahead_remote > 0 && error("METADATA is behind origin/$branch – run `Pkg.update()` before publishing")
     ahead_local == 0 && error("There are no METADATA changes to publish")
 
-    info("Validating METADATA")
-    check_metadata()
-    tags = Dict{ASCIIString,Vector{ASCIIString}}()
+    tags = Dict{ByteString,Vector{ASCIIString}}()
     Git.run(`update-index -q --really-refresh`, dir="METADATA")
     cmd = `diff --name-only --diff-filter=AMR origin/$branch HEAD --`
     for line in eachline(Git.cmd(cmd, dir="METADATA"))
@@ -337,6 +335,8 @@ function publish(branch::AbstractString)
         end || error("$pkg v$ver is incorrectly tagged – $sha1 expected")
     end
     isempty(tags) && info("No new package versions to publish")
+    info("Validating METADATA")
+    check_metadata(Set(keys(tags)))
     @sync for pkg in sort!([keys(tags)...])
         @async begin
             forced = ASCIIString[]
@@ -585,15 +585,15 @@ function tag(pkg::AbstractString, ver::Union(Symbol,VersionNumber), force::Bool=
     end
 end
 
-function check_metadata()
+function check_metadata(pkgs::Set{ByteString} = Set{ByteString}())
     avail = Read.available()
     deps, conflicts = Query.dependencies(avail)
 
     for (dp,dv) in deps, (v,a) in dv, p in keys(a.requires)
         haskey(deps, p) || error("package $dp v$v requires a non-registered package: $p")
     end
 
-    problematic = Resolve.sanity_check(deps)
+    problematic = Resolve.sanity_check(deps, pkgs)
     if !isempty(problematic)
         msg = "packages with unsatisfiable requirements found:\n"
         for (p, vn, rp) in problematic

base/pkg/query.jl

@@ -138,7 +138,7 @@ function prune_versions(reqs::Requires, deps::Dict{ByteString,Dict{VersionNumber
             allowedp[vn] = vn in vs
         end
         @assert !isempty(allowedp)
-        @assert any(collect(values(allowedp)))
+        @assert any(values(allowedp))
     end
 
     filtered_deps = Dict{ByteString,Dict{VersionNumber,Available}}()
@@ -299,35 +299,70 @@ end
 prune_versions(deps::Dict{ByteString,Dict{VersionNumber,Available}}) =
     prune_versions(Dict{ByteString,VersionSet}(), deps)
 
+# Build a graph restricted to a subset of the packages
+function subdeps(deps::Dict{ByteString,Dict{VersionNumber,Available}}, pkgs::Set{ByteString})
+
+    sub_deps = Dict{ByteString,Dict{VersionNumber,Available}}()
+    for p in pkgs
+        haskey(sub_deps, p) || (sub_deps[p] = Dict{VersionNumber,Available}())
+        sub_depsp = sub_deps[p]
+        for (vn, a) in deps[p]
+            sub_depsp[vn] = a
+        end
+    end
+
+    return sub_deps
+end
+
 # Build a subgraph incuding only the (direct and indirect) dependencies
 # of a given package set
 function dependencies_subset(deps::Dict{ByteString,Dict{VersionNumber,Available}}, pkgs::Set{ByteString})
 
-    np = length(deps)
-
     staged = pkgs
-    allpkgs = pkgs
+    allpkgs = copy(pkgs)
     while !isempty(staged)
         staged_next = Set{ByteString}()
         for p in staged, a in values(deps[p]), rp in keys(a.requires)
             if !(rp in allpkgs)
                 push!(staged_next, rp)
             end
         end
-        allpkgs = union(allpkgs, staged_next)
+        union!(allpkgs, staged_next)
         staged = staged_next
     end
 
-    sub_deps = Dict{ByteString,Dict{VersionNumber,Available}}()
-    for p in allpkgs
-        haskey(sub_deps, p) || (sub_deps[p] = Dict{VersionNumber,Available}())
-        sub_depsp = sub_deps[p]
-        for (vn, a) in deps[p]
-            sub_depsp[vn] = a
+    return subdeps(deps, allpkgs)
+end
+
+# Build a subgraph incuding only the (direct and indirect) dependencies and dependants
+# of a given package set
+function undirected_dependencies_subset(deps::Dict{ByteString,Dict{VersionNumber,Available}}, pkgs::Set{ByteString})
+
+    graph = Dict{ByteString, Set{ByteString}}()
+
+    for (p,d) in deps
+        haskey(graph, p) || (graph[p] = Set{ByteString}())
+        for a in values(d), rp in keys(a.requires)
+            push!(graph[p], rp)
+            haskey(graph, rp) || (graph[rp] = Set{ByteString}())
+            push!(graph[rp], p)
         end
     end
 
-    return sub_deps
+    staged = pkgs
+    allpkgs = copy(pkgs)
+    while !isempty(staged)
+        staged_next = Set{ByteString}()
+        for p in staged, rp in graph[p]
+            if !(rp in allpkgs)
+                push!(staged_next, rp)
+            end
+        end
+        union!(allpkgs, staged_next)
+        staged = staged_next
+    end
+
+    return subdeps(deps, allpkgs)
 end
 
 function prune_dependencies(reqs::Requires, deps::Dict{ByteString,Dict{VersionNumber,Available}})

base/pkg/resolve.jl

@@ -44,7 +44,9 @@ function resolve(reqs::Requires, deps::Dict{ByteString,Dict{VersionNumber,Availa
 end
 
 # Scan dependencies for (explicit or implicit) contradictions
-function sanity_check(deps::Dict{ByteString,Dict{VersionNumber,Available}})
+function sanity_check(deps::Dict{ByteString,Dict{VersionNumber,Available}}, pkgs::Set{ByteString} = Set{ByteString}())
+
+    isempty(pkgs) || (deps = Query.undirected_dependencies_subset(deps, pkgs))
 
     deps, eq_classes = Query.prune_versions(deps)
 

base/pkg/resolve/versionweight.jl

@@ -60,7 +60,7 @@ function Base.cmp{T}(a::HierarchicalValue{T}, b::HierarchicalValue{T})
     return cmp(a.rest, b.rest)
 end
 Base.isless{T}(a::HierarchicalValue{T}, b::HierarchicalValue{T}) = cmp(a,b) < 0
-=={T}(a::HierarchicalValue{T}, b::HierarchicalValue{T}) = a.v == b.v && a.rest == b.rest
+=={T}(a::HierarchicalValue{T}, b::HierarchicalValue{T}) = cmp(a,b) == 0
 
 Base.abs{T}(a::HierarchicalValue{T}) = HierarchicalValue(T[abs(x) for x in a.v], abs(a.rest))
 
@@ -97,8 +97,10 @@ immutable VWPreBuild
     w::HierarchicalValue{VWPreBuildItem}
 end
 
+const _vwprebuild_zero = VWPreBuild(0, HierarchicalValue(VWPreBuildItem))
+
 function VWPreBuild(ispre::Bool, desc::(Union(Int,ASCIIString)...))
-    isempty(desc) && return VWPreBuild(0, HierarchicalValue(VWPreBuildItem))
+    isempty(desc) && return _vwprebuild_zero
     desc == ("",) && return VWPreBuild(ispre ? -1 : 1, HierarchicalValue(VWPreBuildItem[]))
     nonempty = ispre ? -1 : 0
     w = Array(VWPreBuildItem, length(desc))
@@ -109,25 +111,41 @@ function VWPreBuild(ispre::Bool, desc::(Union(Int,ASCIIString)...))
     end
     return VWPreBuild(nonempty, HierarchicalValue(w))
 end
-VWPreBuild() = VWPreBuild(0, HierarchicalValue(VWPreBuildItem))
+VWPreBuild() = _vwprebuild_zero
 
 Base.zero(::Type{VWPreBuild}) = VWPreBuild()
 
-Base.typemin(::Type{VWPreBuild}) = VWPreBuild(typemin(Int), typemin(HierarchicalValue{VWPreBuildItem}))
+const _vwprebuild_min = VWPreBuild(typemin(Int), typemin(HierarchicalValue{VWPreBuildItem}))
+Base.typemin(::Type{VWPreBuild}) = _vwprebuild_min
 
-Base.(:-)(a::VWPreBuild, b::VWPreBuild) = VWPreBuild(a.nonempty-b.nonempty, a.w-b.w)
-Base.(:+)(a::VWPreBuild, b::VWPreBuild) = VWPreBuild(a.nonempty+b.nonempty, a.w+b.w)
+function Base.(:-)(a::VWPreBuild, b::VWPreBuild)
+    b === _vwprebuild_zero && return a
+    a === _vwprebuild_zero && return -b
+    VWPreBuild(a.nonempty-b.nonempty, a.w-b.w)
+end
+function Base.(:+)(a::VWPreBuild, b::VWPreBuild)
+    b === _vwprebuild_zero && return a
+    a === _vwprebuild_zero && return b
+    VWPreBuild(a.nonempty+b.nonempty, a.w+b.w)
+end
 
-Base.(:-)(a::VWPreBuild) = VWPreBuild(-a.nonempty, -a.w)
+function Base.(:-)(a::VWPreBuild)
+    a === _vwprebuild_zero && return a
+    VWPreBuild(-a.nonempty, -a.w)
+end
 
-function Base.cmp(a::VWPreBuild, b::VWPreBuild)
+@inline function Base.cmp(a::VWPreBuild, b::VWPreBuild)
+    a === _vwprebuild_zero && b === _vwprebuild_zero && return 0
     c = cmp(a.nonempty, b.nonempty); c != 0 && return c
     return cmp(a.w, b.w)
 end
 Base.isless(a::VWPreBuild, b::VWPreBuild) = cmp(a,b) < 0
 ==(a::VWPreBuild, b::VWPreBuild) = cmp(a,b) == 0
 
-Base.abs(a::VWPreBuild) = VWPreBuild(abs(a.nonempty), abs(a.w))
+function Base.abs(a::VWPreBuild)
+    a === _vwprebuild_zero && return a
+    VWPreBuild(abs(a.nonempty), abs(a.w))
+end
 
 # The numeric type used to determine how the different
 # versions of a package should be weighed

test/resolve.jl

@@ -84,18 +84,18 @@ function reqs_from_data(reqs_data)
     end
     reqs
 end
-function sanity_tst(deps_data, expected_result)
+function sanity_tst(deps_data, expected_result; pkgs=[])
     deps = deps_from_data(deps_data)
     #println("deps=$deps")
     #println()
-    result = sanity_check(deps)
+    result = sanity_check(deps, Set(ByteString[pkgs...]))
     length(result) == length(expected_result) || return false
     for (p, vn, pp) in result
         in((p, vn), expected_result) || return  false
     end
     return true
 end
-sanity_tst(deps_data) = sanity_tst(deps_data, [])
+sanity_tst(deps_data; kw...) = sanity_tst(deps_data, []; kw...)
 
 function resolve_tst(deps_data, reqs_data)
     deps = deps_from_data(deps_data)
@@ -117,6 +117,9 @@ deps_data = Any[
 ]
 
 @test sanity_tst(deps_data)
+@test sanity_tst(deps_data, pkgs=["A", "B"])
+@test sanity_tst(deps_data, pkgs=["B"])
+@test sanity_tst(deps_data, pkgs=["A"])
 
 # require just B
 reqs_data = Any[
@@ -214,6 +217,7 @@ deps_data = Any[
 ]
 
 @test sanity_tst(deps_data, [("A", v"1")])
+@test sanity_tst(deps_data, [("A", v"1")], pkgs=["B"])
 
 # require B (must not give errors)
 reqs_data = Any[
@@ -236,6 +240,8 @@ deps_data = Any[
 ]
 
 @test sanity_tst(deps_data, [("A", v"2")])
+@test sanity_tst(deps_data, [("A", v"2")], pkgs=["B"])
+@test sanity_tst(deps_data, [("A", v"2")], pkgs=["C"])
 
 # require A, any version (must use the highest non-inconsistent)
 reqs_data = Any[
@@ -477,3 +483,48 @@ reqs_data = Any[
 want = resolve_tst(deps_data, reqs_data)
 @test want == Dict("A"=>v"2", "B"=>v"1.0.1", "C"=>v"1+BLD",
                    "D"=>v"2", "E"=>v"1", "F"=>v"2-rc.1")
+
+## DEPENDENCY SCHEME 11: FIVE PACKAGES, SAME AS SCHEMES 5 + 1, UNCONNECTED
+deps_data = Any[
+    ["A", v"1", "B", v"2"],
+    ["A", v"1", "C", v"2"],
+    ["A", v"2", "B", v"1", v"2"],
+    ["A", v"2", "C", v"1", v"2"],
+    ["B", v"1", "C", v"2"],
+    ["B", v"2", "C", v"2"],
+    ["C", v"1"],
+    ["C", v"2"],
+    ["D", v"1", "E", v"1"],
+    ["D", v"2", "E", v"2"],
+    ["E", v"1"],
+    ["E", v"2"]
+]
+
+@test sanity_tst(deps_data, [("A", v"2")])
+@test sanity_tst(deps_data, [("A", v"2")], pkgs=["B"])
+@test sanity_tst(deps_data, pkgs=["D"])
+@test sanity_tst(deps_data, pkgs=["E"])
+@test sanity_tst(deps_data, [("A", v"2")], pkgs=["B", "D"])
+
+# require A, any version (must use the highest non-inconsistent)
+reqs_data = Any[
+    ["A"]
+]
+want = resolve_tst(deps_data, reqs_data)
+@test want == Dict("A"=>v"1", "B"=>v"2", "C"=>v"2")
+
+# require just D: must bring in E
+reqs_data = Any[
+    ["D"]
+]
+want = resolve_tst(deps_data, reqs_data)
+@test want == Dict("D"=>v"2", "E"=>v"2")
+
+
+# require A and D, must be the merge of the previous two cases
+reqs_data = Any[
+    ["A"],
+    ["D"]
+]
+want = resolve_tst(deps_data, reqs_data)
+@test want == Dict("A"=>v"1", "B"=>v"2", "C"=>v"2", "D"=>v"2", "E"=>v"2")