diff --git a/bin/loadmeta.jl b/bin/loadmeta.jl index 798731c870522..05220e3617a39 100644 --- a/bin/loadmeta.jl +++ b/bin/loadmeta.jl @@ -1,14 +1,15 @@ #!/usr/bin/env julia using Base: thispatch, thisminor, nextpatch, nextminor -using Base.Pkg.Reqs: Reqs, Requirement -using Base.Pkg.Types using Base.LinAlg: checksquare using Base.Random: UUID using SHA -## Computing UUID5 values from (namespace, key) pairs ## +import Pkg3.Pkg2 +import Pkg2.Reqs: Reqs, Requirement +import Pkg2.Types: VersionInterval +## Computing UUID5 values from (namespace, key) pairs ## function uuid5(namespace::UUID, key::String) data = [reinterpret(UInt8, [namespace.value]); Vector{UInt8}(key)] u = reinterpret(UInt128, sha1(data)[1:16])[1] @@ -153,6 +154,6 @@ end ## Load package data ## -const pkgs = load_packages(Pkg.dir("METADATA")) +const pkgs = load_packages(Pkg2.dir("METADATA")) delete!(pkgs, "CardinalDicts") # package repo no longer exists prune!(pkgs) diff --git a/bin/sha1map.jl b/bin/sha1map.jl index 47684c57951a5..80d4bdfce766c 100644 --- a/bin/sha1map.jl +++ b/bin/sha1map.jl @@ -1,6 +1,6 @@ #!/usr/bin/env julia -using TOML +using Pkg3.TOML using Base: LibGit2 function sha1map(pkgs::Dict{String,Package}) diff --git a/src/Operations.jl b/src/Operations.jl index a29922ea1a739..b3c46c7092992 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -2,10 +2,9 @@ module Operations using Base.Random: UUID using Base: LibGit2 -using Base: Pkg using Pkg3.TerminalMenus using Pkg3.Types -import Pkg3: depots, BinaryProvider, USE_LIBGIT2_FOR_ALL_DOWNLOADS, NUM_CONCURRENT_DOWNLOADS +import Pkg3: Pkg2, depots, BinaryProvider, USE_LIBGIT2_FOR_ALL_DOWNLOADS, NUM_CONCURRENT_DOWNLOADS const SlugInt = UInt32 # max p = 4 const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" @@ -153,7 +152,9 @@ function resolve_versions!(env::EnvCache, pkgs::Vector{PackageSpec})::Dict{UUID, info("Resolving package versions") # anything not mentioned is fixed uuids = UUID[pkg.uuid for pkg in pkgs] + uuid_to_name = Dict{String, String}() for (name::String, uuid::UUID) in env.project["deps"] + uuid_to_name[string(uuid)] = name uuid in uuids && continue info = manifest_info(env, uuid) haskey(info, "version") || continue @@ -161,10 +162,16 @@ function resolve_versions!(env::EnvCache, pkgs::Vector{PackageSpec})::Dict{UUID, push!(pkgs, PackageSpec(name, uuid, ver)) end # construct data structures for resolver and call it - reqs = Dict{String,Pkg.Types.VersionSet}(string(pkg.uuid) => pkg.version for pkg in pkgs) - deps = convert(Dict{String,Dict{VersionNumber,Pkg.Types.Available}}, deps_graph(env, pkgs)) - deps = Pkg.Query.prune_dependencies(reqs, deps) - vers = convert(Dict{UUID,VersionNumber}, Pkg.Resolve.resolve(reqs, deps)) + reqs = Dict{String,Pkg2.Types.VersionSet}(string(pkg.uuid) => pkg.version for pkg in pkgs) + deps = convert(Dict{String,Dict{VersionNumber,Pkg2.Types.Available}}, deps_graph(env, pkgs)) + for dep_uuid in keys(deps) + info = manifest_info(env, UUID(dep_uuid)) + if info != nothing + uuid_to_name[info["uuid"]] = info["name"] + end + end + deps = Pkg2.Query.prune_dependencies(reqs, deps, uuid_to_name) + vers = convert(Dict{UUID,VersionNumber}, Pkg2.Resolve.resolve(reqs, deps, uuid_to_name)) find_registered!(env, collect(keys(vers))) # update vector of package versions for pkg in pkgs diff --git a/src/Pkg2/Pkg2.jl b/src/Pkg2/Pkg2.jl new file mode 100644 index 0000000000000..196c5d436d2b5 --- /dev/null +++ b/src/Pkg2/Pkg2.jl @@ -0,0 +1,47 @@ +module Pkg2 + +struct PkgError <: Exception + msg::AbstractString + ex::Nullable{Exception} +end +PkgError(msg::AbstractString) = PkgError(msg, Nullable{Exception}()) +function Base.showerror(io::IO, pkgerr::PkgError) + print(io, pkgerr.msg) + if !isnull(pkgerr.ex) + pkgex = get(pkgerr.ex) + if isa(pkgex, CompositeException) + for cex in pkgex + print(io, "\n=> ") + showerror(io, cex) + end + else + print(io, "\n") + showerror(io, pkgex) + end + end +end + +# DIR +const DIR_NAME = ".julia" +_pkgroot() = abspath(get(ENV,"JULIA_PKGDIR",joinpath(homedir(),DIR_NAME))) +isversioned(p::AbstractString) = ((x,y) = (VERSION.major, VERSION.minor); basename(p) == "v$x.$y") + +function dir() + b = _pkgroot() + x, y = VERSION.major, VERSION.minor + d = joinpath(b,"v$x.$y") + if isdir(d) || !isdir(b) || !isdir(joinpath(b, "METADATA")) + return d + end + return b +end +dir(pkg::AbstractString...) = normpath(dir(),pkg...) + +include("types.jl") +include("reqs.jl") +include("query.jl") +# include("read.jl") +include("resolve.jl") + + +end diff --git a/src/Pkg2/query.jl b/src/Pkg2/query.jl new file mode 100644 index 0000000000000..912972b8e058b --- /dev/null +++ b/src/Pkg2/query.jl @@ -0,0 +1,310 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module Query + +import ..PkgError +using ..Types + +# If there are explicitly required packages, dicards all versions outside +# the allowed range. +# It also propagates requirements: when all allowed versions of a required package +# require some other package, this creates a new implicit requirement. +# The propagation is tracked so that in case a contradiction is detected the error +# message allows to determine the cause. +# This is a pre-pruning step, so it also creates some structures which are later used by pruning +function filter_versions(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Available}}, + bktrc::ResolveBacktrace, uuid_to_name::Dict{String, String}) + allowed = Dict{String,Dict{VersionNumber,Bool}}() + staged = copy(reqs) + while !isempty(staged) + staged_next = Requires() + for (p,vs) in staged + # Parse requirements and store allowed versions. + depsp = deps[p] + if !haskey(allowed, p) + allowedp = Dict{VersionNumber,Bool}(vn=>true for vn in keys(depsp)) + allowed[p] = allowedp + seen = false + else + allowedp = allowed[p] + oldallowedp = copy(allowedp) + seen = true + end + for vn in keys(depsp) + allowedp[vn] &= vn ∈ vs + end + @assert !isempty(allowedp) + if !any(values(allowedp)) + name = haskey(uuid_to_name, p) ? uuid_to_name[p] : "UNKNOWN" + uuid_short = p[1:8] + err_msg = "Unsatisfiable requirements detected for package $name [$uuid_short]:\n" + err_msg *= string(bktrc[p]) + err_msg *= """The intersection of the requirements is $(bktrc[p].versionreq). + None of the available versions can satisfy this requirement.""" + throw(PkgError(err_msg)) + end + + # If we've seen this package already and nothing has changed since + # the last time, we stop here. + seen && allowedp == oldallowedp && continue + + # Propagate requirements: + # if all allowed versions of a required package require some other package, + # then compute the union of the allowed versions for that other package, and + # treat that as a new requirement. + # Start by filtering out the non-allowed versions + fdepsp = Dict{VersionNumber,Available}(vn=>depsp[vn] for vn in keys(depsp) if allowedp[vn]) + # Collect all required packages + isreq = Dict{String,Bool}(rp=>true for a in values(fdepsp) for rp in keys(a.requires)) + # Compute whether a required package appears in all requirements + for a in values(fdepsp), rp in keys(isreq) + haskey(a.requires, rp) || (isreq[rp] = false) + end + staged_new = Set{String}() + for a in values(fdepsp) + for (rp,rvs) in a.requires + # Skip packages that may not be required + isreq[rp] || continue + # Compute the union of the version sets + snvs = get!(staged_next, rp, copy(rvs)) + union!(snvs, rvs) + push!(staged_new, rp) + end + end + for rp in staged_new + srvs = staged_next[rp] + isreq[rp] || continue + bktrcp = get!(bktrc, rp) do; ResolveBacktraceItem(); end + push!(bktrcp, p=>bktrc[p], srvs) + if isa(bktrcp.versionreq, VersionSet) && isempty(bktrcp.versionreq) + name = haskey(uuid_to_name, rp) ? uuid_to_name[rp] : "UNKNOWN" + uuid_short = rp[1:8] + err_msg = "Unsatisfiable requirements detected for package $name [$uuid_short]:\n" + err_msg *= string(bktrcp) + err_msg *= "The intersection of the requirements is empty." + throw(PkgError(err_msg)) + end + end + end + staged = staged_next + end + + filtered_deps = Dict{String,Dict{VersionNumber,Available}}() + for (p,depsp) in deps + filtered_deps[p] = Dict{VersionNumber,Available}() + allowedp = get(allowed, p, Dict{VersionNumber,Bool}()) + fdepsp = filtered_deps[p] + for (vn,a) in depsp + get(allowedp, vn, true) || continue + fdepsp[vn] = a + end + end + + return filtered_deps, allowed +end + +# Reduce the number of versions by creating equivalence classes, and retaining +# only the highest version for each equivalence class. +# Two versions are equivalent if: +# 1) They appear together as dependecies of another package (i.e. for each +# dependency relation, they are both required or both not required) +# 2) They have the same dependencies +# Preliminarily calls filter_versions. +function prune_versions(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Available}}, bktrc::ResolveBacktrace, uuid_to_name::Dict{String, String}) + filtered_deps, allowed = filter_versions(reqs, deps, bktrc, uuid_to_name) + + # To each version in each package, we associate a BitVector. + # It is going to hold a pattern such that all versions with + # the same pattern are equivalent. + vmask = Dict{String,Dict{VersionNumber, BitVector}}() + + # For each package, we examine the dependencies of its versions + # and put together those which are equal. + # While we're at it, we also collect all dependencies into alldeps + alldeps = Dict{String,Set{VersionSet}}() + for (p,fdepsp) in filtered_deps + # Extract unique dependencies lists (aka classes), thereby + # assigning an index to each class. + uniqdepssets = unique(a.requires for a in values(fdepsp)) + + # Store all dependencies seen so far for later use + for r in uniqdepssets, (rp,rvs) in r + get!(alldeps, rp) do; Set{VersionSet}() end + push!(alldeps[rp], rvs) + end + + # If the package has just one version, it's uninteresting + length(deps[p]) == 1 && continue + + # Grow the pattern by the number of classes + luds = length(uniqdepssets) + @assert !haskey(vmask, p) + vmask[p] = Dict{VersionNumber,BitVector}() + vmaskp = vmask[p] + for vn in keys(fdepsp) + vmaskp[vn] = falses(luds) + end + for (vn,a) in fdepsp + vmind = findfirst(uniqdepssets, a.requires) + @assert vmind > 0 + vm = vmaskp[vn] + vm[vmind] = true + end + end + + # Produce dependency patterns. + for (p,vss) in alldeps, vs in vss + # packages with just one version, or dependencies + # which do not distiguish between versions, are not + # interesting + (length(deps[p]) == 1 || vs == VersionSet()) && continue + + # Store the dependency info in the patterns + @assert haskey(vmask, p) + for (vn,vm) in vmask[p] + push!(vm, vn in vs) + end + end + + # At this point, the vmask patterns are computed. We divide them into + # classes so that we can keep just one version for each class. + pruned_vers = Dict{String,Vector{VersionNumber}}() + eq_classes = Dict{String,Dict{VersionNumber,Vector{VersionNumber}}}() + for (p, vmaskp) in vmask + vmask0_uniq = unique(values(vmaskp)) + nc = length(vmask0_uniq) + classes = [VersionNumber[] for c0 = 1:nc] + for (vn,vm) in vmaskp + c0 = findfirst(vmask0_uniq, vm) + push!(classes[c0], vn) + end + map(sort!, classes) + + # For each nonempty class, we store only the highest version) + pruned_vers[p] = VersionNumber[] + prunedp = pruned_vers[p] + eq_classes[p] = Dict{VersionNumber,Vector{VersionNumber}}() + eqclassp = eq_classes[p] + for cl in classes + if !isempty(cl) + vtop = maximum(cl) + push!(prunedp, vtop) + @assert !haskey(eqclassp, vtop) + eqclassp[vtop] = cl + end + end + sort!(prunedp) + end + # Put non-allowed versions into eq_classes + for (p, allowedp) in allowed + haskey(eq_classes, p) || continue + eqclassp = eq_classes[p] + for (vn, a) in allowedp + a && continue + eqclassp[vn] = [vn] + end + end + # Put all remaining packages into eq_classes + for (p, depsp) in deps + haskey(eq_classes, p) && continue + eq_classes[p] = Dict{VersionNumber,Vector{VersionNumber}}() + eqclassp = eq_classes[p] + for vn in keys(depsp) + eqclassp[vn] = [vn] + end + end + + + # Recompute deps. We could simplify them, but it's not worth it + new_deps = Dict{String,Dict{VersionNumber,Available}}() + + for (p,depsp) in deps + @assert !haskey(new_deps, p) + if !haskey(pruned_vers, p) + new_deps[p] = depsp + continue + end + new_deps[p] = Dict{VersionNumber,Available}() + pruned_versp = pruned_vers[p] + for (vn,a) in depsp + vn ∈ pruned_versp || continue + new_deps[p][vn] = a + end + end + + #println("pruning stats:") + #numvers = 0 + #numdeps = 0 + #for (p,d) in deps, (vn,a) in d + # numvers += 1 + # for r in a.requires + # numdeps += 1 + # end + #end + #numnewvers = 0 + #numnewdeps = 0 + #for (p,d) in new_deps, (vn,a) in d + # numnewvers += 1 + # for r in a.requires + # numnewdeps += 1 + # end + #end + #println(" before: vers=$numvers deps=$numdeps") + #println(" after: vers=$numnewvers deps=$numnewdeps") + #println() + + return new_deps, eq_classes +end +prune_versions(env, deps::Dict{String,Dict{VersionNumber,Available}}) = + prune_versions(env, Dict{String,VersionSet}(), deps, ResolveBacktrace()) +prune_versions(env, deps::Dict{String,Dict{VersionNumber,Available}}, bktrc::ResolveBacktrace) = + prune_versions(env, Dict{String,VersionSet}(), deps, bktrc) + +# Build a graph restricted to a subset of the packages +function subdeps(deps::Dict{String,Dict{VersionNumber,Available}}, pkgs::Set{String}) + sub_deps = Dict{String,Dict{VersionNumber,Available}}() + for p in pkgs + haskey(sub_deps, p) || (sub_deps[p] = Dict{VersionNumber,Available}()) + sub_depsp = sub_deps[p] + for (vn,a) in deps[p] + sub_depsp[vn] = a + end + end + + return sub_deps +end + +# Build a subgraph incuding only the (direct and indirect) dependencies +# of a given package set +function dependencies_subset(deps::Dict{String,Dict{VersionNumber,Available}}, pkgs::Set{String}) + staged::Set{String} = filter(p->p in keys(deps), pkgs) + allpkgs = copy(staged) + while !isempty(staged) + staged_next = Set{String}() + for p in staged, a in values(get(deps, p, Dict{VersionNumber,Available}())), rp in keys(a.requires) + rp ∉ allpkgs && rp ≠ "julia" && push!(staged_next, rp) + end + union!(allpkgs, staged_next) + staged = staged_next + end + + return subdeps(deps, allpkgs) +end + +function prune_dependencies(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Available}}, uuid_to_name::Dict{String, String}) + bktrc = ResolveBacktrace() + for (p,vs) in reqs + bktrc[p] = ResolveBacktraceItem(:required, vs) + end + return prune_dependencies(reqs, deps, bktrc, uuid_to_name) +end + +function prune_dependencies(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Available}}, + bktrc::ResolveBacktrace, uuid_to_name::Dict{String, String}) + deps = dependencies_subset(deps, Set{String}(keys(reqs))) + deps, _ = prune_versions(reqs, deps, bktrc, uuid_to_name) + + return deps +end + +end # module diff --git a/src/Pkg2/reqs.jl b/src/Pkg2/reqs.jl new file mode 100644 index 0000000000000..943f726e76ef4 --- /dev/null +++ b/src/Pkg2/reqs.jl @@ -0,0 +1,47 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module Reqs + +import Base: == +import ..PkgError +using ..Reqs +using ..Types + +# representing lines of REQUIRE files + +abstract type Line end +struct Comment <: Line + content::AbstractString +end +struct Requirement <: Line + content::AbstractString + package::AbstractString + versions::VersionSet + system::Vector{AbstractString} + + function Requirement(content::AbstractString) + fields = split(replace(content, r"#.*$", "")) + system = AbstractString[] + while !isempty(fields) && fields[1][1] == '@' + push!(system,shift!(fields)[2:end]) + end + isempty(fields) && throw(PkgError("invalid requires entry: $content")) + package = shift!(fields) + all(field->ismatch(Base.VERSION_REGEX, field), fields) || + throw(PkgError("invalid requires entry for $package: $content")) + versions = VersionNumber[fields...] + issorted(versions) || throw(PkgError("invalid requires entry for $package: $content")) + new(content, package, VersionSet(versions), system) + end +end + +function read(readable::Union{IO,Base.AbstractCmd}) + lines = Line[] + for line in eachline(readable) + push!(lines, ismatch(r"^\s*(?:#|$)", line) ? Comment(line) : Requirement(line)) + end + return lines +end +read(file::AbstractString) = isfile(file) ? open(read,file) : Line[] + +end # module diff --git a/src/Pkg2/resolve.jl b/src/Pkg2/resolve.jl new file mode 100644 index 0000000000000..6e10eee3c4f48 --- /dev/null +++ b/src/Pkg2/resolve.jl @@ -0,0 +1,158 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module Resolve + +include(joinpath("resolve", "versionweight.jl")) +include(joinpath("resolve", "interface.jl")) +include(joinpath("resolve", "maxsum.jl")) + +using ..Types, ..Query, .PkgToMaxSumInterface, .MaxSum +import ..PkgError + +export resolve, sanity_check + +# Use the max-sum algorithm to resolve packages dependencies +function resolve(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Available}}, uuid_to_name::Dict{String,String}) + # init interface structures + interface = Interface(reqs, deps) + + # attempt trivial solution first + ok, sol = greedysolver(interface) + if !ok + # trivial solution failed, use maxsum solver + graph = Graph(interface) + msgs = Messages(interface, graph) + + try + sol = maxsum(graph, msgs) + catch err + isa(err, UnsatError) || rethrow(err) + p = interface.pkgs[err.info] + name = haskey(uuid_to_name, p) ? uuid_to_name[p] : "UNKNOWN" + uuid_short = p[1:8] + # TODO: build tools to analyze the problem, and suggest to use them here. + msg = + """ + resolve is unable to satisfy package requirements. + The problem was detected when trying to find a feasible version + for package $name [$uuid_short]. + However, this only means that package $name [$uuid_short] is involved in an + unsatisfiable or difficult dependency relation, and the root of + the problem may be elsewhere. + """ + if msgs.num_nondecimated != graph.np + msg *= """ + (you may try increasing the value of the JULIA_PKGRESOLVE_ACCURACY + environment variable) + """ + end + ## info("ERROR MESSAGE:\n" * msg) + throw(PkgError(msg)) + end + + # verify solution (debug code) and enforce its optimality + @assert verify_solution(sol, interface) + enforce_optimality!(sol, interface) + @assert verify_solution(sol, interface) + end + + # return the solution as a Dict mapping package_name => sha1 + return compute_output_dict(sol, interface) +end + +# Scan dependencies for (explicit or implicit) contradictions +function sanity_check(deps::Dict{String,Dict{VersionNumber,Available}}, + pkgs::Set{String} = Set{String}()) + isempty(pkgs) || (deps = Query.undirected_dependencies_subset(deps, pkgs)) + + deps, eq_classes = Query.prune_versions(deps) + + ndeps = Dict{String,Dict{VersionNumber,Int}}() + + for (p,depsp) in deps + ndeps[p] = ndepsp = Dict{VersionNumber,Int}() + for (vn,a) in depsp + ndepsp[vn] = length(a.requires) + end + end + + vers = Vector{Tuple{String,VersionNumber,VersionNumber}}(0) + for (p,d) in deps, vn in keys(d) + lvns = VersionNumber[Iterators.filter(vn2->(vn2>vn), keys(d))...] + nvn = isempty(lvns) ? typemax(VersionNumber) : minimum(lvns) + push!(vers, (p,vn,nvn)) + end + sort!(vers, by=pvn->(-ndeps[pvn[1]][pvn[2]])) + + nv = length(vers) + + svdict = Dict{Tuple{String,VersionNumber},Int}(vers[i][1:2]=>i for i = 1:nv) + + checked = falses(nv) + + problematic = Vector{Tuple{String,VersionNumber,String}}(0) + i = 1 + psl = 0 + for (p,vn,nvn) in vers + ndeps[p][vn] == 0 && break + checked[i] && (i += 1; continue) + + sub_reqs = Dict{String,VersionSet}(p=>VersionSet([vn, nvn])) + local sub_deps::Dict{String,Dict{VersionNumber,Available}} + try + sub_deps = Query.prune_dependencies(sub_reqs, deps) + catch err + isa(err, PkgError) || rethrow(err) + ## info("ERROR MESSAGE:\n" * err.msg) + for vneq in eq_classes[p][vn] + push!(problematic, (p, vneq, "")) + end + i += 1 + continue + end + interface = Interface(sub_reqs, sub_deps) + + red_pkgs = interface.pkgs + red_np = interface.np + red_spp = interface.spp + red_pvers = interface.pvers + + ok, sol = greedysolver(interface) + + if !ok + try + graph = Graph(interface) + msgs = Messages(interface, graph) + sol = maxsum(graph, msgs) + ok = verify_solution(sol, interface) + @assert ok + catch err + isa(err, UnsatError) || rethrow(err) + pp = red_pkgs[err.info] + for vneq in eq_classes[p][vn] + push!(problematic, (p, vneq, pp)) + end + end + end + if ok + let p0 = interface.pdict[p] + svn = red_pvers[p0][sol[p0]] + @assert svn == vn + end + + for p0 = 1:red_np + s0 = sol[p0] + if s0 != red_spp[p0] + j = svdict[(red_pkgs[p0], red_pvers[p0][s0])] + checked[j] = true + end + end + checked[i] = true + end + i += 1 + end + + return sort!(problematic) +end + +end # module diff --git a/src/Pkg2/resolve/fieldvalue.jl b/src/Pkg2/resolve/fieldvalue.jl new file mode 100644 index 0000000000000..01549cd8ca4b9 --- /dev/null +++ b/src/Pkg2/resolve/fieldvalue.jl @@ -0,0 +1,112 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module FieldValues + +using ...VersionWeights + +export FieldValue, Field, validmax, secondmax + +# FieldValue is a hierarchical numeric type with 6 levels. +# When summing two FieldValues, the levels are summed independently. +# When comparing them, lower levels take precedence. +# The levels are used as such: +# l0 : for hard constraints (dependencies and requirements) +# l1 : for favoring higher versions of the explicitly required +# packages +# l2 : for favoring higher versions of all other packages +# l3 : for favoring uninstallation of non-needed packages +# l4 : for favoring dependants over dependencies +# l5 : for symmetry-breaking random noise +# +struct FieldValue + l0::Int + l1::VersionWeight + l2::VersionWeight + l3::Int + l4::Int + l5::Int128 +end +FieldValue(l0::Integer, l1::VersionWeight, l2::VersionWeight, l3::Integer, l4::Integer) = FieldValue(l0, l1, l2, l3, l4, Int128(0)) +FieldValue(l0::Integer, l1::VersionWeight, l2::VersionWeight, l3::Integer) = FieldValue(l0, l1, l2, l3, 0) +FieldValue(l0::Integer, l1::VersionWeight, l2::VersionWeight) = FieldValue(l0, l1, l2, 0) +FieldValue(l0::Integer, l1::VersionWeight) = FieldValue(l0, l1, zero(VersionWeight)) +FieldValue(l0::Integer) = FieldValue(l0, zero(VersionWeight)) +FieldValue() = FieldValue(0) + +const Field = Vector{FieldValue} + +Base.zero(::Type{FieldValue}) = FieldValue() + +Base.typemin(::Type{FieldValue}) = (x=typemin(Int); y=typemin(VersionWeight); FieldValue(x, y, y, x, x, typemin(Int128))) + +Base.:-(a::FieldValue, b::FieldValue) = FieldValue(a.l0-b.l0, a.l1-b.l1, a.l2-b.l2, a.l3-b.l3, a.l4-b.l4, a.l5-b.l5) +Base.:+(a::FieldValue, b::FieldValue) = FieldValue(a.l0+b.l0, a.l1+b.l1, a.l2+b.l2, a.l3+b.l3, a.l4+b.l4, a.l5+b.l5) + +function Base.isless(a::FieldValue, b::FieldValue) + a.l0 < b.l0 && return true + a.l0 > b.l0 && return false + c = cmp(a.l1, b.l1) + c < 0 && return true + c > 0 && return false + c = cmp(a.l2, b.l2) + c < 0 && return true + c > 0 && return false + a.l3 < b.l3 && return true + a.l3 > b.l3 && return false + a.l4 < b.l4 && return true + a.l4 > b.l4 && return false + a.l5 < b.l5 && return true + return false +end + +Base.:(==)(a::FieldValue, b::FieldValue) = + a.l0 == b.l0 && a.l1 == b.l1 && a.l2 == b.l2 && a.l3 == b.l3 && a.l4 == b.l4 && a.l5 == b.l5 + +Base.abs(a::FieldValue) = FieldValue(abs(a.l0), abs(a.l1), abs(a.l2), abs(a.l3), abs(a.l4), abs(a.l5)) + +Base.copy(a::FieldValue) = FieldValue(a.l0, copy(a.l1), copy(a.l2), a.l3, a.l4, a.l5) + +function Base.unsafe_copy!(dest::Field, doffs, src::Field, soffs, n) + for i = 1:n + dest[doffs+i-1] = copy(src[soffs+i-1]) + end + return dest +end + +# if the maximum field has l0 < 0, it means that +# some hard constraint is being violated +validmax(a::FieldValue) = a.l0 >= 0 + +# like usual indmax, but favors the highest indices +# in case of a tie +function Base.indmax(f::Field) + m = typemin(FieldValue) + mi = 0 + for j = length(f):-1:1 + if f[j] > m + m = f[j] + mi = j + end + end + @assert mi != 0 + return mi +end + +# secondmax returns the (normalized) value of the second maximum in a +# field. It's used to determine the most polarized field. +function secondmax(f::Field) + m = typemin(FieldValue) + m2 = typemin(FieldValue) + for i = 1:length(f) + a = f[i] + if a > m + m2 = m + m = a + elseif a > m2 + m2 = a + end + end + return m2 - m +end + +end diff --git a/src/Pkg2/resolve/interface.jl b/src/Pkg2/resolve/interface.jl new file mode 100644 index 0000000000000..9ed3c6d531db5 --- /dev/null +++ b/src/Pkg2/resolve/interface.jl @@ -0,0 +1,350 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module PkgToMaxSumInterface + +using ...Types, ...Query, ..VersionWeights + +export Interface, compute_output_dict, greedysolver, + verify_solution, enforce_optimality! + +# A collection of objects which allow interfacing external (Pkg) and +# internal (MaxSum) representation +mutable struct Interface + # requirements and dependencies, in external representation + reqs::Requires + deps::Dict{String,Dict{VersionNumber,Available}} + + # packages list + pkgs::Vector{String} + + # number of packages + np::Int + + # states per package: one per version + uninstalled + spp::Vector{Int} + + # pakage dict: associates an index to each package name + pdict::Dict{String,Int} + + # package versions: for each package, keep the list of the + # possible version numbers; this defines a + # mapping from version numbers of a package + # to indices + pvers::Vector{Vector{VersionNumber}} + + # versions dict: associates a version index to each package + # version; such that + # pvers[p0][vdict[p0][vn]] = vn + vdict::Vector{Dict{VersionNumber,Int}} + + # version weights: the weight for each version of each package + # (versions include the uninstalled state; the + # higher the weight, the more favored the version) + vweight::Vector{Vector{VersionWeight}} + + function Interface(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Available}}) + # generate pkgs + pkgs = sort!(String[Set{String}(keys(deps))...]) + + np = length(pkgs) + + # generate pdict + pdict = Dict{String,Int}(pkgs[i] => i for i = 1:np) + + # generate spp and pvers + spp = Vector{Int}(np) + + pvers = [VersionNumber[] for i = 1:np] + + for (p,depsp) in deps, vn in keys(depsp) + p0 = pdict[p] + push!(pvers[p0], vn) + end + for p0 = 1:np + sort!(pvers[p0]) + spp[p0] = length(pvers[p0]) + 1 + end + + # generate vdict + vdict = [Dict{VersionNumber,Int}() for p0 = 1:np] + for (p,depsp) in deps + p0 = pdict[p] + vdict0 = vdict[p0] + pvers0 = pvers[p0] + for vn in keys(depsp) + for v0 in 1:length(pvers0) + if pvers0[v0] == vn + vdict0[vn] = v0 + break + end + end + end + end + + ## generate wveights: + vweight = Vector{Vector{VersionWeight}}(np) + for p0 = 1:np + pvers0 = pvers[p0] + spp0 = spp[p0] + vweight0 = vweight[p0] = Vector{VersionWeight}(spp0) + for v0 = 1:spp0-1 + vweight0[v0] = VersionWeight(pvers0[v0]) + end + vweight0[spp0] = VersionWeight(v"0") # last version means uninstalled + end + + return new(reqs, deps, pkgs, np, spp, pdict, pvers, vdict, vweight) + end +end + +# The output format is a Dict which associates a VersionNumber to each installed package name +function compute_output_dict(sol::Vector{Int}, interface::Interface) + pkgs = interface.pkgs + np = interface.np + pvers = interface.pvers + spp = interface.spp + + want = Dict{String,VersionNumber}() + for p0 = 1:np + p = pkgs[p0] + s = sol[p0] + if s != spp[p0] + v = pvers[p0][s] + want[p] = v + end + end + + return want +end + +# Produce a trivial solution: try to maximize each version; +# bail out as soon as some non-trivial requirements are detected. +function greedysolver(interface::Interface) + reqs = interface.reqs + deps = interface.deps + spp = interface.spp + pdict = interface.pdict + pvers = interface.pvers + np = interface.np + + # initialize solution: all uninstalled + sol = [spp[p0] for p0 = 1:np] + + # set up required packages to their highest allowed versions + for (rp,rvs) in reqs + rp0 = pdict[rp] + # look for the highest version which satisfies the requirements + rv = spp[rp0] - 1 + while rv > 0 + rvn = pvers[rp0][rv] + rvn ∈ rvs && break + rv -= 1 + end + @assert rv > 0 + sol[rp0] = rv + end + + # we start from required packages and explore the graph + # following dependencies + staged = Set{String}(keys(reqs)) + seen = copy(staged) + + while !isempty(staged) + staged_next = Set{String}() + for p in staged + p0 = pdict[p] + @assert sol[p0] < spp[p0] + vn = pvers[p0][sol[p0]] + a = deps[p][vn] + + # scan dependencies + for (rp,rvs) in a.requires + rp0 = pdict[rp] + # look for the highest version which satisfies the requirements + rv = spp[rp0] - 1 + while rv > 0 + rvn = pvers[rp0][rv] + rvn ∈ rvs && break + rv -= 1 + end + # if we found a version, and the package was uninstalled + # or the same version was already selected, we're ok; + # otherwise we can't be sure what the optimal configuration is + # and we bail out + if rv > 0 && (sol[rp0] == spp[rp0] || sol[rp0] == rv) + sol[rp0] = rv + else + return (false, Int[]) + end + + rp ∈ seen || push!(staged_next, rp) + end + end + union!(seen, staged_next) + staged = staged_next + end + + @assert verify_solution(sol, interface) + + return true, sol +end + +# verifies that the solution fulfills all hard constraints +# (requirements and dependencies) +function verify_solution(sol::Vector{Int}, interface::Interface) + reqs = interface.reqs + deps = interface.deps + spp = interface.spp + pdict = interface.pdict + pvers = interface.pvers + vdict = interface.vdict + + # verify requirements + for (p,vs) in reqs + p0 = pdict[p] + sol[p0] != spp[p0] || return false + vn = pvers[p0][sol[p0]] + vn ∈ vs || return false + end + + # verify dependencies + for (p,d) in deps + p0 = pdict[p] + vdict0 = vdict[p0] + for (vn,a) in d + v0 = vdict0[vn] + if sol[p0] == v0 + for (rp, rvs) in a.requires + p1 = pdict[rp] + sol[p1] != spp[p1] || return false + vn = pvers[p1][sol[p1]] + vn ∈ rvs || return false + end + end + end + end + return true +end + +# Push the given solution to a local optimium if needed +function enforce_optimality!(sol::Vector{Int}, interface::Interface) + np = interface.np + + reqs = interface.reqs + deps = interface.deps + pkgs = interface.pkgs + spp = interface.spp + pdict = interface.pdict + pvers = interface.pvers + vdict = interface.vdict + + # prepare some useful structures + # pdeps[p0][v0] has all dependencies of package p0 version v0 + pdeps = [Vector{Requires}(spp[p0]-1) for p0 = 1:np] + # prevdeps[p1][p0][v0] is the VersionSet of package p1 which package p0 version v0 + # depends upon + prevdeps = [Dict{Int,Dict{Int,VersionSet}}() for p0 = 1:np] + + for (p,d) in deps + p0 = pdict[p] + vdict0 = vdict[p0] + for (vn,a) in d + v0 = vdict0[vn] + pdeps[p0][v0] = a.requires + for (rp, rvs) in a.requires + p1 = pdict[rp] + if !haskey(prevdeps[p1], p0) + prevdeps[p1][p0] = Dict{Int,VersionSet}() + end + prevdeps[p1][p0][v0] = rvs + end + end + end + + restart = true + while restart + restart = false + for p0 = 1:np + s0 = sol[p0] + if s0 >= spp[p0] - 1 + # either the package is not installed, + # or it's already at the maximum version + continue + end + viol = false + # check if the higher version has a depencency which + # would be violated by the state of the remaining packages + for (p,vs) in pdeps[p0][s0+1] + p1 = pdict[p] + if sol[p1] == spp[p1] + # the dependency is violated because + # the other package is not being installed + viol = true + break + end + vn = pvers[p1][sol[p1]] + if vn ∉ vs + # the dependency is violated because + # the other package version is invalid + viol = true + break + end + end + viol && continue + + # check if bumping the version would violate some + # dependency of another package + for (p1,d) in prevdeps[p0] + vs = get(d, sol[p1], nothing) + vs === nothing && continue + vn = pvers[p0][s0+1] + if vn ∉ vs + # bumping the version would violate + # the dependency + viol = true + break + end + end + viol && continue + # So the solution is non-optimal: we bump it manually + #warn("nonoptimal solution for package $(interface.pkgs[p0]): sol=$s0") + sol[p0] += 1 + restart = true + end + end + + # Finally uninstall unneeded packages: + # start from the required ones and keep only + # the packages reachable from them along the graph + uninst = trues(np) + staged = Set{String}(keys(reqs)) + seen = copy(staged) + + while !isempty(staged) + staged_next = Set{String}() + for p in staged + p0 = pdict[p] + uninst[p0] = false + @assert sol[p0] < spp[p0] + vn = pvers[p0][sol[p0]] + a = deps[p][vn] + + # scan dependencies + for (rp,rvs) in a.requires + rp0 = pdict[rp] + @assert sol[rp0] < spp[rp0] && pvers[rp0][sol[rp0]] ∈ rvs + rp ∈ seen || push!(staged_next, rp) + end + end + union!(seen, staged_next) + staged = staged_next + end + + for p0 in find(uninst) + sol[p0] = spp[p0] + end + + return +end + +end diff --git a/src/Pkg2/resolve/maxsum.jl b/src/Pkg2/resolve/maxsum.jl new file mode 100644 index 0000000000000..167d2710e7f37 --- /dev/null +++ b/src/Pkg2/resolve/maxsum.jl @@ -0,0 +1,488 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module MaxSum + +include("fieldvalue.jl") + +using .FieldValues, ..VersionWeights, ..PkgToMaxSumInterface + +export UnsatError, Graph, Messages, maxsum + +# An exception type used internally to signal that an unsatisfiable +# constraint was detected +struct UnsatError <: Exception + info +end + +# Some parameters to drive the decimation process +mutable struct MaxSumParams + nondec_iterations # number of initial iterations before starting + # decimation + dec_interval # number of iterations between decimations + dec_fraction # fraction of nodes to decimate at every decimation + # step + + function MaxSumParams() + accuracy = parse(Int, get(ENV, "JULIA_PKGRESOLVE_ACCURACY", "1")) + if accuracy <= 0 + error("JULIA_PKGRESOLVE_ACCURACY must be > 0") + end + nondec_iterations = accuracy * 20 + dec_interval = accuracy * 10 + dec_fraction = 0.05 / accuracy + return new(nondec_iterations, dec_interval, dec_fraction) + end +end + +# Graph holds the graph structure onto which max-sum is run, in +# sparse format +mutable struct Graph + # adjacency matrix: + # for each package, has the list of neighbors + # indices (both dependencies and dependants) + gadj::Vector{Vector{Int}} + + # compatibility mask: + # for each package p0 has a list of bool masks. + # Each entry in the list gmsk[p0] is relative to the + # package p1 as read from gadj[p0]. + # Each mask has dimension spp1 x spp0, where + # spp0 is the number of states of p0, and + # spp1 is the number of states of p1. + gmsk::Vector{Vector{BitMatrix}} + + # dependency direction: + # keeps track of which direction the dependency goes + # takes 3 values: + # 1 = dependant + # -1 = dependency + # 0 = both + # Used to break symmetry between dependants and + # dependencies (introduces a FieldValue at level l3). + # The "both" case is for when there are dependency + # relations which go both ways, in which case the + # noise is left to discriminate in case of ties + gdir::Vector{Vector{Int}} + + # adjacency dict: + # allows one to retrieve the indices in gadj, so that + # gadj[p0][adjdict[p1][p0]] = p1 + adjdict::Vector{Dict{Int,Int}} + + # states per package: same as in Interface + spp::Vector{Int} + + # update order: shuffled at each iteration + perm::Vector{Int} + + # number of packages (all Vectors above have this length) + np::Int + + function Graph(interface::Interface) + deps = interface.deps + np = interface.np + + spp = interface.spp + pdict = interface.pdict + pvers = interface.pvers + vdict = interface.vdict + + gadj = [Int[] for i = 1:np] + gmsk = [BitMatrix[] for i = 1:np] + gdir = [Int[] for i = 1:np] + adjdict = [Dict{Int,Int}() for i = 1:np] + + for (p,d) in deps + p0 = pdict[p] + vdict0 = vdict[p0] + for (vn,a) in d + v0 = vdict0[vn] + for (rp, rvs) in a.requires + p1 = pdict[rp] + + j0 = 1 + while j0 <= length(gadj[p0]) && gadj[p0][j0] != p1 + j0 += 1 + end + j1 = 1 + while j1 <= length(gadj[p1]) && gadj[p1][j1] != p0 + j1 += 1 + end + @assert (j0 > length(gadj[p0]) && j1 > length(gadj[p1])) || + (j0 <= length(gadj[p0]) && j1 <= length(gadj[p1])) + + if j0 > length(gadj[p0]) + push!(gadj[p0], p1) + push!(gadj[p1], p0) + j0 = length(gadj[p0]) + j1 = length(gadj[p1]) + + adjdict[p1][p0] = j0 + adjdict[p0][p1] = j1 + + bm = trues(spp[p1], spp[p0]) + bmt = bm' + + push!(gmsk[p0], bm) + push!(gmsk[p1], bmt) + + push!(gdir[p0], 1) + push!(gdir[p1], -1) + else + bm = gmsk[p0][j0] + bmt = gmsk[p1][j1] + if gdir[p0][j0] == -1 + gdir[p0][j0] = 0 + gdir[p1][j1] = 0 + end + end + + for v1 = 1:length(pvers[p1]) + if pvers[p1][v1] ∉ rvs + bm[v1, v0] = false + bmt[v0, v1] = false + end + end + bm[end,v0] = false + bmt[v0,end] = false + end + end + end + + perm = [1:np;] + + return new(gadj, gmsk, gdir, adjdict, spp, perm, np) + end +end + +# Messages has the cavity messages and the total fields, and +# gets updated iteratively (and occasionally decimated) until +# convergence +mutable struct Messages + # cavity incoming messages: for each package p0, + # for each neighbor p1 of p0, + # msg[p0][p1] is a vector of length spp[p0] + # messages are normalized (i.e. the max is always 0) + msg::Vector{Vector{Field}} + + # overall fields: for each package p0, + # fld[p0] is a vector of length spp[p0] + # fields are not normalized + fld::Vector{Field} + + # backup of the initial value of fld, to be used when resetting + initial_fld::Vector{Field} + + # keep track of which variables have been decimated + decimated::BitVector + num_nondecimated::Int + + function Messages(interface::Interface, graph::Graph) + reqs = interface.reqs + pkgs = interface.pkgs + np = interface.np + spp = interface.spp + pvers = interface.pvers + pdict = interface.pdict + vweight = interface.vweight + + # a "deterministic noise" function based on hashes + function noise(p0::Int, v0::Int) + s = pkgs[p0] * string(v0 == spp[p0] ? "UNINST" : pvers[p0][v0]) + Int128(hash(s)) + end + + # external fields: there are 2 terms, a noise to break potential symmetries + # and one to favor newest versions over older, and no-version over all + fld = [[FieldValue(0, zero(VersionWeight), vweight[p0][v0], (v0==spp[p0]), 0, noise(p0,v0)) for v0 = 1:spp[p0]] for p0 = 1:np] + + # enforce requirements + for (rp, rvs) in reqs + p0 = pdict[rp] + pvers0 = pvers[p0] + fld0 = fld[p0] + for v0 = 1:spp[p0]-1 + vn = pvers0[v0] + if !in(vn, rvs) + # the state is forbidden by requirements + fld0[v0] = FieldValue(-1) + else + # the state is one of those explicitly requested: + # favor it at a higer level than normal (upgrade + # FieldValue from l2 to l1) + fld0[v0] += FieldValue(0, vweight[p0][v0], -vweight[p0][v0]) + end + end + # the uninstalled state is forbidden by requirements + fld0[spp[p0]] = FieldValue(-1) + end + # normalize fields + for p0 = 1:np + m = maximum(fld[p0]) + for v0 = 1:spp[p0] + fld[p0][v0] -= m + end + end + + initial_fld = deepcopy(fld) + + # initialize cavity messages to 0 + gadj = graph.gadj + msg = [[zeros(FieldValue, spp[p0]) for p1 = 1:length(gadj[p0])] for p0 = 1:np] + + return new(msg, fld, initial_fld, falses(np), np) + end +end + +function getsolution(msgs::Messages) + # the solution is just the location of the maximum in + # each field + + fld = msgs.fld + np = length(fld) + sol = Vector{Int}(np) + for p0 = 1:np + fld0 = fld[p0] + s0 = indmax(fld0) + if !validmax(fld0[s0]) + throw(UnsatError(p0)) + end + sol[p0] = s0 + end + return sol +end + +# This is the core of the max-sum solver: +# for a given node p0 (i.e. a package) updates all +# input cavity messages and fields of its neighbors +function update(p0::Int, graph::Graph, msgs::Messages) + gadj = graph.gadj + gmsk = graph.gmsk + gdir = graph.gdir + adjdict = graph.adjdict + spp = graph.spp + np = graph.np + msg = msgs.msg + fld = msgs.fld + decimated = msgs.decimated + + maxdiff = zero(FieldValue) + + gadj0 = gadj[p0] + msg0 = msg[p0] + fld0 = fld[p0] + spp0 = spp[p0] + adjdict0 = adjdict[p0] + + # iterate over all neighbors of p0 + for j0 in 1:length(gadj0) + + p1 = gadj0[j0] + decimated[p1] && continue + j1 = adjdict0[p1] + #@assert j0 == adjdict[p1][p0] + bm1 = gmsk[p1][j1] + dir1 = gdir[p1][j1] + spp1 = spp[p1] + msg1 = msg[p1] + + # compute the output cavity message p0->p1 + cavmsg = fld0 - msg0[j0] + + if dir1 == -1 + # p0 depends on p1 + for v0 = 1:spp0-1 + cavmsg[v0] += FieldValue(0, VersionWeight(0), VersionWeight(0), 0, v0) + end + end + + # keep the old input cavity message p0->p1 + oldmsg = msg1[j1] + + # init the new message to minus infinity + newmsg = [FieldValue(-1) for v1 = 1:spp1] + + # compute the new message by passing cavmsg + # through the constraint encoded in the bitmask + # (nearly equivalent to: + # newmsg = [maximum(cavmsg[bm1[:,v1]]) for v1 = 1:spp1] + # except for the gnrg term) + m = FieldValue(-1) + for v1 = 1:spp1 + for v0 = 1:spp0 + if bm1[v0, v1] + newmsg[v1] = max(newmsg[v1], cavmsg[v0]) + end + end + if dir1 == 1 && v1 != spp1 + # p1 depends on p0 + newmsg[v1] += FieldValue(0, VersionWeight(0), VersionWeight(0), 0, v1) + end + m = max(m, newmsg[v1]) + end + if !validmax(m) + # No state available without violating some + # hard constraint + throw(UnsatError(p1)) + end + + # normalize the new message + for v1 = 1:spp1 + newmsg[v1] -= m + end + + diff = newmsg - oldmsg + maxdiff = max(maxdiff, maximum(abs.(diff))) + + # update the field of p1 + fld1 = fld[p1] + for v1 = 1:spp1 + fld1[v1] += diff[v1] + end + + # put the newly computed message in place + msg1[j1] = newmsg + end + return maxdiff +end + +# A simple shuffling machinery for the update order in iterate() +# (woulnd't pass any random quality test but it's arguably enough) +let step=1 +global shuffleperm, shuffleperminit +shuffleperminit() = (step = 1) +function shuffleperm(graph::Graph) + perm = graph.perm + np = graph.np + for j = np:-1:2 + k = mod(step,j)+1 + perm[j], perm[k] = perm[k], perm[j] + step += isodd(j) ? 1 : k + end + #@assert isperm(perm) +end +end + +# Call update for all nodes (i.e. packages) in +# random order +function iterate(graph::Graph, msgs::Messages) + np = graph.np + + maxdiff = zero(FieldValue) + shuffleperm(graph) + perm = graph.perm + for p0 in perm + maxdiff0 = update(p0, graph, msgs) + maxdiff = max(maxdiff, maxdiff0) + end + return maxdiff +end + +function decimate1(p0::Int, graph::Graph, msgs::Messages) + @assert !msgs.decimated[p0] + fld0 = msgs.fld[p0] + s0 = indmax(fld0) + #println("DECIMATING $p0 ($(packages()[p0]) s0=$s0)") + for v0 = 1:length(fld0) + if v0 != s0 + fld0[v0] = FieldValue(-1) + end + end + msgs.decimated[p0] = true + msgs.num_nondecimated -= 1 +end + +function reset_messages!(msgs::Messages) + msg = msgs.msg + fld = msgs.fld + initial_fld = msgs.initial_fld + decimated = msgs.decimated + np = length(fld) + for p0 = 1:np + map(m->fill!(m, zero(FieldValue)), msg[p0]) + decimated[p0] && continue + fld[p0] = copy(initial_fld[p0]) + end + return msgs +end + +# If normal convergence fails (or is too slow) fix the most +# polarized packages by adding extra infinite fields on every state +# but the maximum +function decimate(n::Int, graph::Graph, msgs::Messages) + #println("DECIMATING $n NODES") + fld = msgs.fld + decimated = msgs.decimated + fldorder = sortperm(fld, by=secondmax) + for p0 in fldorder + decimated[p0] && continue + decimate1(p0, graph, msgs) + n -= 1 + n == 0 && break + end + @assert n == 0 + reset_messages!(msgs) + return +end + +# In case ties still exist at convergence, break them and +# keep converging +function break_ties(msgs::Messages) + fld = msgs.fld + for p0 = 1:length(fld) + fld0 = fld[p0] + z = 0 + m = typemin(FieldValue) + for v0 = 1:length(fld0) + if fld0[v0] > m + m = fld0[v0] + z = 1 + elseif fld0[v0] == m + z += 1 + end + end + if z > 1 + #println("TIE! p0=$p0") + decimate1(p0, msgs) + return false + end + end + return true +end + +# Iterative solver: run iterate() until convergence +# (occasionally calling decimate()) +function maxsum(graph::Graph, msgs::Messages) + params = MaxSumParams() + + it = 0 + shuffleperminit() + while true + it += 1 + maxdiff = iterate(graph, msgs) + #println("it = $it maxdiff = $maxdiff") + + if maxdiff == zero(FieldValue) + break_ties(msgs) && break + continue + end + if it >= params.nondec_iterations && + (it - params.nondec_iterations) % params.dec_interval == 0 + numdec = clamp(floor(Int, params.dec_fraction * graph.np), 1, msgs.num_nondecimated) + decimate(numdec, graph, msgs) + msgs.num_nondecimated == 0 && break + end + end + + # Finally, decimate everything just to + # check against inconsistencies + # (old_numnondec is saved just to prevent + # wrong messages about accuracy) + old_numnondec = msgs.num_nondecimated + decimate(msgs.num_nondecimated, graph, msgs) + msgs.num_nondecimated = old_numnondec + + return getsolution(msgs) +end + +end diff --git a/src/Pkg2/resolve/versionweight.jl b/src/Pkg2/resolve/versionweight.jl new file mode 100644 index 0000000000000..38c091cafc12f --- /dev/null +++ b/src/Pkg2/resolve/versionweight.jl @@ -0,0 +1,221 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module VersionWeights + +export VersionWeight + +struct HierarchicalValue{T} + v::Vector{T} + rest::T +end + +HierarchicalValue(v::Vector{T}) where {T} = HierarchicalValue{T}(v, zero(T)) +HierarchicalValue(T::Type) = HierarchicalValue(T[]) + +Base.zero(::Type{HierarchicalValue{T}}) where {T} = HierarchicalValue(T) + +Base.typemin(::Type{HierarchicalValue{T}}) where {T} = HierarchicalValue(T[], typemin(T)) + +for f in (:-, :+) + @eval function Base.$f(a::HierarchicalValue{T}, b::HierarchicalValue{T}) where T + av = a.v + bv = b.v + la = length(a.v) + lb = length(b.v) + l0 = min(la, lb) + l1 = max(la, lb) + ld = la - lb + rv = Vector{T}(l1) + rf = ($f)(a.rest, b.rest) + @inbounds for i = 1:l0 + rv[i] = ($f)(av[i], bv[i]) + end + @inbounds for i = l0+1:l0+ld + rv[i] = ($f)(av[i], b.rest) + end + @inbounds for i = l0+1:l0-ld + rv[i] = ($f)(a.rest, bv[i]) + end + return HierarchicalValue(rv, rf) + end +end + +Base.:-(a::HierarchicalValue) = HierarchicalValue(-a.v, -a.rest) + +function Base.cmp(a::HierarchicalValue{T}, b::HierarchicalValue{T}) where T + av = a.v + bv = b.v + la = length(a.v) + lb = length(b.v) + l0 = min(la, lb) + l1 = max(la, lb) + ld = la - lb + @inbounds for i = 1:l0 + c = cmp(av[i], bv[i]); c != 0 && return c + end + @inbounds for i = l0+1:l0+ld + c = cmp(av[i], b.rest); c != 0 && return c + end + @inbounds for i = l0+1:l0-ld + c = cmp(a.rest, bv[i]); c != 0 && return c + end + return cmp(a.rest, b.rest) +end +Base.isless(a::HierarchicalValue{T}, b::HierarchicalValue{T}) where {T} = cmp(a,b) < 0 +Base.:(==)(a::HierarchicalValue{T}, b::HierarchicalValue{T}) where {T} = cmp(a,b) == 0 + +Base.abs(a::HierarchicalValue{T}) where {T} = HierarchicalValue(T[abs(x) for x in a.v], abs(a.rest)) + +Base.copy(a::HierarchicalValue{T}) where {T} = HierarchicalValue(T[copy(x) for x in a.v], copy(a.rest)) + +struct VWPreBuildItem + nonempty::Int + s::HierarchicalValue{Int} + i::Int +end +VWPreBuildItem() = VWPreBuildItem(0, HierarchicalValue(Int), 0) +VWPreBuildItem(i::Integer) = VWPreBuildItem(1, HierarchicalValue(Int), i) +VWPreBuildItem(s::String) = VWPreBuildItem(1, HierarchicalValue(Int[s...]), 0) + +Base.zero(::Type{VWPreBuildItem}) = VWPreBuildItem() + +Base.typemin(::Type{VWPreBuildItem}) = (x=typemin(Int); VWPreBuildItem(x, typemin(HierarchicalValue{Int}), x)) + +Base.:-(a::VWPreBuildItem, b::VWPreBuildItem) = VWPreBuildItem(a.nonempty-b.nonempty, a.s-b.s, a.i-b.i) +Base.:+(a::VWPreBuildItem, b::VWPreBuildItem) = VWPreBuildItem(a.nonempty+b.nonempty, a.s+b.s, a.i+b.i) + +Base.:-(a::VWPreBuildItem) = VWPreBuildItem(-a.nonempty, -a.s, -a.i) + +function Base.cmp(a::VWPreBuildItem, b::VWPreBuildItem) + c = cmp(a.nonempty, b.nonempty); c != 0 && return c + c = cmp(a.s, b.s); c != 0 && return c + return cmp(a.i, b.i) +end +Base.isless(a::VWPreBuildItem, b::VWPreBuildItem) = cmp(a,b) < 0 +Base.:(==)(a::VWPreBuildItem, b::VWPreBuildItem) = cmp(a,b) == 0 + +Base.abs(a::VWPreBuildItem) = VWPreBuildItem(abs(a.nonempty), abs(a.s), abs(a.i)) + +Base.copy(a::VWPreBuildItem) = VWPreBuildItem(a.nonempty, copy(a.s), a.i) + +struct VWPreBuild + nonempty::Int + w::HierarchicalValue{VWPreBuildItem} +end + +const _vwprebuild_zero = VWPreBuild(0, HierarchicalValue(VWPreBuildItem)) + +function VWPreBuild(ispre::Bool, desc::Tuple{Vararg{Union{Integer,String}}}) + isempty(desc) && return _vwprebuild_zero + desc == ("",) && return VWPreBuild(ispre ? -1 : 1, HierarchicalValue(VWPreBuildItem[])) + nonempty = ispre ? -1 : 0 + w = Vector{VWPreBuildItem}(length(desc)) + i = 1 + @inbounds for item in desc + w[i] = VWPreBuildItem(item) + i += 1 + end + return VWPreBuild(nonempty, HierarchicalValue(w)) +end +VWPreBuild() = _vwprebuild_zero + +Base.zero(::Type{VWPreBuild}) = VWPreBuild() + +const _vwprebuild_min = VWPreBuild(typemin(Int), typemin(HierarchicalValue{VWPreBuildItem})) +Base.typemin(::Type{VWPreBuild}) = _vwprebuild_min + +function Base.:(-)(a::VWPreBuild, b::VWPreBuild) + b === _vwprebuild_zero && return a + a === _vwprebuild_zero && return -b + VWPreBuild(a.nonempty-b.nonempty, a.w-b.w) +end +function Base.:(+)(a::VWPreBuild, b::VWPreBuild) + b === _vwprebuild_zero && return a + a === _vwprebuild_zero && return b + VWPreBuild(a.nonempty+b.nonempty, a.w+b.w) +end + +function Base.:(-)(a::VWPreBuild) + a === _vwprebuild_zero && return a + VWPreBuild(-a.nonempty, -a.w) +end + +@inline function Base.cmp(a::VWPreBuild, b::VWPreBuild) + a === _vwprebuild_zero && b === _vwprebuild_zero && return 0 + c = cmp(a.nonempty, b.nonempty); c != 0 && return c + return cmp(a.w, b.w) +end +Base.isless(a::VWPreBuild, b::VWPreBuild) = cmp(a,b) < 0 +Base.:(==)(a::VWPreBuild, b::VWPreBuild) = cmp(a,b) == 0 + +function Base.abs(a::VWPreBuild) + a === _vwprebuild_zero && return a + VWPreBuild(abs(a.nonempty), abs(a.w)) +end + +function Base.copy(a::VWPreBuild) + a === _vwprebuild_zero && return a + VWPreBuild(a.nonempty, copy(a.w)) +end + +function Base.deepcopy_internal(a::VWPreBuild, dict::ObjectIdDict) + haskey(dict, a) && return dict[a] + b = (a === _vwprebuild_zero) ? _vwprebuild_zero : VWPreBuild(a.nonempty, Base.deepcopy_internal(a.w, dict)) + dict[a] = b + return b +end + +# The numeric type used to determine how the different +# versions of a package should be weighed +struct VersionWeight + major::Int + minor::Int + patch::Int + prerelease::VWPreBuild + build::VWPreBuild +end +VersionWeight(major::Int, minor::Int, patch::Int, prerelease::VWPreBuild) = VersionWeight(major, minor, patch, prerelease, zero(VWPreBuild)) +VersionWeight(major::Int, minor::Int, patch::Int) = VersionWeight(major, minor, patch, zero(VWPreBuild)) +VersionWeight(major::Int, minor::Int) = VersionWeight(major, minor, 0) +VersionWeight(major::Int) = VersionWeight(major, 0) +VersionWeight() = VersionWeight(0) + +VersionWeight(vn::VersionNumber) = + VersionWeight(vn.major, vn.minor, vn.patch, + VWPreBuild(true, vn.prerelease), VWPreBuild(false, vn.build)) + +Base.zero(::Type{VersionWeight}) = VersionWeight() + +Base.typemin(::Type{VersionWeight}) = (x=typemin(Int); y=typemin(VWPreBuild); VersionWeight(x, x, x, y, y)) + +Base.:(-)(a::VersionWeight, b::VersionWeight) = + VersionWeight(a.major-b.major, a.minor-b.minor, a.patch-b.patch, + a.prerelease-b.prerelease, a.build-b.build) + +Base.:(+)(a::VersionWeight, b::VersionWeight) = + VersionWeight(a.major+b.major, a.minor+b.minor, a.patch+b.patch, + a.prerelease+b.prerelease, a.build+b.build) + +Base.:(-)(a::VersionWeight) = + VersionWeight(-a.major, -a.minor, -a.patch, + -a.prerelease, -a.build) + +function Base.cmp(a::VersionWeight, b::VersionWeight) + c = cmp(a.major, b.major); c != 0 && return c + c = cmp(a.minor, b.minor); c != 0 && return c + c = cmp(a.patch, b.patch); c != 0 && return c + c = cmp(a.prerelease, b.prerelease); c != 0 && return c + return cmp(a.build, b.build) +end +Base.isless(a::VersionWeight, b::VersionWeight) = cmp(a,b) < 0 +Base.:(==)(a::VersionWeight, b::VersionWeight) = cmp(a,b) == 0 + +Base.abs(a::VersionWeight) = + VersionWeight(abs(a.major), abs(a.minor), abs(a.patch), + abs(a.prerelease), abs(a.build)) + +Base.copy(a::VersionWeight) = + VersionWeight(a.major, a.minor, a.patch, + copy(a.prerelease), copy(a.build)) + +end diff --git a/src/Pkg2/types.jl b/src/Pkg2/types.jl new file mode 100644 index 0000000000000..1ee9edfbd9358 --- /dev/null +++ b/src/Pkg2/types.jl @@ -0,0 +1,258 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module Types + +export VersionInterval, VersionSet, Requires, Available, Fixed, merge_requires!, satisfies, + ResolveBacktraceItem, ResolveBacktrace +import Base: show, isempty, in, intersect, union!, union, ==, hash, copy, deepcopy_internal, push! + +import ...iswindows + +struct VersionInterval + lower::VersionNumber + upper::VersionNumber +end +VersionInterval(lower::VersionNumber) = VersionInterval(lower,typemax(VersionNumber)) +VersionInterval() = VersionInterval(typemin(VersionNumber)) + +show(io::IO, i::VersionInterval) = print(io, "[$(i.lower),$(i.upper))") +isempty(i::VersionInterval) = i.upper <= i.lower +in(v::VersionNumber, i::VersionInterval) = i.lower <= v < i.upper +intersect(a::VersionInterval, b::VersionInterval) = VersionInterval(max(a.lower,b.lower), min(a.upper,b.upper)) +==(a::VersionInterval, b::VersionInterval) = a.lower == b.lower && a.upper == b.upper +hash(i::VersionInterval, h::UInt) = hash((i.lower, i.upper), h + (0x0f870a92db508386 % UInt)) + +function normalize!(ivals::Vector{VersionInterval}) + # VersionSet internal normalization: + # removes empty intervals and fuses intervals without gaps + # e.g.: + # [0.0.0,1.0.0) ∪ [1.0.0,1.5.0) ∪ [1.6.0,1.6.0) ∪ [2.0.0,∞) + # becomes: + # [0.0.0,1.5.0) ∪ [2.0.0,∞) + # (still assumes that lower bounds are sorted, and intervals do + # not overlap) + l = length(ivals) + l == 0 && return ivals + + lo, up, k0 = ivals[1].lower, ivals[1].upper, 1 + fusing = false + for k = 2:l + lo1, up1 = ivals[k].lower, ivals[k].upper + if lo1 == up + up = up1 + fusing = true + continue + end + if lo < up + # The only purpose of the "fusing" check is to avoid + # extra allocations + ivals[k0] = fusing ? VersionInterval(lo, up) : ivals[k-1] + k0 += 1 + end + fusing = false + lo, up = lo1, up1 + end + if lo < up + ivals[k0] = fusing ? VersionInterval(lo, up) : ivals[l] + k0 += 1 + end + resize!(ivals, k0 - 1) + return ivals +end + +struct VersionSet + intervals::Vector{VersionInterval} + VersionSet(intervals::Vector{VersionInterval}) = new(normalize!(intervals)) + # copy is defined inside the struct block to call `new` directly + # without going through `normalize!` + Base.copy(vset::VersionSet) = new(copy(vset.intervals)) +end +function VersionSet(versions::Vector{VersionNumber}) + intervals = VersionInterval[] + if isempty(versions) + push!(intervals, VersionInterval()) + else + isodd(length(versions)) && push!(versions, typemax(VersionNumber)) + while !isempty(versions) + push!(intervals, VersionInterval(shift!(versions), shift!(versions))) + end + end + VersionSet(intervals) +end +VersionSet(versions::VersionNumber...) = VersionSet(VersionNumber[versions...]) + +const empty_versionset = VersionSet(VersionInterval[]) + +# Windows console doesn't like Unicode +const _empty_symbol = @static iswindows() ? "empty" : "∅" +const _union_symbol = @static iswindows() ? " or " : " ∪ " +show(io::IO, s::VersionSet) = isempty(s) ? print(io, _empty_symbol) : + join(io, s.intervals, _union_symbol) +isempty(s::VersionSet) = all(isempty, s.intervals) +in(v::VersionNumber, s::VersionSet) = any(i->in(v,i), s.intervals) +function intersect(A::VersionSet, B::VersionSet) + (isempty(A) || isempty(B)) && return copy(empty_versionset) + ivals = [intersect(a,b) for a in A.intervals for b in B.intervals] + sort!(ivals, by=i->i.lower) + VersionSet(ivals) +end + +union(A::VersionSet, B::VersionSet) = union!(copy(A), B) +function union!(A::VersionSet, B::VersionSet) + A == B && return A + ivals = A.intervals + for intB in B.intervals + lB, uB = intB.lower, intB.upper + k0 = findfirst(i->(i.upper > lB), ivals) + if k0 == 0 + push!(ivals, intB) + continue + end + lB = min(lB, ivals[k0].lower) + for k1 = k0:length(ivals) + intA = ivals[k1] + if uB < intA.lower + splice!(ivals, k0:(k1-1), (VersionInterval(lB, uB),)) + break + elseif uB ∈ intA || k1 == length(ivals) + splice!(ivals, k0:k1, (VersionInterval(lB, max(uB, intA.upper)),)) + break + end + end + end + normalize!(ivals) + return A +end + +==(A::VersionSet, B::VersionSet) = A.intervals == B.intervals +hash(s::VersionSet, h::UInt) = hash(s.intervals, h + (0x2fd2ca6efa023f44 % UInt)) +deepcopy_internal(vs::VersionSet, ::ObjectIdDict) = copy(vs) + +const Requires = Dict{String,VersionSet} + +function merge_requires!(A::Requires, B::Requires) + for (pkg,vers) in B + A[pkg] = haskey(A,pkg) ? intersect(A[pkg],vers) : vers + end + return A +end + +satisfies(pkg::AbstractString, ver::VersionNumber, reqs::Requires) = + !haskey(reqs, pkg) || in(ver, reqs[pkg]) + +struct Available + sha1::String + requires::Requires +end + +==(a::Available, b::Available) = a.sha1 == b.sha1 && a.requires == b.requires +hash(a::Available, h::UInt) = hash((a.sha1, a.requires), h + (0xbc8ae0de9d11d972 % UInt)) +copy(a::Available) = Available(a.sha1, copy(a.requires)) + +show(io::IO, a::Available) = isempty(a.requires) ? + print(io, "Available(", repr(a.sha1), ")") : + print(io, "Available(", repr(a.sha1), ",", a.requires, ")") + +struct Fixed + version::VersionNumber + requires::Requires +end +Fixed(v::VersionNumber) = Fixed(v,Requires()) + +==(a::Fixed, b::Fixed) = a.version == b.version && a.requires == b.requires +hash(f::Fixed, h::UInt) = hash((f.version, f.requires), h + (0x68628b809fd417ca % UInt)) + +show(io::IO, f::Fixed) = isempty(f.requires) ? + print(io, "Fixed(", repr(f.version), ")") : + print(io, "Fixed(", repr(f.version), ",", f.requires, ")") + +# TODO: Available & Fixed are almost the same – merge them? +# Free could include the same information too, it just isn't +# required by anything that processes these things. + + +const VersionReq = Union{VersionNumber,VersionSet} +const WhyReq = Tuple{VersionReq,Any} + +# This is used to keep track of dependency relations when propagating +# requirements, so as to emit useful information in case of unsatisfiable +# conditions. +# The `versionreq` field keeps track of the remaining allowed versions, +# intersecting all requirements. +# The `why` field is a Vector which keeps track of the requirements. Each +# entry is a Tuple of two elements: +# 1) the first element is the version requirement (can be a single VersionNumber +# or a VersionSet). +# 2) the second element can be either :fixed (for requirements induced by +# fixed packages), :required (for requirements induced by explicitly +# required packages), or a Pair p=>backtrace_item (for requirements induced +# indirectly, where `p` is the package name and `backtrace_item` is +# another ResolveBacktraceItem. +mutable struct ResolveBacktraceItem + versionreq::VersionReq + why::Vector{WhyReq} + ResolveBacktraceItem() = new(VersionSet(), WhyReq[]) + ResolveBacktraceItem(reason, versionreq::VersionReq) = new(versionreq, WhyReq[(versionreq,reason)]) +end + +function push!(ritem::ResolveBacktraceItem, reason, versionset::VersionSet) + if isa(ritem.versionreq, VersionSet) + ritem.versionreq = ritem.versionreq ∩ versionset + elseif ritem.versionreq ∉ versionset + ritem.versionreq = copy(empty_versionset) + end + push!(ritem.why, (versionset,reason)) +end + +function push!(ritem::ResolveBacktraceItem, reason, version::VersionNumber) + if isa(ritem.versionreq, VersionSet) + if version ∈ ritem.versionreq + ritem.versionreq = version + else + ritem.versionreq = copy(empty_versionset) + end + elseif ritem.versionreq ≠ version + ritem.versionreq = copy(empty_versionset) + end + push!(ritem.why, (version,reason)) +end + + +show(io::IO, ritem::ResolveBacktraceItem) = _show(io, ritem, "", Set{ResolveBacktraceItem}([ritem])) + +function _show(io::IO, ritem::ResolveBacktraceItem, indent::String, seen::Set{ResolveBacktraceItem}) + l = length(ritem.why) + for (i,(vs,w)) in enumerate(ritem.why) + print(io, indent, (i==l ? '└' : '├'), '─') + if w ≡ :fixed + @assert isa(vs, VersionNumber) + println(io, "version $vs set by fixed requirement (package is checked out, dirty or pinned)") + elseif w ≡ :required + @assert isa(vs, VersionSet) + println(io, "version range $vs set by an explicit requirement") + else + @assert isa(w, Pair{<:AbstractString,ResolveBacktraceItem}) + if isa(vs, VersionNumber) + print(io, "version $vs ") + else + print(io, "version range $vs ") + end + print(io, "required by package $(w[1]), ") + if isa(w[2].versionreq, VersionSet) + println(io, "whose allowed version range is $(w[2].versionreq):") + else + println(io, "whose only allowed version is $(w[2].versionreq):") + end + if w[2] ∈ seen + println(io, (i==l ? " " : "│ ") * indent, "└─[see above for $(w[1]) backtrace]") + continue + end + push!(seen, w[2]) + _show(io, w[2], (i==l ? " " : "│ ") * indent, seen) + end + end +end + +const ResolveBacktrace = Dict{AbstractString,ResolveBacktraceItem} + +end # module diff --git a/src/Pkg3.jl b/src/Pkg3.jl index f40af17d3fd14..1ee6e33bafd57 100644 --- a/src/Pkg3.jl +++ b/src/Pkg3.jl @@ -12,6 +12,9 @@ include("../ext/BinaryProvider/src/BinaryProvider.jl") include("../ext/TOML/src/TOML.jl") include("../ext/TerminalMenus/src/TerminalMenus.jl") +iswindows() = @static VERSION < v"0.7-" ? Sys.is_windows() : Sys.iswindows() + +include("Pkg2/Pkg2.jl") include("Types.jl") include("Display.jl") include("Operations.jl") diff --git a/src/Types.jl b/src/Types.jl index 45fe9ba71d6fc..850003ec81de5 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -1,12 +1,12 @@ module Types using Base.Random: UUID -using Base.Pkg.Types: VersionSet, Available +using Pkg3.Pkg2.Types: VersionSet, Available using Pkg3.TOML using Pkg3.TerminalMenus import Pkg3 -import Pkg3: depots +import Pkg3: depots, iswindows export SHA1, VersionRange, VersionSpec, PackageSpec, UpgradeLevel, EnvCache, CommandError, cmderror, has_name, has_uuid, write_env, parse_toml, find_registered!, @@ -330,7 +330,7 @@ function git_discover( ceiling::Union{AbstractString,Vector} = "", across_fs::Bool = false, ) - sep = @static is_windows() ? ";" : ":" + sep = @static iswindows() ? ";" : ":" ceil = ceiling isa AbstractString ? ceiling : join(convert(Vector{String}, ceiling), sep) buf_ref = Ref(LibGit2.Buffer()) @@ -735,8 +735,6 @@ function manifest_info(env::EnvCache, uuid::UUID)::Union{Dict{String,Any},Void} return nothing end -iswindows() = @static VERSION < v"0.7-" ? Sys.is_windows() : Sys.iswindows() - "Give a short path string representation" function pathrepr(env::EnvCache, path::String, base::String=pwd()) path = abspath(base, path)