diff --git a/Project.toml b/Project.toml
index 77d2d50..4e4d22a 100644
--- a/Project.toml
+++ b/Project.toml
@@ -4,12 +4,13 @@ version = "0.3.0"
[deps]
ArnoldiMethod = "ec485272-7323-5ecc-a04f-4719b315124d"
+CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
+ConstructionBase = "187b0558-2788-49d3-abe0-74a17ed4e7c9"
DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
-Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
ProgressLogging = "33c8b6b6-d38a-422a-b730-caa89a2f386c"
Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689"
SimpleWeightedGraphs = "47aef6b3-ad0c-573a-a1e2-d07658019622"
@@ -18,18 +19,21 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
[compat]
ArnoldiMethod = "0.0.4, 0.4"
+CommonSolve = "0.2"
+ConstructionBase = "1.5.8"
DelimitedFiles = "1"
Graphs = "1"
LaTeXStrings = "1.1"
LinearSolve = "2.38.0"
Plots = "1.4"
ProgressLogging = "0.1"
-Rasters = "0.13"
+Rasters = "0.14"
SimpleWeightedGraphs = "1.1"
julia = "1.10"
[extras]
ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
[targets]
-test = ["ArchGDAL"]
+test = ["ArchGDAL","Plots"]
diff --git a/examples/2_landmarks.jmd b/examples/2_landmarks.jmd
index f320622..8a4146c 100644
--- a/examples/2_landmarks.jmd
+++ b/examples/2_landmarks.jmd
@@ -152,7 +152,7 @@ plot(result.func_exp)
```julia
-stored_problem = ConScape.StoredProblem(problem;
+stored_problem = ConScape.BatchProblem(problem;
path=".", radius=20, overlap=30, threaded=true
)
ConScape.solve(stored_problem, rast)
diff --git a/src/ConScape.jl b/src/ConScape.jl
index 4c7ba14..f1957af 100644
--- a/src/ConScape.jl
+++ b/src/ConScape.jl
@@ -1,30 +1,46 @@
module ConScape
-using SparseArrays, LinearAlgebra
-using Graphs, Plots, SimpleWeightedGraphs, ProgressLogging, ArnoldiMethod
-using Rasters
+using ArnoldiMethod
+using ConstructionBase
+using Graphs
+using LinearAlgebra
using LinearSolve
+using ProgressLogging
+using Rasters
+using SimpleWeightedGraphs
+using SparseArrays
using Rasters.DimensionalData
+import CommonSolve
+import CommonSolve: solve, init
+
# Old funcion-based interface
abstract type ConnectivityFunction <: Function end
abstract type DistanceFunction <: ConnectivityFunction end
abstract type ProximityFunction <: ConnectivityFunction end
-struct least_cost_distance <: DistanceFunction end
-struct expected_cost <: DistanceFunction end
-struct free_energy_distance <: DistanceFunction end
+struct least_cost_distance <: DistanceFunction end
+struct expected_cost <: DistanceFunction end
+struct free_energy_distance <: DistanceFunction end
-struct survival_probability <: ProximityFunction end
-struct power_mean_proximity <: ProximityFunction end
+struct survival_probability <: ProximityFunction end
+struct power_mean_proximity <: ProximityFunction end
# Need to define before loading files
+
+"""
+ Solver
+
+Abstract supertype for ConScape solvers.
+"""
abstract type AbstractProblem end
abstract type Solver end
# Randomized shortest path algorithms
include("randomizedshortestpath.jl")
# Grid struct and methods
+include("transformations.jl")
+# Grid struct and methods
include("grid.jl")
# GridRSP (randomized shortest path) struct and methods
include("gridrsp.jl")
@@ -32,10 +48,12 @@ include("gridrsp.jl")
include("io.jl")
# Utilities
include("utils.jl")
+# Problems
include("graph_measure.jl")
include("connectivity_measure.jl")
include("problem.jl")
include("solvers.jl")
-include("tiles.jl")
+include("windows.jl")
+include("assessment.jl")
end
diff --git a/src/assessment.jl b/src/assessment.jl
new file mode 100644
index 0000000..fef5a6c
--- /dev/null
+++ b/src/assessment.jl
@@ -0,0 +1,250 @@
+
+struct AssessmentWarnings
+ source_qualities_nan_found::Bool
+ target_qualities_nan_found::Bool
+end
+
+function Base.:(|)(aw1::AssessmentWarnings, aw2::AssessmentWarnings)
+ AssessmentWarnings(
+ aw1.source_qualities_nan_found | aw2.source_qualities_nan_found,
+ aw1.target_qualities_nan_found | aw2.target_qualities_nan_found,
+ )
+end
+function Base.:(&)(aw1::AssessmentWarnings, aw2::AssessmentWarnings)
+ AssessmentWarnings(
+ aw1.source_qualities_nan_found & aw2.source_qualities_nan_found,
+ aw1.target_qualities_nan_found & aw2.target_qualities_nan_found,
+ )
+end
+Base.any(aw::AssessmentWarnings) = aw.source_qualities_nan_found | aw.target_qualities_nan_found
+Base.all(aw::AssessmentWarnings) = aw.source_qualities_nan_found & aw.target_qualities_nan_found
+
+"""
+ ProblemAssessment
+
+Abstract supertype for problem assessments.
+
+These calculate the computation size of an
+`AbstractWindowedProblem` for a specific `RasterStack`.
+"""
+abstract type ProblemAssessment end
+
+Base.size(a::ProblemAssessment) = a.size
+
+"""
+ WindowAssessment <: ProblemAssessment
+
+Assessment of an AbstractWindowedProblem that holds
+a `Problem`.
+
+# Fields
+- `size::Tuple{Int,Int}`: the size of the input and output RasterStack
+- `shape::Tuple{Int,Int}`: the shape of the windowing
+- `njobs::Int`: the number of problem runs required to finish the problem
+- `grid_sizes::Vector{Tuple{Int,Int}}`: the sizes of each window
+- `mask::Vector{Bool}`: Vector{Bool} where `true` values are jobs that need to be run.
+- `indices::Vector{Int}`: the indices of `mask` that are `true`.
+"""
+@kwdef struct WindowAssessment <: ProblemAssessment
+ size::Tuple{Int,Int}
+ shape::Tuple{Int,Int}
+ njobs::Int
+ mask::Vector{Bool}
+ indices::Vector{Int}
+ warnings::AssessmentWarnings
+ grid_sizes::Vector{Tuple{Int,Int}}
+end
+
+"""
+ NestedAssessment <: ProblemAssessment
+
+Assessment of a nested `AbstractWindowedProblem`,
+that holds another `AbstractWindowedProblem`.
+
+# Fields
+- `size::Tuple{Int,Int}`: the size of the `RasterStack` input and output.
+- `shape::Tuple{Int,Int}`: the shape of the windowing
+- `njobs::Int`: the number of problem runs required to finish the problem
+- `mask::Vector{Bool}`: Vector{Bool} where `true` values are jobs that need to be run.
+- `indices::Vector{Int}`: the indices of `mask` that are `true`.
+- `assessments::Vector{WindowAssessment}`: asessments at the next level down.
+"""
+@kwdef struct NestedAssessment <: ProblemAssessment
+ size::Tuple{Int,Int}
+ shape::Tuple{Int,Int}
+ njobs::Int
+ mask::Vector{Bool}
+ indices::Vector{Int}
+ warnings::AssessmentWarnings
+ assessments::Vector{WindowAssessment}
+end
+
+function Base.show(io::IO, mime::MIME"text/plain", a::ProblemAssessment)
+ println(io, "NestedAssessment")
+ println(io)
+ println(io, "Shape: $(a.shape)")
+ println(io, "Number of jobs: $(a.njobs)")
+ # Use SparseArrays nice matrix printing for the mask
+ println(io, "Job mask: ")
+ mask = sparse(reshape(a.mask, a.shape))
+ Base.print_array(io, mask)
+ if any(a.warnings)
+ show(io, mime, a.warnings)
+ end
+end
+
+
+"""
+ assess(p::AbstractProblem, rast::RasterStack)
+
+Assess the computational requirements of problem
+`p` for `RasterStack` `rastr`.
+
+This can be used to indicate memory and time reequiremtents on a cluster.
+"""
+function assess end
+
+function assess(p::AbstractWindowedProblem{<:Problem}, rast::AbstractRasterStack;
+ inner_target_bools=nothing,
+ target_ranges=_target_ranges(p, rast),
+ kw...
+)
+ # Define the ranges of each window
+ window_ranges = _window_ranges(p, rast)
+
+ # Convert everything to Bool at the batch level so window assessments are fast
+ inner_targets = view(rast.target_qualities, target_ranges...)
+ warnings = AssessmentWarnings(
+ any(isnan, rast.qualities),
+ any(isnan, inner_targets),
+ )
+ inner_target_bools = isnothing(inner_target_bools) ? _isvalid.(inner_targets) : inner_target_bools
+ qualities = _isvalid.(rast.qualities)
+ target_qualities = falses(size(rast))
+ target_qualities[target_ranges...] .= inner_target_bools
+ bool_rast = RasterStack((; qualities, target_qualities), dims(rast))
+
+ # Calculate window sizes and allocations
+ grid_sizes = vec(_estimate_grid_sizes(p, bool_rast; window_ranges))
+
+ # Organise stats for each window into vectors
+ # Windows must have more than one source and more than one target
+ window_mask = map(s -> s[1] > 1 && s[2] > 1, grid_sizes)
+ non_empty_indices = eachindex(window_mask)[window_mask]
+
+ # Calculate global stats
+ njobs = count(window_mask)
+ shape = size(window_ranges)
+
+ WindowAssessment(size(rast), shape, njobs, window_mask, non_empty_indices, warnings, grid_sizes)
+end
+function assess(
+ p::AbstractWindowedProblem{<:AbstractWindowedProblem},
+ rast::AbstractRasterStack;
+ nthreads=Threads.nthreads(),
+ verbose=true,
+ kw...
+)
+ # Calculate outer window ranges
+ window_ranges = _window_ranges(p, rast)
+ verbose && println("Assessing $(length(window_ranges)) jobs")
+
+ # Define a vector for all assessment data
+ assessments = Vector{WindowAssessment}(undef, length(window_ranges))
+ # Run assessments threaded as they can take a long time for large rasters
+ Threads.@threads for i in eachindex(vec(window_ranges))
+ rs = window_ranges[i]
+ verbose && println("Assessing batch: $i, $rs")
+ function empty_assesment(size)
+ verbose && println(" No targets found")
+ WindowAssessment(;
+ size,
+ shape=(0, 0),
+ njobs=0,
+ mask=Bool[],
+ indices=Int[],
+ warnings=AssessmentWarnings(false, false),
+ grid_sizes=Tuple{Int,Int}[],
+ )
+ end
+ # We only need qualities for the assessment
+ window_rast = rast[(:qualities, :target_qualities)][rs...]
+ target_ranges = _target_ranges(p, window_rast)
+ # Convert targets to bool as early as possible
+ inner_targets = view(window_rast.target_qualities, target_ranges...)
+ inner_target_bools = _isvalid.(inner_targets)
+ assessments[i] = if count(inner_target_bools) > 0
+ assess(p.problem, window_rast; inner_target_bools, target_ranges, nthreads, kw...)
+ else
+ empty_assesment(size(window_rast))
+ end
+ end
+ # Get mask and indices
+ mask = map(a -> any(a.mask), assessments)
+ non_empty_indices = eachindex(vec(mask))[mask]
+ # Calculate global stats
+ njobs = count(mask)
+ shape = size(window_ranges)
+ warnings = reduce(|, (a.warnings for a in assessments))
+ return NestedAssessment(size(rast), shape, njobs, mask, non_empty_indices, warnings, assessments)
+end
+
+"""
+ reassess(p::BatchProblem, a::NestedAssessment)
+
+Re-asses an existing nested assesment of a [`BatchProblem`](@ref).
+
+The returned `NestedAssessment` will exclude any batches that
+already have a data folder (assumed to be successfully completed).
+"""
+function reassess(p::BatchProblem, a::NestedAssessment)
+ patch = _reassess(p, a)
+ a1 = ConstructionBase.setproperties(a, patch)
+ # Update nan_target_found from remaining indices
+ warnings = reduce(|, (a1.assessments[i].warnings for i in a1.indices);
+ init=AssessmentWarnings(false, false)
+ )
+ return ConstructionBase.setproperties(a1, (; warnings))
+end
+function reassess(p::BatchProblem, a::WindowAssessment)
+ patch = _reassess(p, a)
+ return ConstructionBase.setproperties(a, patch)
+end
+
+function _reassess(p, a)
+ # Paths for all batches
+ paths = batch_paths(p, size(a))
+ # Paths for non-empty batches
+ jobpaths = paths[a.indices]
+ # Find all the jobs that havent been saved (failed)
+ idxmask = .!(isdir.(jobpaths))
+ # Generate new arrays of indices and assessments for the remaining jobs
+ indices = a.indices[idxmask]
+ mask = fill(false, prod(a.shape))
+ mask[indices] .= true
+ njobs = length(indices)
+ return (; njobs, mask, indices)
+end
+
+# Accept ProblemAssessment as an argument to solve and init
+# To used instead of keywords
+solve(p::BatchProblem, rast::RasterStack, a::ProblemAssessment, i::Int...; verbose=false) =
+ solve!(init(p, rast, a), p, i...; verbose)
+
+function init(p::BatchProblem{<:WindowedProblem}, rast::RasterStack, a::NestedAssessment, i::Int...; kw...)
+ batch_ranges = _window_ranges(p, rast)
+ grid_sizes = map(a.assessments) do a_w
+ a_w.grid_sizes
+ end
+ selected_window_indices = map(a.assessments) do a_w
+ a_w.indices
+ end
+ init(p, rast, i...;
+ batch_ranges,
+ batch_indices=a.indices,
+ grid_sizes,
+ selected_window_indices,
+ )
+end
+init(p::BatchProblem{<:Problem}, rast::RasterStack, a::WindowAssessment, i::Int...; kw...) =
+ init(p, rast, i...; batch_indices=a.indices)
\ No newline at end of file
diff --git a/src/connectivity_measure.jl b/src/connectivity_measure.jl
index 3ecf38f..e11646c 100644
--- a/src/connectivity_measure.jl
+++ b/src/connectivity_measure.jl
@@ -1,34 +1,50 @@
-# New type-based interface
-# Easier to add parameters to these
+"""
+ GraphMeasure
+
+Abstract supertype for connectivity measures.
+
+These are lazy definitions of conscape functions,
+with required parameters attached rather than passed
+in through keywords.
+"""
abstract type ConnectivityMeasure end
abstract type FundamentalMeasure <: ConnectivityMeasure end
abstract type DistanceMeasure <: FundamentalMeasure end
struct LeastCostDistance <: ConnectivityMeasure end
-@kwdef struct ExpectedCost{T<:Union{Real,Nothing},CM} <: DistanceMeasure
- θ::T=nothing
- distance_transformation::CM=nothing
- approx::Bool=false
+@kwdef struct ExpectedCost{T<:Union{Real,Nothing},CM,DV} <: DistanceMeasure
+ θ::T
+ distance_transformation::CM
+ diagvalue::DV = nothing
+ approx::Bool = false
end
-@kwdef struct FreeEnergyDistance{T<:Union{Real,Nothing},CM} <: DistanceMeasure
- θ::T=nothing
- distance_transformation::CM=nothing
- approx::Bool=false
+@kwdef struct FreeEnergyDistance{T<:Union{Real,Nothing},CM,DV} <: DistanceMeasure
+ θ::T
+ distance_transformation::CM
+ diagvalue::DV = nothing
+ approx::Bool = false
end
-@kwdef struct SurvivalProbability{T<:Union{Real,Nothing}} <: FundamentalMeasure
- θ::T=nothing
- approx::Bool=false
+@kwdef struct SurvivalProbability{T<:Union{Real,Nothing},DV} <: FundamentalMeasure
+ θ::T = nothing
+ diagvalue::DV = nothing # TODO should be 1
+ approx::Bool = false
end
-@kwdef struct PowerMeanProximity{T<:Union{Real,Nothing}} <: FundamentalMeasure
- θ::T=nothing
- approx::Bool=false
+@kwdef struct PowerMeanProximity{T<:Union{Real,Nothing},DV} <: FundamentalMeasure
+ θ::T = nothing
+ diagvalue::DV = nothing
+ approx::Bool = false
end
keywords(cm::ConnectivityMeasure) = _keywords(cm)
+distance_transformation(cm::FundamentalMeasure) = nothing
+distance_transformation(cm::DistanceMeasure) = cm.distance_transformation
+
# TODO remove the complexity of the connectivity_function
# These methods are mostly to avoid changing the original interface for now
+# Its a quirk of how MeanKullbackLeiblerDivergence is implemented
+# that these can be calculated separately from the main grid
connectivity_function(::LeastCostDistance) = least_cost_distance
connectivity_function(::ExpectedCost) = expected_cost
connectivity_function(::FreeEnergyDistance) = free_energy_distance
@@ -36,5 +52,5 @@ connectivity_function(::SurvivalProbability) = survival_probability
connectivity_function(::PowerMeanProximity) = power_mean_proximity
# This is not used yet but could be
-compute(cm::ConnectivityMeasure, g; kw...) =
- connectivity_function(m)(g; keywords(cm)..., kw...)
\ No newline at end of file
+compute(cm::ConnectivityMeasure, g; kw...) =
+ connectivity_function(m)(g; keywords(cm)..., kw...)
diff --git a/src/graph_measure.jl b/src/graph_measure.jl
index 329338b..d5d033b 100644
--- a/src/graph_measure.jl
+++ b/src/graph_measure.jl
@@ -6,102 +6,125 @@ These are lazy definitions of conscape functions.
"""
abstract type GraphMeasure end
-keywords(o::GraphMeasure) = _keywords(o)
-
+abstract type SpatialMeasure <: GraphMeasure end
abstract type TopologicalMeasure <: GraphMeasure end
-abstract type BetweennessMeasure <: GraphMeasure end
-abstract type PerturbationMeasure <: GraphMeasure end
+abstract type BetweennessMeasure <: SpatialMeasure end
+abstract type PerturbationMeasure <: SpatialMeasure end
abstract type PathDistributionMeasure <: GraphMeasure end
+# Concrete GraphMeasure structs
+
struct BetweennessQweighted <: BetweennessMeasure end
-@kwdef struct BetweennessKweighted{DV} <: BetweennessMeasure
- diagvalue::DV=nothing
-end
-struct EdgeBetweennessQweighted <: BetweennessMeasure end
-@kwdef struct EdgeBetweennessKweighted{DV} <: BetweennessMeasure
- diagvalue::DV=nothing
-end
+struct BetweennessKweighted <: BetweennessMeasure end
-@kwdef struct ConnectedHabitat{DV} <: GraphMeasure
- diagvalue::DV=nothing
-end
+abstract type EdgeBetweennessMeasure <: BetweennessMeasure end
+struct EdgeBetweennessQweighted <: EdgeBetweennessMeasure end
+struct EdgeBetweennessKweighted <: EdgeBetweennessMeasure end
-@kwdef struct Criticality{DV,AV,QT,QS} <: PerturbationMeasure
- diagvalue::DV=nothing
- avalue::AV=floatmin()
- qˢvalue::QS=0.0
- qᵗvalue::QT=0.0
+struct ConnectedHabitat <: SpatialMeasure end
+
+@kwdef struct Criticality{AV,QT,QS} <: PerturbationMeasure
+ avalue::AV = floatmin()
+ qˢvalue::QS = 0.0
+ qᵗvalue::QT = 0.0
end
-# These maybe don't quite belong here?
-@kwdef struct EigMax{DV,T} <: TopologicalMeasure
- diagvalue::DV=nothing
- tol::T=1e-14
+@kwdef struct EigMax{T} <: TopologicalMeasure
+ tol::T = 1e-14
end
struct MeanLeastCostKullbackLeiblerDivergence <: PathDistributionMeasure end
struct MeanKullbackLeiblerDivergence <: PathDistributionMeasure end
-# Map structs to functions
+# Map structs to function calls
-# These return Rasters
graph_function(m::BetweennessKweighted) = betweenness_kweighted
graph_function(m::BetweennessQweighted) = betweenness_qweighted
graph_function(m::ConnectedHabitat) = connected_habitat
graph_function(m::Criticality) = criticality
-# These return scalars
graph_function(m::MeanLeastCostKullbackLeiblerDivergence) = mean_lc_kl_divergence
graph_function(m::MeanKullbackLeiblerDivergence) = mean_kl_divergence
-# These return sparse arrays
graph_function(m::EdgeBetweennessKweighted) = edge_betweenness_kweighted
graph_function(m::EdgeBetweennessQweighted) = edge_betweenness_qweighted
-# Returns a tuple
graph_function(m::EigMax) = eigmax
-# Map structs to function keywords,
-# a bit of a hack until we refactor the rest
-keywords(gm::GraphMeasure, p::AbstractProblem) =
- (; _keywords(gm)...)#, solver=solver(p))
-keywords(gm::ConnectedHabitat, p::AbstractProblem) =
- (; _keywords(gm)..., approx=connectivity_measure(p).approx)#, solver=solver(p))
+# Function keywords
-# A trait for connectivity requirement
-struct NeedsConnectivity end
-struct NoConnectivity end
-needs_connectivity(::GraphMeasure) = NoConnectivity()
-needs_connectivity(::BetweennessKweighted) = NeedsConnectivity()
-needs_connectivity(::EdgeBetweennessKweighted) = NeedsConnectivity()
-needs_connectivity(::EigMax) = NeedsConnectivity()
-needs_connectivity(::ConnectedHabitat) = NeedsConnectivity()
-needs_connectivity(::Criticality) = NeedsConnectivity()
-
-# compute
-# This is where things actually happen
-#
-# Add dispatch on connectivity measure
-compute(gm::GraphMeasure, p::AbstractProblem, g::Union{Grid,GridRSP}) =
- compute(needs_connectivity(gm), gm, p, g)
-function compute(::NeedsConnectivity,
- gm::GraphMeasure,
- p::AbstractProblem,
- g::Union{Grid,GridRSP}
-)
- cm = p.connectivity_measure
- distance_transformation = cm.distance_transformation
- connectivity_function = ConScape.connectivity_function(cm)
- # Handle multiple distance transformations
- if distance_transformation isa NamedTuple
- map(distance_transformation) do dt
- graph_function(gm)(g; keywords(gm, p)..., distance_transformation=dt, connectivity_function)
- end
+keywords(gm::GraphMeasure, p::AbstractProblem) =
+ (; _keywords(gm)..., solver=solver(p), _connectivity_keywords(gm, p)...)
+keywords(gm::ConnectedHabitat, p::AbstractProblem) =
+ (; _keywords(gm)..., approx=connectivity_measure(p).approx, solver=solver(p), _connectivity_keywords(gm, p)...)
+function _connectivity_keywords(gm::GraphMeasure, p::AbstractProblem)
+ cm = connectivity_measure(p)
+ if needs_connectivity(gm)
+ (;
+ _keywords(gm)...,
+ distance_transformation=distance_transformation(cm),
+ connectivity_function=connectivity_function(cm)
+ )
else
- graph_function(gm)(g; keywords(gm, p)..., distance_transformation=dt, connectivity_function)
+ _keywords(gm)
+ end
+end
+
+# Traits
+
+abstract type ReturnType end
+struct ReturnsDenseSpatial <: ReturnType end
+struct ReturnsSparse <: ReturnType end
+struct ReturnsScalar <: ReturnType end
+struct ReturnsOther{F} <: ReturnType
+ f::F
+end
+
+# These allow calculation of return allocations
+returntype(::SpatialMeasure) = ReturnsDenseSpatial()
+returntype(::EdgeBetweennessMeasure) = ReturnsSparse()
+returntype(::PathDistributionMeasure) = ReturnsScalar()
+returntype(::EigMax) = ReturnsOther((n, m) -> n + m)
+
+# A trait for connectivity requirement
+needs_connectivity(::GraphMeasure) = false
+needs_connectivity(::BetweennessKweighted) = true
+needs_connectivity(::EdgeBetweennessKweighted) = true
+needs_connectivity(::EigMax) = true
+needs_connectivity(::ConnectedHabitat) = true
+needs_connectivity(::Criticality) = true
+
+# Workspace allocation traits
+needs_inv(::GraphMeasure) = false
+needs_inv(::BetweennessMeasure) = true
+needs_Z(::GraphMeasure) = true
+needs_workspaces(::GraphMeasure) = 0
+needs_workspaces(::BetweennessMeasure) = 1
+needs_workspaces(::EdgeBetweennessKweighted) = 2
+needs_workspaces(::EdgeBetweennessQweighted) = 3
+needs_permuted_workspaces(::GraphMeasure) = 0
+needs_permuted_workspaces(::EdgeBetweennessKweighted) = 1
+needs_proximity(::GraphMeasure) = false
+needs_proximity(::Union{BetweennessKweighted,EdgeBetweennessKweighted}) = true
+needs_expected_cost(::GraphMeasure) = false
+needs_expected_cost(::EdgeBetweennessKweighted) = true
+needs_expected_cost(::MeanKullbackLeiblerDivergence) = true
+needs_free_energy_distance(::GraphMeasure) = false
+needs_free_energy_distance(::MeanKullbackLeiblerDivergence) = true
+needs_adjoint_init(::GraphMeasure) = true # TODO which dont?
+
+# Trait helpers
+
+function count_workspaces(p::AbstractProblem)
+ gms = graph_measures(p)
+ n = mapreduce(needs_workspaces, max, gms)
+ if hastrait(needs_expected_cost, gms) || connectivity_function(p) == ConScape.expected_cost
+ max(n, 2)
end
end
-function compute(::NoConnectivity,
- gm::GraphMeasure,
- p::AbstractProblem,
- g::Union{Grid,GridRSP}
-)
- graph_function(gm)(g; keywords(gm, p)...)
-end
\ No newline at end of file
+count_permuted_workspaces(p::AbstractProblem) =
+ mapreduce(needs_permuted_workspaces, max, graph_measures(p))
+
+# Trait aggregator
+hastrait(t, gms) = reduce(|, map(t, gms); init=false)
+
+# compute: run a graph function with the appropriate keywords
+compute(gm::GraphMeasure, p::AbstractProblem, g::Union{Grid,GridRSP}; kw...) =
+ graph_function(gm)(g; keywords(gm, p)..., kw...)
\ No newline at end of file
diff --git a/src/grid.jl b/src/grid.jl
index d4a7105..85161ff 100644
--- a/src/grid.jl
+++ b/src/grid.jl
@@ -1,75 +1,56 @@
-abstract type Transformation end
-struct MinusLog <: Transformation end
-struct ExpMinus <: Transformation end
-struct Inv <: Transformation end
-struct OddsAgainst <: Transformation end
-struct OddsFor <: Transformation end
-
-(::MinusLog)(x::Number) = -log(x)
-(::ExpMinus)(x::Number) = exp(-x)
-(::Inv)(x::Number) = inv(x)
-(::OddsAgainst)(x::Number) = inv(x) - 1
-(::OddsFor)(x::Number) = x/(1 - x)
-
-Base.inv(::MinusLog) = ExpMinus()
-Base.inv(::ExpMinus) = MinusLog()
-Base.inv(::Inv) = Inv()
-Base.inv(::OddsAgainst) = OddsFor()
-Base.inv(::OddsFor) = OddsAgainst()
-
-struct Grid
+"""
+ Grid(nrows::Integer,
+ ncols::Integer;
+ affinities=nothing,
+ qualities::Matrix=ones(nrows, ncols),
+ source_qualities::Matrix=qualities,
+ target_qualities::AbstractMatrix=qualities,
+ costs::Union{Transformation,SparseMatrixCSC{Float64,Int}}=MinusLog(),
+ prune=true)::Grid
+
+Construct a `Grid` from an `affinities` matrix of type `SparseMatrixCSC`.
+
+It is possible to also supply matrices of `source_qualities` and `target_qualities` as well as
+a `costs` function that maps the `affinities` matrix to a `costs` matrix.
+
+Alternatively, it is possible to supply a matrix to `costs` directly. If `prune=true` (the default),
+the affinity and cost matrices will be pruned to exclude unreachable nodes.
+"""
+struct Grid{D<:Union{Tuple,Nothing},SQ,TQ}
nrows::Int
ncols::Int
affinities::SparseMatrixCSC{Float64,Int}
costfunction::Union{Nothing,Transformation}
costmatrix::SparseMatrixCSC{Float64,Int}
id_to_grid_coordinate_list::Vector{CartesianIndex{2}}
- source_qualities::AbstractMatrix{Float64}
- target_qualities::AbstractMatrix{Float64}
+ source_qualities::SQ
+ target_qualities::TQ
targetidx::Vector{CartesianIndex{2}}
targetnodes::Vector{Int}
qs::Vector{Float64}
qt::Vector{Float64}
- dims::Union{Tuple,Nothing}
+ dims::D
end
-
-"""
- Grid(nrows::Integer,
- ncols::Integer;
- affinities=nothing,
- qualities::Matrix=ones(nrows, ncols),
- source_qualities::Matrix=qualities,
- target_qualities::AbstractMatrix=qualities,
- costs::Union{Transformation,SparseMatrixCSC{Float64,Int}}=MinusLog(),
- prune=true)::Grid
-
-Construct a `Grid` from an `affinities` matrix of type `SparseMatrixCSC`. It is possible
-to also supply matrices of `source_qualities` and `target_qualities` as well as
-a `costs` function that maps the `affinities` matrix to a `costs` matrix. Alternatively,
-it is possible to supply a matrix to `costs` directly. If `prune=true` (the default), the
-affinity and cost matrices will be pruned to exclude unreachable nodes.
-"""
-function Grid(nrows::Integer,
- ncols::Integer;
- affinities=nothing,
- qualities::AbstractMatrix=ones(nrows, ncols),
- source_qualities::AbstractMatrix=qualities,
- target_qualities::AbstractMatrix=qualities,
- costs::Union{Transformation,SparseMatrixCSC{Float64,Int}}=MinusLog(),
- prune=true)
-
- if affinities === nothing
- throw(ArgumentError("matrix of affinities must be supplied"))
- end
-
- if nrows*ncols != LinearAlgebra.checksquare(affinities)
+function Grid(
+ nrows::Integer,
+ ncols::Integer;
+ affinities,
+ qualities::AbstractMatrix=ones(nrows, ncols),
+ source_qualities::AbstractMatrix=qualities,
+ target_qualities::AbstractMatrix=qualities,
+ costs::Union{Transformation,SparseMatrixCSC{Float64,Int}}=MinusLog(),
+ prune=true,
+ check=false,
+)
+ if nrows * ncols != LinearAlgebra.checksquare(affinities)
n = size(affinities, 1)
throw(ArgumentError("grid size ($nrows, $ncols) is incompatible with size of affinity matrix ($n, $n)"))
end
- _source_qualities = convert(Matrix{Float64} , _unwrap(source_qualities))
- _target_qualities = convert(AbstractMatrix{Float64}, _unwrap(target_qualities))
+ _source_qualities = _prepare_qualities(source_qualities)
+ _target_qualities = _prepare_qualities(target_qualities)
+ # TODO use or remove this
# Prune
# id_to_grid_coordinate_list = if prune
# nonzerocells = findall(!iszero, vec(sum(affinities, dims=1)))
@@ -91,12 +72,17 @@ function Grid(nrows::Integer,
nothing, costs
end
- if any(t -> t < 0, nonzeros(costmatrix))
- throw(ArgumentError("The cost graph can have only non-negative edge weights. Perhaps you should change the cost function?"))
- end
+ # This is too expensive to calculate for small target grids
+ if check
+ if any(t -> t < 0, nonzeros(costmatrix))
+ throw(ArgumentError("The cost graph can have only non-negative edge weights. Perhaps you should change the cost function?"))
+ end
+ cost_digraph = SimpleDiGraph(costmatrix)
+ affinity_digraph = SimpleDiGraph(affinities)
- if ne(difference(SimpleDiGraph(costmatrix), SimpleDiGraph(affinities))) > 0
- throw(ArgumentError("cost graph contains edges not present in the affinity graph"))
+ if ne(difference(cost_digraph, affinity_digraph)) > 0
+ throw(ArgumentError("cost graph contains edges not present in the affinity graph"))
+ end
end
targetidx, targetnodes = _targetidx_and_nodes(target_qualities, id_to_grid_coordinate_list)
@@ -119,74 +105,65 @@ function Grid(nrows::Integer,
dims(source_qualities),
)
- if prune
- return largest_subgraph(g)
- else
- return g
- end
+ return prune ? largest_subgraph(g) : g
end
-function Grid(rast::RasterStack;
- qualities=get(rast, :qualities) do
- ones(nrows, ncols)
- end,
- affinities=let
- affinities_raster = get(rast, :affinities, nothing)
- ConScape.graph_matrix_from_raster(affinities_raster)
+function Grid(rast::RasterStack;
+ qualities=get(rast, :qualities) do
+ ones(size(rast))
end,
+ affinities=ConScape.graph_matrix_from_raster(rast.affinities),
source_qualities=get(rast, :source_qualities, qualities),
- target_qualities=get(rast, :target_qualities, qualities),
- costs=get(rast, :costs, MinusLog()),
+ target_qualities=get(rast, :target_qualities, qualities),
kw...
)
- Grid(size(rast)...; affinities, qualities, source_qualities, target_qualities, costs, kw...)
+ Grid(size(rast)...; affinities, qualities, source_qualities, target_qualities, kw...)
end
-# TODO move functions like MinusLog to problems and pass in here
-Grid(p::AbstractProblem, rast::RasterStack) = Grid(rast)
+Grid(p::AbstractProblem, rast::RasterStack; kw...) =
+ Grid(rast; costs=costs(p), prune=prune(p), kw...)
-Base.size(g::Grid) = (g.nrows, g.ncols)
-DimensionalData.dims(g::Grid) = g.dims
+# TODO: better name?
+target_size(g::Grid) = size(g.costmatrix, 1), length(g.targetnodes)
+Base.size(g::Grid) = (g.nrows, g.ncols)
function Base.show(io::IO, ::MIME"text/plain", g::Grid)
print(io, summary(g), " of size ", g.nrows, "x", g.ncols)
end
-function Base.show(io::IO, ::MIME"text/html", g::Grid)
- t = string(summary(g), " of size ", g.nrows, "x", g.ncols)
- write(io, "
$t
")
- write(io, "| Affinities")
- show(io, MIME"text/html"(), plot_outdegrees(g))
- write(io, " |
")
- if g.source_qualities === g.target_qualities
- write(io, "")
- show(io, MIME"text/html"(), heatmap(g.source_qualities, yflip=true))
- else
- write(io, "| Source qualities")
- show(io, MIME"text/html"(), heatmap(g.source_qualities, yflip=true))
- write(io, " | Target qualities")
- show(io, MIME"text/html"(), heatmap(Matrix(g.target_qualities), yflip=true))
- write(io, " |
")
- end
-end
+
+DimensionalData.dims(g::Grid) = g.dims
_id_gc_list(nrows, ncols) = vec(collect(CartesianIndices((nrows, ncols))))
-_unwrap(R::Raster) = parent(R)
-_unwrap(R::AbstractMatrix) = R
+
+_prepare_qualities(A::AbstractMatrix) = _no_nan_f64.(_unwrap_raster(A))
+
+_no_nan_f64(x) = isnan(x) ? 0.0 : Float64(x)
+
+_unwrap_raster(R::Raster) = parent(R)
+_unwrap_raster(R::AbstractMatrix) = R
+
# Compute a vector of the cartesian indices of nonzero target qualities and
# the corresponding node id corresponding to the indices
-_targetidx(q::AbstractMatrix, grididxs::Vector) = grididxs
-_targetidx(q::SparseMatrixCSC, grididxs::Vector) =
+_targetidx(q::AbstractMatrix, grididxs::AbstractVector) = grididxs
+_targetidx(q::Raster, grididxs::AbstractVector) = _targetidx(parent(q), grididxs)
+_targetidx(q::SparseMatrixCSC, grididxs::AbstractVector) =
CartesianIndex.(findnz(q)[1:2]...) ∩ grididxs
-_targetidx_and_nodes(g::Grid) =
+_targetidx_and_nodes(g::Grid) =
_targetidx_and_nodes(g.target_qualities, g.id_to_grid_coordinate_list)
function _targetidx_and_nodes(target_qualities, id_to_grid_coordinate_list)
targetidx = _targetidx(target_qualities, id_to_grid_coordinate_list)
+ # targetnodes = Vector{Int}(undef, length(targetidx))
+ # n = findfirst(==(id_to_grid_coordinate_list[1]), targetnodes)
+ # targetnodes[1] = n
+ # for i in eachindex(id_to_grid_coordinate_list)[2:end]
+ # findnext(==(id_to_grid_coordinate_list[i]), targetnodes, n)
+ # end
targetnodes = findall(
t -> t ∈ targetidx,
id_to_grid_coordinate_list)
return targetidx, targetnodes
end
-function _fill_matrix(values, g)
+function _fill_matrix(values, g)
M = fill(NaN, g.nrows, g.ncols)
for (i, v) in enumerate(values)
M[g.id_to_grid_coordinate_list[i]] = v
@@ -209,22 +186,6 @@ function indegrees(g::Grid; kwargs...)
_maybe_raster(_fill_matrix(values, g), g)
end
-plot_values(g::Grid, values::Vector; kwargs...) =
- _heatmap(_fill_matrix(values, g), g; kwargs...)
-plot_outdegrees(g::Grid; kwargs...) = _heatmap(outdegrees(g), g; kwargs...)
-plot_indegrees(g::Grid; kwargs...) = _heatmap(indegrees(g), g; kwargs...)
-
-
-# If the grid has raster dimensions,
-# plot as a raster on a spatial grid
-function _heatmap(canvas, g; kwargs...)
- if isnothing(dims(g))
- heatmap(canvas; yflip=true, axis=nothing, border=:none, aspect_ratio=:equal, kwargs...)
- else
- heatmap(Raster(canvas, dims(g)); kwargs...)
- end
-end
-
"""
is_strongly_connected(g::Grid)::Bool
@@ -247,6 +208,46 @@ false
"""
Graphs.is_strongly_connected(g::Grid) = is_strongly_connected(SimpleWeightedDiGraph(g.affinities))
+function split_subgraphs(g::Grid)
+ # Convert cost matrix to graph, todo: is `permute=false` needed
+ graph = SimpleWeightedDiGraph(g.costmatrix, permute=false)
+
+ # Find the subgraphs
+ scc = strongly_connected_components(graph)
+
+ # Keep all subgraphs that contain target nodes
+ subgraphs_with_targets = map(scc) do c
+ length(c) > 1 && any(n -> n in c, g.targetnodes)
+ end
+ subgraphs = sort!(scc[subgraphs_with_targets]; by=length, rev=true)
+
+ # Return a Vector of Grids for each subgraph
+ return map(subgraphs) do scci
+ sort!(scci)
+ affinities = g.affinities[scci, scci]
+ costmatrix = g.costfunction === nothing ? g.costmatrix[scci, scci] : mapnz(g.costfunction, affinities)
+ id_to_grid_coordinate_list = g.id_to_grid_coordinate_list[scci]
+ targetidx, targetnodes = _targetidx_and_nodes(g.target_qualities, id_to_grid_coordinate_list)
+ qs = [g.source_qualities[i] for i in id_to_grid_coordinate_list]
+ qt = [g.target_qualities[i] for i in id_to_grid_coordinate_list ∩ targetidx]
+ Grid(
+ g.nrows,
+ g.ncols,
+ affinities,
+ g.costfunction,
+ costmatrix,
+ id_to_grid_coordinate_list,
+ g.source_qualities,
+ g.target_qualities,
+ targetidx,
+ targetnodes,
+ qs,
+ qt,
+ g.dims,
+ )
+ end
+end
+
"""
largest_subgraph(g::Grid)::Grid
@@ -255,29 +256,30 @@ will have the same size as the input `Grid` but only nodes associated with the
largest subgraph of the affinities will be active.
"""
function largest_subgraph(g::Grid)
- # Convert cost matrix to graph
+ # Convert cost matrix to graph, todo: is `permute=false` needed
graph = SimpleWeightedDiGraph(g.costmatrix, permute=false)
# Find the subgraphs
scc = strongly_connected_components(graph)
- @info "cost graph contains $(length(scc)) strongly connected subgraphs"
+ # @info "cost graph contains $(length(scc)) strongly connected subgraphs"
# Find the largest subgraph
- i = argmax(length.(scc))
+ _, i = findmax(length, scc)
# extract node list and sort it
scci = sort(scc[i])
- ndiffnodes = size(g.costmatrix, 1) - length(scci)
- if ndiffnodes > 0
- @info "removing $ndiffnodes nodes from affinity and cost graphs"
- end
+ # ndiffnodes = size(g.costmatrix, 1) - length(scci)
+ # if ndiffnodes > 0
+ # @info "removing $ndiffnodes nodes from affinity and cost graphs"
+ # end
# Extract the adjacency matrix of the largest subgraph
affinities = g.affinities[scci, scci]
# affinities = convert(SparseMatrixCSC{Float64,Int}, graph[scci])
+ costmatrix = g.costfunction === nothing ? g.costmatrix[scci, scci] : mapnz(g.costfunction, affinities)
id_to_grid_coordinate_list = g.id_to_grid_coordinate_list[scci]
targetidx, targetnodes = _targetidx_and_nodes(g.target_qualities, id_to_grid_coordinate_list)
qs = [g.source_qualities[i] for i in id_to_grid_coordinate_list]
@@ -287,7 +289,7 @@ function largest_subgraph(g::Grid)
g.ncols,
affinities,
g.costfunction,
- g.costfunction === nothing ? g.costmatrix[scci, scci] : mapnz(g.costfunction, affinities),
+ costmatrix,
id_to_grid_coordinate_list,
g.source_qualities,
g.target_qualities,
@@ -361,8 +363,8 @@ A helper-function, used by coarse_graining, that computes the sum of pixels with
"""
sum_neighborhood(g, rc, npix) = sum_neighborhood(g.target_qualities, rc, npix)
function sum_neighborhood(target_qualities::AbstractMatrix, rc, npix)
- getrows = (rc[1] - floor(Int, npix/2)):(rc[1] + (ceil(Int, npix/2) - 1))
- getcols = (rc[2] - floor(Int, npix/2)):(rc[2] + (ceil(Int, npix/2) - 1))
+ getrows = (rc[1]-floor(Int, npix / 2)):(rc[1]+(ceil(Int, npix / 2)-1))
+ getcols = (rc[2]-floor(Int, npix / 2)):(rc[2]+(ceil(Int, npix / 2)-1))
# pixels outside of the landscape are encoded with NaNs but we don't want
# the NaNs to propagate to the coarse grained values
return sum(t -> isnan(t) ? 0.0 : t, target_qualities[getrows, getcols])
@@ -374,7 +376,7 @@ end
Creates a sparse matrix of target qualities for the landmarks based on merging npix pixels into the center pixel.
"""
function coarse_graining(g, npix)
- coarse_graining(g.target_qualities, npix;
+ coarse_graining(g.target_qualities, npix;
id_to_grid_coordinate_list=g.id_to_grid_coordinate_list
)
end
@@ -390,16 +392,16 @@ function coarse_graining(M::AbstractMatrix, npix;
id_to_grid_coordinate_list=_id_gc_list(size(M)...)
)
nrows, ncols = size(M)
- getrows = (floor(Int, npix/2)+1):npix:(nrows-ceil(Int, npix/2)+1)
- getcols = (floor(Int, npix/2)+1):npix:(ncols-ceil(Int, npix/2)+1)
+ getrows = (floor(Int, npix / 2)+1):npix:(nrows-ceil(Int, npix / 2)+1)
+ getcols = (floor(Int, npix / 2)+1):npix:(ncols-ceil(Int, npix / 2)+1)
coarse_target_rc = Base.product(getrows, getcols)
coarse_target_ids = vec(
[
- findfirst(
- isequal(CartesianIndex(ij)),
- id_to_grid_coordinate_list
- ) for ij in coarse_target_rc
- ]
+ findfirst(
+ isequal(CartesianIndex(ij)),
+ id_to_grid_coordinate_list
+ ) for ij in coarse_target_rc
+ ]
)
coarse_target_rc = [ij for ij in coarse_target_rc if !ismissing(ij)]
filter!(!ismissing, coarse_target_ids)
@@ -428,10 +430,11 @@ end
approx::Bool=false
)
-Compute the randomized shorted path based expected costs from all source nodes to
-all target nodes in the graph defined by `g` using the inverse temperature parameter
-`θ`. The computation can either continue until convergence when setting `approx=false`
-(the default) or return an approximate result based on just a single iteration of the Bellman-Ford
+Compute the randomized shorted path based expected costs from all source nodes to all
+target nodes in the graph defined by `g` using the inverse temperature parameter `θ`.
+
+The computation can either continue until convergence when setting `approx=false` (the default)
+or return an approximate result based on just a single iteration of the Bellman-Ford
algorithm when `approx=true`.
"""
function expected_cost(
@@ -473,10 +476,11 @@ end
approx::Bool=false
)
-Compute the directed free energy distance from all source nodes to
-all target nodes in the graph defined by `g` using the inverse temperature parameter
-`θ`. The computation can either continue until convergence when setting `approx=false`
-(the default) or return an approximate result based on just a single iteration of the Bellman-Ford
+Compute the directed free energy distance from all source nodes to all target
+nodes in the graph defined by `g` using the inverse temperature parameter `θ`.
+
+The computation can either continue until convergence when setting `approx=false` (the default),
+or return an approximate result based on just a single iteration of the Bellman-Ford
algorithm when `approx=true`.
"""
function free_energy_distance(
@@ -484,10 +488,8 @@ function free_energy_distance(
θ::Union{Real,Nothing}=nothing,
approx::Bool=false
)
- # FIXME! This should be multithreaded. However, ProgressLogging currently
- # does not support multithreading
targets = ConScape._targetidx_and_nodes(g)[1]
- @progress vec_of_vecs = [_free_energy_distance(g, target, θ, approx) for target in targets]
+ vec_of_vecs = [_free_energy_distance(g, target, θ, approx) for target in targets]
return reduce(hcat, vec_of_vecs)
end
@@ -519,4 +521,4 @@ power_mean_proximity(
g::Grid;
θ::Union{Real,Nothing}=nothing,
approx::Bool=false
-) = survival_probability(g; θ=θ, approx=approx) .^ (1/θ)
\ No newline at end of file
+) = survival_probability(g; θ=θ, approx=approx) .^ (1 / θ)
\ No newline at end of file
diff --git a/src/gridrsp.jl b/src/gridrsp.jl
index c3aa298..391b2b9 100644
--- a/src/gridrsp.jl
+++ b/src/gridrsp.jl
@@ -12,41 +12,33 @@ end
Construct a GridRSP from a `g::Grid` based on the inverse temperature parameter `θ::Real`.
"""
-function GridRSP(g::Grid; θ=nothing)
+function GridRSP(g::Grid; θ=nothing, verbose=true)
Pref = _Pref(g.affinities)
- W = _W(Pref, θ, g.costmatrix)
+ W = _W(Pref, θ, g.costmatrix)
@debug("Computing fundamental matrix of non-absorbing paths (Z). Please be patient...")
- Z = (I - W)\Matrix(sparse(g.targetnodes,
- 1:length(g.targetnodes),
- 1.0,
- size(g.costmatrix, 1),
- length(g.targetnodes)))
+ Z = (I - W) \ Matrix(sparse(g.targetnodes,
+ 1:length(g.targetnodes),
+ 1.0,
+ size(g.costmatrix, 1),
+ length(g.targetnodes)))
# Check that values in Z are not too small:
- if minimum(Z)*minimum(nonzeros(g.costmatrix .* W)) == 0
+ verbose && if minimum(Z) * minimum(nonzeros(g.costmatrix .* W)) == 0
@warn "Warning: Z-matrix contains too small values, which can lead to inaccurate results! Check that the graph is connected or try decreasing θ."
end
return GridRSP(g, θ, Pref, W, Z)
end
-_get_grid(grsp::GridRSP) = grsp.g
-_get_grid(g::Grid) = g
-
-_maybe_raster(mat::Raster, g) = mat
-_maybe_raster(mat::AbstractMatrix, g::Union{Grid,GridRSP}) =
- _maybe_raster(mat, dims(g))
-_maybe_raster(mat::AbstractMatrix, ::Nothing) = mat
-_maybe_raster(mat::AbstractMatrix, dims::Tuple) = Raster(mat, dims)
-
+Base.size(grsp::GridRSP) = size(grsp.g)
function Base.show(io::IO, ::MIME"text/plain", grsp::GridRSP)
print(io, summary(grsp), " of size ", grsp.g.nrows, "x", grsp.g.ncols)
end
-function Base.show(io::IO, ::MIME"text/html", grsp::GridRSP)
- t = string(summary(grsp), " of size ", grsp.g.nrows, "x", grsp.g.ncols)
- write(io, "$t
")
- show(io, MIME"text/html"(), plot_outdegrees(grsp.g))
-end
+# function Base.show(io::IO, ::MIME"text/html", grsp::GridRSP)
+# t = string(summary(grsp), " of size ", grsp.g.nrows, "x", grsp.g.ncols)
+# write(io, "$t
")
+# show(io, MIME"text/html"(), grsp.g))
+# end
DimensionalData.dims(grsp::GridRSP) = dims(grsp.g)
"""
@@ -54,15 +46,21 @@ DimensionalData.dims(grsp::GridRSP) = dims(grsp.g)
Compute RSP betweenness of all nodes weighted by source and target qualities.
"""
-function betweenness_qweighted(grsp::GridRSP)
+function betweenness_qweighted(grsp::Union{GridRSP,NamedTuple};
+ output=_init_output(grsp.g),
+ kw...
+)
g = grsp.g
- betvec = RSP_betweenness_qweighted(grsp.W, grsp.Z, g.qs, g.qt, g.targetnodes)
- bet = fill(NaN, g.nrows, g.ncols)
- for (i, v) in enumerate(betvec)
- bet[g.id_to_grid_coordinate_list[i]] = v
- end
+ betvec = RSP_betweenness_qweighted(grsp.W, grsp.Z, g.qs, g.qt, g.targetnodes; kw...)
+ _update_output!(output, g, betvec)
+ return _maybe_raster(output, g)
+end
- return _maybe_raster(bet, grsp)
+function _update_output!(output, g, betvec)
+ for (I, v) in zip(g.id_to_grid_coordinate_list, betvec)
+ x = output[I]
+ output[I] = isnan(x) ? v : x + v
+ end
end
"""
@@ -71,10 +69,9 @@ end
Compute RSP betweenness of all edges weighted by source and target qualities. Returns a
sparse matrix where element (i,j) is the betweenness of edge (i,j).
"""
-function edge_betweenness_qweighted(grsp::GridRSP)
+function edge_betweenness_qweighted(grsp::Union{GridRSP,NamedTuple}; kw...)
g = grsp.g
- betmatrix = RSP_edge_betweenness_qweighted(grsp.W, grsp.Z, g.qs, g.qt, g.targetnodes)
- return betmatrix
+ return RSP_edge_betweenness_qweighted(grsp.W, grsp.Z, g.qs, g.qt, g.targetnodes; kw...)
end
"""
@@ -83,71 +80,57 @@ end
distance_transformation=inv(grsp.g.costfunction),
diagvalue=nothing])::SparseMatrixCSC{Float64,Int}
-Compute RSP betweenness of all nodes weighted with proximities computed with respect to the distance/proximity measure defined by `connectivity_function`. Optionally, an inverse cost function can be passed. The function will be applied elementwise to the matrix of distances to convert it to a matrix of proximities. If no inverse cost function is passed the the inverse of the cost function is used for the conversion of distances.
+Compute RSP betweenness of all nodes weighted with proximities computed with
+respect to the distance/proximity measure defined by `connectivity_function`.
+Optionally, an inverse cost function can be passed. The function will be applied
+elementwise to the matrix of distances to convert it to a matrix of proximities.
+If no inverse cost function is passed the the inverse of the cost function is
+used for the conversion of distances.
-The optional `diagvalue` element specifies which value to use for the diagonal of the matrix of proximities, i.e. after applying the inverse cost function to the matrix of distances. When nothing is specified, the diagonal elements won't be adjusted.
+The optional `diagvalue` element specifies which value to use for the diagonal
+of the matrix of proximities, i.e. after applying the inverse cost function to the
+matrix of distances. When nothing is specified, the diagonal elements won't be adjusted.
"""
-function betweenness_kweighted(grsp::GridRSP;
- connectivity_function=expected_cost,
- distance_transformation=nothing,
- diagvalue=nothing)
-
+function betweenness_kweighted(grsp::Union{GridRSP,NamedTuple};
+ output=_init_output(grsp.g),
+ proximities=nothing,
+ kw...
+)
g = grsp.g
-
- # Check that distance_transformation function has been passed if no cost function is saved
- if distance_transformation === nothing && connectivity_function <: DistanceFunction
- if g.costfunction === nothing
- throw(ArgumentError("no distance_transformation function supplied and cost matrix in GridRSP isn't based on a cost function."))
- else
- distance_transformation = inv(g.costfunction)
- end
- end
-
- proximities = connectivity_function(grsp)
-
- if connectivity_function <: DistanceFunction
- map!(distance_transformation, proximities, proximities)
+ if isnothing(proximities)
+ proximities = _computeproximities(grsp; kw...)
end
- if diagvalue !== nothing
- for (j, i) in enumerate(targetnodes)
- proximities[i, j] = diagvalue
- end
- end
-
- betvec = RSP_betweenness_kweighted(grsp.W, grsp.Z, g.qs, g.qt, proximities, g.targetnodes)
-
- bet = fill(NaN, g.nrows, g.ncols)
- for (i, v) in enumerate(betvec)
- bet[g.id_to_grid_coordinate_list[i]] = v
- end
-
- return _maybe_raster(bet, grsp)
+ betvec = RSP_betweenness_kweighted(grsp.W, grsp.Z, g.qs, g.qt, proximities, g.targetnodes; kw...)
+ _update_output!(output, g, betvec)
+ return _maybe_raster(output, g)
end
+
"""
edge_betweenness_kweighted(grsp::GridRSP; [distance_transformation=inv(grsp.g.costfunction), diagvalue=nothing])::SparseMatrixCSC{Float64,Int}
- Compute RSP betweenness of all edges weighted by qualities of source s and target t and the proximity between s and t. Returns a
- sparse matrix where element (i,j) is the betweenness of edge (i,j).
+ Compute RSP betweenness of all edges weighted by qualities of source s and target t and the proximity between s and t. Returns a sparse matrix where element (i,j) is the betweenness of edge (i,j).
- The optional `diagvalue` element specifies which value to use for the diagonal of the matrix
of proximities, i.e. after applying the inverse cost function to the matrix of expected costs.
When nothing is specified, the diagonal elements won't be adjusted.
"""
-function edge_betweenness_kweighted(grsp::GridRSP;
- distance_transformation=inv(grsp.g.costfunction),
+function edge_betweenness_kweighted(grsp::Union{GridRSP,NamedTuple};
+ proximities=nothing,
+ distance_transformation=nothing,
diagvalue=nothing,
+ kw...
)
+ if isnothing(distance_transformation)
+ distance_transformation = inv(grsp.g.costfunction)
+ end
+ # TODO why does this only use `expected_cost`?
g = grsp.g
+ # S = map(distance_transformation, expected_cost(grsp))
proximities = map(distance_transformation, expected_cost(grsp))
- if diagvalue !== nothing
- for (j, i) in enumerate(g.targetnodes)
- proximities[i, j] = diagvalue
- end
- end
+ maybe_set_diagonal!(proximities, diagvalue, g.targetnodes)
- betmatrix = RSP_edge_betweenness_kweighted(grsp.W, grsp.Z, g.qs, g.qt, proximities, g.targetnodes)
+ betmatrix = RSP_edge_betweenness_kweighted(grsp.W, grsp.Z, g.qs, g.qt, proximities, g.targetnodes; kw...)
return betmatrix
end
@@ -156,65 +139,104 @@ end
Compute RSP expected costs from all nodes.
"""
-expected_cost(grsp::GridRSP) =
- RSP_expected_cost(grsp.W, grsp.g.costmatrix, grsp.Z, grsp.g.targetnodes)
+expected_cost(grsp::Union{GridRSP,NamedTuple}; kw...) =
+ RSP_expected_cost(grsp.W, grsp.g.costmatrix, grsp.Z, grsp.g.targetnodes; kw...)
-free_energy_distance(grsp::GridRSP) =
- RSP_free_energy_distance(grsp.Z, grsp.θ, grsp.g.targetnodes)
+free_energy_distance(grsp::Union{GridRSP,NamedTuple}; kw...) =
+ RSP_free_energy_distance(grsp.Z, grsp.θ, grsp.g.targetnodes; kw...)
-survival_probability(grsp::GridRSP) =
- RSP_survival_probability(grsp.Z, grsp.θ, grsp.g.targetnodes)
+survival_probability(grsp::Union{GridRSP,NamedTuple}; kw...) =
+ RSP_survival_probability(grsp.Z, grsp.θ, grsp.g.targetnodes; kw...)
-power_mean_proximity(grsp::GridRSP) =
- RSP_power_mean_proximity(grsp.Z, grsp.θ, grsp.g.targetnodes)
+power_mean_proximity(grsp::Union{GridRSP,NamedTuple}; kw...) =
+ RSP_power_mean_proximity(grsp.Z, grsp.θ, grsp.g.targetnodes; kw...)
-least_cost_distance(grsp::GridRSP) = least_cost_distance(grsp.g)
+least_cost_distance(grsp::Union{GridRSP,NamedTuple}; kw...) = least_cost_distance(grsp.g; kw...)
"""
mean_kl_divergence(grsp::GridRSP)::Float64
-Compute the mean Kullback–Leibler divergence between the free energy distances and the RSP expected costs for `grsp::GridRSP`.
+Compute the mean Kullback–Leibler divergence between the free
+energy distances and the RSP expected costs for `grsp::GridRSP`.
"""
-function mean_kl_divergence(grsp::GridRSP)
+function mean_kl_divergence(grsp::Union{GridRSP,NamedTuple};
+ free_energy_distances=nothing,
+ expected_costs=nothing,
+ kw...
+)
g = grsp.g
- return g.qs' * (RSP_free_energy_distance(grsp.Z, grsp.θ, g.targetnodes) - expected_cost(grsp)) * g.qt * grsp.θ
+ free_energy_distances = if isnothing(free_energy_distances)
+ RSP_free_energy_distance(grsp.Z, grsp.θ, g.targetnodes; kw...)
+ else
+ free_energy_distances
+ end
+ expected_costs = if isnothing(expected_costs)
+ ConScape.expected_cost(grsp; kw...)
+ else
+ expected_costs
+ end
+ return mean_kl_divergence(grsp::Union{GridRSP,NamedTuple}, free_energy_distances, expected_costs; kw...)
+end
+function mean_kl_divergence(grsp::Union{GridRSP,NamedTuple}, free_energy_distances, expected_costs;
+ workspaces=(similar(grsp.Z),), kw...
+)
+ g = grsp.g
+ fed_exp = workspaces[1] .= free_energy_distances .- expected_costs
+ return g.qs' * fed_exp * g.qt * grsp.θ
end
"""
mean_lc_kl_divergence(grsp::GridRSP)::Float64
-Compute the mean Kullback–Leibler divergence between the least-cost path and the random path distribution for `grsp::GridRSP`, weighted by the qualities of the source and target node.
+Compute the mean Kullback–Leibler divergence between the least-cost path and the random path
+distribution for `grsp::GridRSP`, weighted by the qualities of the source and target node.
"""
-function mean_lc_kl_divergence(grsp::GridRSP)
+function mean_lc_kl_divergence(grsp::Union{GridRSP,NamedTuple};
+ workspaces=[similar(grsp.Z)],
+ kw...
+)
+ workspace1 = workspaces[1]
g = grsp.g
- div = hcat([least_cost_kl_divergence(g.costmatrix, grsp.Pref, i) for i in g.targetnodes]...)
+ C = g.costmatrix
+ cost_weighted_digraph = SimpleWeightedDiGraph(C)
+ n = size(C, 1)
+ from = Array{Int}(undef, n)
+ kl_div = Array{Float64}(undef, n)
+ # Previously
+ # div = hcat([least_cost_kl_divergence(C, grsp.Pref, i; cost_weighted_digraph, from, kl_div, kw...) for i in g.targetnodes]...)
+ div = workspace1
+ for i in g.targetnodes
+ div[i, :] .= least_cost_kl_divergence(C, grsp.Pref, i; cost_weighted_digraph, from, kl_div, kw...)
+ end
return g.qs' * div * g.qt
end
-function least_cost_kl_divergence(C::SparseMatrixCSC, Pref::SparseMatrixCSC, targetnode::Integer)
+function least_cost_kl_divergence(C::SparseMatrixCSC, Pref::SparseMatrixCSC, targetnode::Integer;
+ cost_weighted_digraph=SimpleWeightedDiGraph(C),
+ n=size(C, 1),
+ from=Array{Int}(undef, n),
+ kl_div=Array{Float64}(undef, n),
+ kw...
+)
+ from .= 1:n
+ fill!(kl_div, 0)
- n = size(C, 1)
- graph = SimpleWeightedDiGraph(C)
if !(1 <= targetnode <= n)
throw(ArgumentError("target node not found"))
end
- dsp = dijkstra_shortest_paths(graph, targetnode)
+ dsp = dijkstra_shortest_paths(cost_weighted_digraph, targetnode)
parents = dsp.parents
parents[targetnode] = targetnode
-
- from = collect(1:n)
- to = copy(parents)
-
- kl_div = zeros(n)
+ to = copy(parents)
while true
notdone = false
for i in 1:n
fromᵢ = from[i]
- toᵢ = to[i]
+ toᵢ = to[i]
notdone |= fromᵢ != toᵢ
if fromᵢ == toᵢ
continue
@@ -228,29 +250,28 @@ function least_cost_kl_divergence(C::SparseMatrixCSC, Pref::SparseMatrixCSC, tar
end
# Pointer swap
- tmp = from
+ tmp = from
from = to
- to = tmp
+ to = tmp
end
return kl_div
end
-
"""
least_cost_kl_divergence(grsp::GridRSP, target::Tuple{Int,Int})
-Compute the least cost Kullback-Leibler divergence from each cell in the g in
-`h` to the `target` cell.
+Compute the least cost Kullback-Leibler divergence from each
+cell in the g in `h` to the `target` cell.
"""
-function least_cost_kl_divergence(grsp::GridRSP, target::Tuple{Int,Int})
+function least_cost_kl_divergence(grsp::Union{GridRSP,NamedTuple}, target::Tuple{Int,Int}; kw...)
g = grsp.g
targetnode = findfirst(isequal(CartesianIndex(target)), g.id_to_grid_coordinate_list)
if targetnode === nothing
throw(ArgumentError("target cell not found"))
end
- div = least_cost_kl_divergence(g.costmatrix, grsp.Pref, targetnode)
+ div = least_cost_kl_divergence(g.costmatrix, grsp.Pref, targetnode; kw...)
return reshape(div, g.nrows, g.ncols)
end
@@ -283,103 +304,105 @@ requires it such as `expected_cost`. Also for `Grid` objects, the `approx` Boole
argument can be set to `true` to switch to a cheaper approximate solution of the
`connectivity_function`. The default value is `false`.
"""
-function connected_habitat(grsp::Union{Grid,GridRSP};
+function connected_habitat(
+ grsp::Grid;
connectivity_function=expected_cost,
distance_transformation=nothing,
diagvalue=nothing,
θ::Union{Nothing,Real}=nothing,
- approx::Bool=false)
-
+ approx::Bool=false
+)
# Check that distance_transformation function has been passed if no cost function is saved
if distance_transformation === nothing && connectivity_function <: DistanceFunction
- if grsp isa Grid
- throw(ArgumentError("distance_transformation function is required when passing a Grid together with a Distance function"))
- elseif grsp.g.costfunction === nothing
- throw(ArgumentError("no distance_transformation function supplied and cost matrix in GridRSP isn't based on a cost function."))
- else
- distance_transformation = inv(grsp.g.costfunction)
- end
+ throw(ArgumentError("distance_transformation function is required when passing a Grid together with a Distance function"))
end
- S = if grsp isa Grid
- if θ === nothing && connectivity_function !== least_cost_distance
- throw(ArgumentError("θ must be a positive real number when passing a Grid"))
- end
- connectivity_function(grsp; θ=θ, approx=approx)
- else
- if θ !== nothing
- throw(ArgumentError("θ must be unspecified when passing a GridRSP"))
- end
- connectivity_function(grsp)
+ if θ === nothing && connectivity_function !== least_cost_distance
+ throw(ArgumentError("θ must be a positive real number when passing a Grid"))
end
-
+ proximities = connectivity_function(grsp; θ=θ, approx=approx)
if connectivity_function <: DistanceFunction
- map!(distance_transformation, S, S)
+ map!(distance_transformation, proximities, proximities)
end
- return connected_habitat(grsp, S, diagvalue=diagvalue)
+ return connected_habitat(grsp, proximities; diagvalue)
end
-function connected_habitat(grsp::Union{Grid,GridRSP}, S::Matrix; diagvalue::Union{Nothing,Real}=nothing)
- g = _get_grid(grsp)
- if diagvalue !== nothing
- for (j, i) in enumerate(targetnodes)
- S[i, j] = diagvalue
- end
+function connected_habitat(grsp::GridRSP; proximities=nothing, kw...)
+ if isnothing(proximities)
+ proximities = _computeproximities(grsp; kw...)
end
+ return connected_habitat(grsp, proximities; kw...)
+end
+connected_habitat(grsp::GridRSP, S::Matrix; kw...) =
+ connected_habitat(grsp.g, S; kw...)
+function connected_habitat(g::Grid, S::Matrix;
+ diagvalue::Union{Nothing,Real}=nothing,
+ output=_init_output(g),
+ kw...
+)
+ maybe_set_diagonal!(S, diagvalue, g.targetnodes)
- funvec = connected_habitat(g.qs, g.qt, S)
+ funvec = connected_habitat(g.qs, g.qt, S; kw...)
- func = fill(NaN, g.nrows, g.ncols)
- for (ij, x) in zip(g.id_to_grid_coordinate_list, funvec)
- func[ij] = x
+ for (I, x) in zip(g.id_to_grid_coordinate_list, funvec)
+ output[I] = x
end
- return _maybe_raster(func, grsp)
+ return _maybe_raster(output, g)
end
+function connected_habitat(grsp::GridRSP, cell::CartesianIndex{2};
+ distance_transformation=nothing,
+ diagvalue=nothing,
+ avalue=floatmin(), # smallest non-zero value
+ qˢvalue=0.0,
+ qᵗvalue=0.0,
+ kw...)
-function connected_habitat(grsp::GridRSP,
- cell::CartesianIndex{2};
- distance_transformation=nothing,
- diagvalue=nothing,
- avalue=floatmin(), # smallest non-zero value
- qˢvalue=0.0,
- qᵗvalue=0.0)
+ g = grsp.g
if avalue <= 0.0
throw("Affinity value has to be positive. Otherwise the graph will become disconnected.")
end
# Compute (linear) node indices from (cartesian) grid indices
- node = findfirst(isequal(cell), grsp.g.id_to_grid_coordinate_list)
+ node = findfirst(isequal(cell), g.id_to_grid_coordinate_list)
# Check that cell is in targetidx
- if cell ∉ targetidx
+ if cell ∉ g.targetidx
throw(ArgumentError("Computing adjusted connected_habitat is only supported for target cells"))
end
- affinities = copy(grsp.g.affinities)
+ affinities = copy(g.affinities)
affinities[:, node] .= ifelse.(iszero.(affinities[:, node]), 0, avalue)
affinities[node, :] .= ifelse.(iszero.(affinities[node, :]), 0, avalue)
- newsource_qualities = copy(grsp.g.source_qualities)
+ newsource_qualities = copy(g.source_qualities)
newsource_qualities[cell] = qˢvalue
- newtarget_qualities = copy(grsp.g.target_qualities)
+ newtarget_qualities = copy(g.target_qualities)
newtarget_qualities[cell] = qᵗvalue
- newg = Grid(grsp.g.nrows,
- grsp.g.ncols,
- affinities,
- grsp.g.costfunction,
- grsp.g.costfunction === nothing ? grsp.g.costmatrix : mapnz(grsp.g.costfunction, affinities),
- grsp.g.id_to_grid_coordinate_list,
- newsource_qualities,
- newtarget_qualities,
- dims(grsp))
+ newtargetidx, newtargetnodes = _targetidx_and_nodes(newtarget_qualities, g.id_to_grid_coordinate_list)
+ newqs = [newsource_qualities[i] for i in g.id_to_grid_coordinate_list]
+ newqt = [newtarget_qualities[i] for i in g.id_to_grid_coordinate_list ∩ newtargetidx]
+
+ newg = Grid(g.nrows,
+ g.ncols,
+ affinities,
+ g.costfunction,
+ g.costfunction === nothing ? g.costmatrix : mapnz(g.costfunction, affinities),
+ g.id_to_grid_coordinate_list,
+ newsource_qualities,
+ newtarget_qualities,
+ newtargetidx,
+ newtargetnodes,
+ newqs,
+ newqt,
+ dims(g))
newh = GridRSP(newg; θ=grsp.θ)
- return connected_habitat(newh; diagvalue=diagvalue, distance_transformation=distance_transformation)
+ return connected_habitat(newh; diagvalue, distance_transformation)
end
"""
@@ -389,15 +412,27 @@ end
diagvalue=nothing,
tol=1e-14)
-Compute the largest eigenvalue triple (left vector, value, and right vector) of the quality scaled proximities with respect to the distance/proximity measure defined by `connectivity_function`. If `connectivity_function` is a distance measure then the distances are transformed to proximities by `distance_transformation` which defaults to the inverse of the `costfunction` in the underlying `Grid` (if defined). Optionally, the diagonal values of the proximity matrix may be set to `diagvalue`. The `tol` argument specifies the convergence tolerance in the Arnoldi based eigensolver.
+Compute the largest eigenvalue triple (left vector, value, and right vector) of the
+quality scaled proximities with respect to the distance/proximity measure defined by
+`connectivity_function`.
+
+If `connectivity_function` is a distance measure then the distances are transformed
+to proximities by `distance_transformation` which defaults to the inverse of the `costfunction`
+in the underlying `Grid` (if defined). Optionally, the diagonal values of the proximity matrix may
+be set to `diagvalue`. The `tol` argument specifies the convergence tolerance in the Arnoldi based eigensolver.
"""
-function LinearAlgebra.eigmax(grsp::GridRSP;
+function LinearAlgebra.eigmax(grsp::Union{GridRSP,NamedTuple};
connectivity_function=expected_cost,
distance_transformation=nothing,
diagvalue=nothing,
- tol=1e-14)
-
+ workspaces=[similar(grsp.Z), similar(grsp.Z), similar(grsp.Z)],
+ tol=1e-14,
+ expected_costs=nothing,
+ free_energy_distances=nothing,
+ kw...
+)
g = grsp.g
+ workspace1, workspace2, workspace3 = workspaces
# Check that distance_transformation function has been passed if no cost function is saved
if distance_transformation === nothing && connectivity_function <: DistanceFunction
@@ -408,32 +443,39 @@ function LinearAlgebra.eigmax(grsp::GridRSP;
end
end
- S = connectivity_function(grsp)
+ proximities = if connectivity_function == ConScape.expected_cost && !isnothing(expected_costs)
+ # workspace1 .= expected_costs
+ # workspace1
+ copy(expected_costs)
+ elseif connectivity_function == ConScape.free_energy_distance && !isnothing(free_energy_distances)
+ workspace1 .= free_energy_distances
+ workspace1
+ else
+ connectivity_function(grsp; kw...)
+ end
+ # proximities = connectivity_function(grsp; kw...)
if connectivity_function <: DistanceFunction
- map!(distance_transformation, S, S)
+ map!(distance_transformation, proximities, proximities)
end
- if diagvalue !== nothing
- for (j, i) in enumerate(g.targetnodes)
- S[i, j] = diagvalue
- end
- end
+ maybe_set_diagonal!(S, diagvalue, g.targetnodes)
# quality scaled proximity matrix
- qSq = qˢ .* S .* qᵗ'
+ qSq = workspace2 .= g.qs .* S .* g.qt'
# square submatrix defined by extracting the rows corresponding to landmarks
- qSq₀₀ = qSq[targetnodes,:]
+ qSq₀₀ = view(workspace3, 1:size(workspace3, 2), :)
+ qSq₀₀ .= view(qSq, g.targetnodes, :)
# size of the full problem
n = size(g.affinities, 1)
# node ids for the non-landmarks
- p₁ = setdiff(1:n, targetnodes)
+ p₁ = setdiff(1:n, g.targetnodes)
# use an Arnoldi based eigensolver to compute the largest (absolute) eigenvalue and right vector (of submatrix)
- Fps = partialschur(qSq₀₀, nev=1, tol=tol)
+ Fps = partialschur(qSq₀₀, nev=1, tol=tol)
λ₀, vʳ₀ = partialeigen(Fps[1])
# Some notes on handling intended or unintended landmarks. When the Grid includes landmarks,
@@ -487,19 +529,19 @@ function LinearAlgebra.eigmax(grsp::GridRSP;
# construct full right vector
vʳ = fill(NaN, n)
- vʳ[targetnodes] = vʳ₀
- vʳ[p₁] = qSq[p₁,:]*vʳ₀/λ₀[1]
+ vʳ[g.targetnodes] = vʳ₀
+ vʳ[p₁] = view(qSq, p₁, :) * vʳ₀ / λ₀[1]
# compute left vector (of submatrix) by shift-invert
- Flu = lu(qSq₀₀ - λ₀[1]*I)
- vˡ₀ = ldiv!(Flu', rand(length(targetidx)))
+ Flu = lu(qSq₀₀ - λ₀[1] * I)
+ vˡ₀ = ldiv!(Flu', rand(length(g.targetidx)))
rmul!(vˡ₀, inv(vˡ₀[1]))
# construct full left vector
vˡ = zeros(n)
- vˡ[targetnodes] = vˡ₀
+ vˡ[g.targetnodes] = vˡ₀
- return vˡ, λ₀[1], vʳ
+ return (vˡ, λ₀=λ₀[1], vʳ)
end
"""
@@ -515,33 +557,66 @@ for the cell to `avalue` as well as the source and target qualities associated w
the cell to `qˢvalue` and `qᵗvalue` respectively. It is required that `avalue` is
positive to avoid that the graph becomes disconnected.
"""
-function criticality(grsp::GridRSP;
- distance_transformation=nothing,
- diagvalue=nothing,
- avalue=floatmin(),
- qˢvalue=0.0,
- qᵗvalue=0.0)
-
+function criticality(grsp::Union{GridRSP,NamedTuple};
+ distance_transformation=nothing,
+ diagvalue=nothing,
+ avalue=floatmin(),
+ output=_init_output(grsp.g),
+ qˢvalue=0.0,
+ qᵗvalue=0.0,
+ kw...
+)
g = grsp.g
- nl = length(targetidx)
- reference_connected_habitat = sum(connected_habitat(grsp;
- distance_transformation=distance_transformation, diagvalue=diagvalue
+ nl = length(g.targetidx)
+ reference_connected_habitat = sum(connected_habitat(grsp;
+ distance_transformation, diagvalue, kw...
))
critvec = fill(reference_connected_habitat, nl)
- @progress name="Computing criticality..." for i in 1:nl
- critvec[i] -= sum(connected_habitat(
- grsp,
- g.targetidx[i];
- distance_transformation=distance_transformation,
- diagvalue=diagvalue,
- avalue=avalue,
- qˢvalue=qˢvalue,
- qᵗvalue=qᵗvalue))
+ @progress name = "Computing criticality..." for i in 1:nl
+ critvec[i] = sum(connected_habitat(grsp, g.targetidx[i];
+ distance_transformation, diagvalue, avalue, qˢvalue, qᵗvalue, kw...
+ ))
+ end
+
+ output[g.targetidx] = critvec
+
+ return _maybe_raster(output, grsp)
+end
+
+function _computeproximities(grsp;
+ connectivity_function=expected_cost,
+ distance_transformation=nothing,
+ diagvalue=nothing,
+ kw...
+)
+ g = grsp.g
+ proximities = connectivity_function(grsp; kw...)
+
+ # Check that distance_transformation function has been passed if no cost function is saved
+ if connectivity_function <: DistanceFunction
+ if distance_transformation === nothing
+ if g.costfunction === nothing
+ throw(ArgumentError("no distance_transformation function supplied and cost matrix in GridRSP isn't based on a cost function."))
+ else
+ distance_transformation = inv(g.costfunction)
+ end
+ end
+ map!(distance_transformation, proximities, proximities)
end
+ maybe_set_diagonal!(proximities, diagvalue, g.targetnodes)
+ return proximities
+end
- landscape = fill(NaN, size(grsp.g))
- landscape[targetidx] = critvec
+maybe_set_diagonal!(proximities, diagvalue::Nothing, targetnodes::AbstractVector) = nothing
+function maybe_set_diagonal!(proximities, diagvalue, targetnodes::AbstractVector)
+ for (j, i) in enumerate(targetnodes)
+ proximities[i, j] = diagvalue
+ end
+end
- return _maybe_raster(landscape, grsp)
+function _init_output(g::Grid)
+ o = fill(eltype(g.affinities)(0.0), size(g))
+ o[g.id_to_grid_coordinate_list] .= 0
+ return o
end
\ No newline at end of file
diff --git a/src/operations.jl b/src/operations.jl
deleted file mode 100644
index e69de29..0000000
diff --git a/src/path_distributions.jl b/src/path_distributions.jl
deleted file mode 100644
index e69de29..0000000
diff --git a/src/problem.jl b/src/problem.jl
index 9f58f48..e7f937e 100644
--- a/src/problem.jl
+++ b/src/problem.jl
@@ -1,39 +1,20 @@
-# Defined earlier in ConScape.jl for load order
-# abstract type AbstractProblem end
-@doc """
- Problem
-
-Abstract supertype for ConScape problem specifications.
-""" Problem
-
# Recusive getters for nested problems
graph_measures(p::AbstractProblem) = graph_measures(p.problem)
connectivity_measure(p::AbstractProblem) = connectivity_measure(p.problem)
connectivity_function(p::AbstractProblem) =
connectivity_function(connectivity_measure(p))
solver(p::AbstractProblem) = solver(p.problem)
-
-"""
- solve(problem, grid::Union{Grid,GridRSP})
-
-Solve problem `o` for a grid.
-"""
-function solve end
-
-"""
- assess(p::AbstractProblem, g)
-
-Assess the memory and solve requirements of problem
-`p` on grid `g`. This can be used to indicate memory
-and time reequiremtents on a cluster
-"""
-function assess end
+isthreaded(p::AbstractProblem) = false
"""
Problem(graph_measures...; solver, θ)
-Combine multiple solve operations into a single object,
-to be run in the same job.
+A `Problem` specifies graph and connectivity measures,
+and a method to solve them.
+
+This lazy specification allows ConScape to minimise the work
+required to calculate multiple outputs: habitat conectivity
+betweenness metrics etc can use the same memory allocations and solves.
# Keywords
@@ -41,39 +22,45 @@ to be run in the same job.
- `connectivity_measure`: A [`ConnectivityMeasure`](@ref).
- `solver`: A [`Solver`](@ref) specification.
"""
-@kwdef struct Problem{GM,CM<:ConnectivityMeasure,SM<:Solver} <: AbstractProblem
+@kwdef struct Problem{GM,CM<:ConnectivityMeasure,SM<:Solver,DV,CO} <: AbstractProblem
graph_measures::GM
connectivity_measure::CM = LeastCostDistance()
solver::SM = MatrixSolver()
+ diagvalue::DV = nothing
+ costs::CO = MinusLog()
+ prune::Bool = true
end
Problem(graph_measures::Union{Tuple,NamedTuple}; kw...) = Problem(; graph_measures, kw...)
+function Base.show(io, mime, p::Problem; indent="")
+ println(io, typeof(p).name.wrapper)
+ # println(io, indent, "graph_measures: ", p.graph_measures)
+ # println(io, indent, "connectivity_measure: ", p.connectivity_measure)
+ # println(io, indent, "costs: ", p.costs)
+ # println(io, indent, "solver: ", p.solver)
+ # println(io, indent, "diagvalue: ", typeof(p.diagvalue))
+ # println(io, indent, "prune: ", p.prune)
+end
+
+diagvalue(p::Problem) = p.diagvalue
graph_measures(p::Problem) = p.graph_measures
connectivity_measure(p::Problem) = p.connectivity_measure
solver(p::Problem) = p.solver
-
-solve(p::Problem, rast::RasterStack) = solve(p, Grid(p, rast))
-solve(p::Problem, g::Grid) = solve(p.solver, connectivity_measure(p), p, g)
-
-# @kwdef struct ComputeAssesment{P,M,T}
-# problem::P
-# mem_stats::M
-# totalmem::T
-# end
-
-# """
-# allocate(co::ComputeAssesment)
-
-# Allocate memory required to run `solve` for the assessed ops.
-
-# The returned object can be passed as the `allocs` keyword to `solve`.
-# """
-# function allocate(co::ComputeAssesment)
-# zmax = co.zmax
-# # But actually do this with GenericMemory using Julia v1.11
-# Z = Matrix{Float64}(undef, co.zmax)
-# S = sparse(1:zmax[1], 1:zmax[2], 1.0, zmax...)
-# L = lu(S)
-# # Just return a NamedTuple for now
-# return (; Z, S, L)
-# end
+costs(p::Problem) = p.costs
+prune(p::Problem) = p.prune
+isthreaded(p::Problem) = p.threaded
+
+# Solve just calls `init` and `solve!`
+solve(p::Problem, rast::RasterStack; kw...) = solve!(init(p, rast; kw...), p; kw...)
+# Solve defers to specific solver methods in solvers.jl
+solve!(workspace::NamedTuple, p::Problem; kw...) =
+ solve!(workspace, solver(p), connectivity_measure(p), p; kw...)
+
+# `init`` calls `init!` on an empty workspace
+init(p::Problem, args...; kw...) = init!((;), p, args...; kw...)
+# init! requirements are conditional on solver and connectivity measure
+# See solvers.jl
+function init!(workspace::NamedTuple, p::Problem, rast::RasterStack; verbose=false, kw...)
+ verbose && println("Initialising for $(solver(p))")
+ init!(workspace, solver(p), connectivity_measure(p), p, rast; kw...)
+end
diff --git a/src/randomizedshortestpath.jl b/src/randomizedshortestpath.jl
index d4edbbf..3261b16 100644
--- a/src/randomizedshortestpath.jl
+++ b/src/randomizedshortestpath.jl
@@ -1,3 +1,21 @@
+# Generate the sparse diagonal rhs matrix
+function sparse_rhs(targetnodes, n)
+ sparse(targetnodes,
+ 1:length(targetnodes),
+ 1.0,
+ n,
+ length(targetnodes),
+ )
+end
+
+_inv(Z) = _inv!(similar(Z), Z)
+function _inv!(Zⁱ, Z)
+ broadcast!(Zⁱ, Z) do x
+ x = inv(x)
+ isfinite(x) ? x : floatmax(eltype(Z))
+ end
+end
+
_Pref(A::SparseMatrixCSC) = Diagonal(inv.(vec(sum(A, dims=2)))) * A
function _W(Pref::SparseMatrixCSC, θ::Real, C::SparseMatrixCSC)
@@ -14,34 +32,49 @@ function _W(Pref::SparseMatrixCSC, θ::Real, C::SparseMatrixCSC)
end
function RSP_betweenness_qweighted(W::SparseMatrixCSC,
- Z::AbstractMatrix,
- qˢ::AbstractVector,
- qᵗ::AbstractVector,
- targetnodes::AbstractVector)
- Zⁱ = inv.(Z)
- Zⁱ[.!isfinite.(Zⁱ)] .= floatmax(eltype(Z)) # To prevent Inf*0 later...
-
- qˢZⁱqᵗ = qˢ .* Zⁱ .* qᵗ'
+ Z::AbstractMatrix,
+ qˢ::AbstractVector,
+ qᵗ::AbstractVector,
+ targetnodes::AbstractVector;
+ Zⁱ=_inv(Z),
+ workspaces=[similar(Z), similar(Z)],
+ solver=nothing,
+ Aadj=(I - W)',
+ Aadj_init=init(solver, Aadj),
+ kw...
+)
+ workspace1, workspace2 = workspaces
+
+ qˢZⁱqᵗ = workspace1
+ qˢZⁱqᵗ .= qˢ .* Zⁱ .* qᵗ'
sumqˢ = sum(qˢ)
for j in axes(Z, 2)
- qˢZⁱqᵗ[targetnodes[j], j] -= sumqˢ * qᵗ[j] * Zⁱ[targetnodes[j], j]
+ qˢZⁱqᵗ[targetnodes[j], j] -= sumqˢ * qᵗ[j] * Zⁱ[targetnodes[j], j]
end
- ZqˢZⁱqᵗZt = (I - W)'\qˢZⁱqᵗ
+ # TODO adjoint of LinearSolver?
+ ZqˢZⁱqᵗZt = ldiv!(solver, Aadj_init, qˢZⁱqᵗ; B_copy=copy!(workspace2, qˢZⁱqᵗ))
ZqˢZⁱqᵗZt .*= Z
- return sum(ZqˢZⁱqᵗZt, dims=2) # diag(Z * ZqˢZⁱqᵗ')
+ # TODO remove this allocation
+ return sum.(eachslice(ZqˢZⁱqᵗZt, dims=1)) # diag(Z * ZqˢZⁱqᵗ')
end
function RSP_betweenness_kweighted(W::SparseMatrixCSC,
- Z::AbstractMatrix, # Fundamental matrix of non-absorbing paths
- qˢ::AbstractVector, # Source qualities
- qᵗ::AbstractVector, # Target qualities
- S::AbstractMatrix, # Matrix of proximities
- landmarks::AbstractVector)
-
-
+ Z::AbstractMatrix, # Fundamental matrix of non-absorbing paths
+ qˢ::AbstractVector, # Source qualities
+ qᵗ::AbstractVector, # Target qualities
+ S::AbstractMatrix, # Matrix of proximities
+ landmarks::AbstractVector;
+ Zⁱ=_inv(Z),
+ workspaces=[similar(Z)],
+ solver=nothing,
+ Aadj=(I - W)',
+ Aadj_init=init(solver, Aadj),
+ kw...
+)
+ workspace1 = workspaces[1]
axis1, axis2 = axes(Z)
if axis1 != axes(qˢ, 1)
throw(DimensionMismatch(""))
@@ -56,71 +89,77 @@ function RSP_betweenness_kweighted(W::SparseMatrixCSC,
throw(DimensionMismatch(""))
end
- Zⁱ = inv.(Z)
- Zⁱ[.!isfinite.(Zⁱ)] .= floatmax(eltype(Z)) # To prevent Inf*0 later...
-
- KZⁱ = qˢ .* S .* qᵗ'
+ # Write into proximities
+ KZⁱ = S
+ KZⁱ .*= qˢ .* qᵗ'
# If any of the values of KZⁱ is above one then there is a risk of overflow.
# Hence, we scale the matrix and apply the scale factor by the end of the
# computation.
λ = max(1.0, maximum(KZⁱ))
- k = vec(sum(KZⁱ, dims=1)) * inv(λ)
+ # k = vec(sum(KZⁱ, dims=1)) * inv(λ)
+ ws_col = view(workspace1, 1:1, :)
+ k = vec(sum!(ws_col, KZⁱ))
+ k .*= inv(λ)
KZⁱ .*= inv.(λ) .* Zⁱ
for j in axis2
KZⁱ[landmarks[j], j] -= k[j] .* Zⁱ[landmarks[j], j]
end
- ZKZⁱt = (I - W)'\KZⁱ
+ # KZi overwritten from here
+ # ZKZⁱt = (I - W)'\KZⁱ
+ ZKZⁱt = ldiv!(solver, Aadj_init, KZⁱ; B_copy=copy!(workspace1, KZⁱ))
ZKZⁱt .*= λ .* Z
- return vec(sum(ZKZⁱt, dims=2)) # diag(Z * KZⁱ')
+ scratch = view(workspace1, :, 1:1)
+ return vec(sum!(scratch, ZKZⁱt)) # diag(Z * KZⁱ')
+ # return vec(sum(ZKZⁱt, dims=2)) # diag(Z * KZⁱ')
end
function RSP_edge_betweenness_qweighted(W::SparseMatrixCSC,
- Z::AbstractMatrix,
- qˢ::AbstractVector,
- qᵗ::AbstractVector,
- targetnodes::AbstractVector)
+ Z::AbstractMatrix,
+ qˢ::AbstractVector,
+ qᵗ::AbstractVector,
+ targetnodes::AbstractVector;
+ solver=nothing,
+ Zⁱ=_inv(Z),
+ workspaces=[similar(Z), similar(Z), similar(Z)],
+ Aadj=(I - W)',
+ Aadj_init=init(solver, Aadj),
+ B_sparse=sparse_rhs(targetnodes, size(W, 1)),
+ kw...
+)
+ edge_betweennesses = copy(W)
+ n = size(W, 1)
+ workspace1, workspace2, workspace3 = workspaces
- Zⁱ = inv.(Z)
- Zⁱ[.!isfinite.(Zⁱ)] .= floatmax(eltype(Z)) # To prevent Inf*0 later...
+ diagZⁱ = [Zⁱ[targetnodes[t], t] for t in 1:length(targetnodes)]
+ sumqˢ = sum(qˢ)
- # FIXME: This should be only done when actually size(Z,2) < size(Z,1)/K where K ≈ 10 or so.
+ # FIXME: This should be only done when actually size(Z, 2) < size(Z, 1)/K where K ≈ 10 or so.
# Otherwise we just compute many of the elements of Z twice...
- if size(Z,2) < size(Z,1)
- Zrows = ((I - W')\Matrix(sparse(targetnodes,
- 1:length(targetnodes),
- 1.0,
- size(W, 1),
- length(targetnodes))))'
+ if size(Z, 2) < size(Z, 1)
+ B = workspace1 .= B_sparse
+ Zrows = ldiv!(solver, Aadj_init, B; B_copy=copy!(workspace2, B))'
+ Zrows .*= sumqˢ * qᵗ .* diagZⁱ
else
- Zrows = Z
+ Zrows = workspace1 .= Z .* (sumqˢ * qᵗ .* diagZⁱ)
end
- n = size(W,1)
+ qˢZⁱqᵗ = workspace2 .= qˢ .* Zⁱ .* qᵗ'
+ # QZⁱᵀZ = qˢZⁱqᵗ' / A
+ QZⁱᵀZ = ldiv!(solver, Aadj_init, qˢZⁱqᵗ; B_copy=copy!(workspace3, qˢZⁱqᵗ))'
-
- diagZⁱ = [Zⁱ[targetnodes[t], t] for t in 1:length(targetnodes)]
- sumqˢ = sum(qˢ)
-
- Zrows = Zrows .* (sumqˢ*qᵗ.*diagZⁱ)
-
- qˢZⁱqᵗ = qˢ .* Zⁱ .* qᵗ'
-
- QZⁱᵀZ = qˢZⁱqᵗ'/(I - W)
-
- RHS = QZⁱᵀZ-Zrows
-
- edge_betweennesses = copy(W)
+ RHS = workspace3 .= QZⁱᵀZ .- Zrows
for i in axes(W, 1)
# ZᵀZⁱ_minus_diag = Z[:,i]'*qˢZⁱqᵗ .- sumqˢ.* (Z[:,i].*diag(Zⁱ).*qᵗ)'
- for j in findall(W[i,:].>0)
+ for (j, x) in enumerate(view(W, i, :))
+ x > 0 || continue
# edge_betweennesses[i,j] = W[i,j] .* Zqt[j,:]'* (ZᵀZⁱ_minus_diag * Z[j,:])[1]
- edge_betweennesses[i,j] = W[i,j] .* (Z[j,:]' * RHS[:,i])[1]
+ edge_betweennesses[i, j] = W[i, j] .* (view(Z, j, :)'*view(RHS, :, i))[1]
end
end
@@ -128,41 +167,45 @@ function RSP_edge_betweenness_qweighted(W::SparseMatrixCSC,
end
function RSP_edge_betweenness_kweighted(W::SparseMatrixCSC,
- Z::AbstractMatrix,
- qˢ::AbstractVector,
- qᵗ::AbstractVector,
- K::AbstractMatrix, # Matrix of proximities
- targetnodes::AbstractVector)
-
- Zⁱ = inv.(Z)
- Zⁱ[.!isfinite.(Zⁱ)] .= floatmax(eltype(Z)) # To prevent Inf*0 later...
+ Z::AbstractMatrix,
+ qˢ::AbstractVector,
+ qᵗ::AbstractVector,
+ K::AbstractMatrix, # Matrix of proximities
+ targetnodes::AbstractVector;
+ solver=nothing,
+ workspaces=[similar(Z), similar(Z)],
+ permuted_workspaces=(similar(Z'),),
+ Zⁱ=_inv(Z),
+ Aadj=(I - W)',
+ Aadj_init=init(solver, Aadj),
+ B_sparse=sparse_rhs(targetnodes, size(W, 1)),
+ kw...
+)
+ edge_betweennesses = copy(W)
+ workspace1, workspace2 = workspaces
+ permuted_workspace1 = permuted_workspaces[1]
- K̂ = qˢ .* K .* qᵗ'
+ K̂ = K .= qˢ .* K .* qᵗ'
k̂ = vec(sum(K̂, dims=1))
K̂ .*= Zⁱ
+ # K̂ᵀZ = K̂' / A # is equivalent to the below
+ K̂ᵀZ = ldiv!(solver, Aadj_init, K̂; B_copy=copy!(workspace2, K̂))'
- K̂ᵀZ = K̂'/(I - W)
+ k̂diagZⁱ = k̂ .* [Zⁱ[targetnodes[t], t] for t in 1:length(targetnodes)]
- k̂diagZⁱ = k̂.*[Zⁱ[targetnodes[t], t] for t in 1:length(targetnodes)]
-
- Zrows = (I - W')\Matrix(sparse(targetnodes,
- 1:length(targetnodes),
- 1.0,
- size(W, 1),
- length(targetnodes)))
- k̂diagZⁱZ = k̂diagZⁱ .* Zrows'
-
- K̂ᵀZ_minus_diag = K̂ᵀZ - k̂diagZⁱZ
-
- edge_betweennesses = copy(W)
+ B = workspace1 .= B_sparse
+ Zrows = ldiv!(solver, Aadj_init, B; B_copy=copy!(workspace2, B))
+ k̂diagZⁱZ = permuted_workspace1 .= k̂diagZⁱ .* Zrows'
+ K̂ᵀZ_minus_diag = k̂diagZⁱZ .= K̂ᵀZ .- k̂diagZⁱZ
for i in axes(W, 1)
# ZᵀZⁱ_minus_diag = ZᵀKZⁱ[i,:] .- (k.*Z[targetnodes,i].*(Zⁱ[targetnodes,targetnodes]))'
# ZᵀZⁱ_minus_diag = Z[:,i]'*K̂ .- (k.*Z[targetnodes,i].*diag(Zⁱ))'
- for j in findall(W[i,:].>0)
- edge_betweennesses[i,j] = W[i,j] .* (Z[j,:]'*K̂ᵀZ_minus_diag[:,i])[1]
+ for (j, x) in enumerate(view(W, i, :))
+ x > 0 || continue
+ edge_betweennesses[i, j] = W[i, j] .* (view(Z, j, :)'*view(K̂ᵀZ_minus_diag, :, i))[1]
end
end
@@ -170,13 +213,19 @@ function RSP_edge_betweenness_kweighted(W::SparseMatrixCSC,
end
-
-
function RSP_expected_cost(W::SparseMatrixCSC,
- C::SparseMatrixCSC,
- Z::AbstractMatrix,
- landmarks::AbstractVector)
-
+ C::SparseMatrixCSC,
+ Z::AbstractMatrix,
+ landmarks::AbstractVector;
+ solver=nothing,
+ A=(I - W),
+ A_init=init(solver, A),
+ workspaces=[similar(Z), similar(Z)],
+ expected_costs=similar(Z),
+ CW=C .* W,
+ kw...
+)
+ workspace1, workspace2 = workspaces
if axes(W) != axes(C)
throw(DimensionMismatch(""))
end
@@ -184,37 +233,70 @@ function RSP_expected_cost(W::SparseMatrixCSC,
throw(DimensionMismatch(""))
end
if axes(Z, 2) != axes(landmarks, 1)
- Z = Z[:,landmarks]
+ Z = Z[:, landmarks]
end
- if size(Z, 1) == size(Z, 2)
- C̄ = Z*((C .* W)*Z)
- else
- C̄ = (I - W)\((C .* W)*Z)
- end
+
+ # When threaded the solver is faster than a dense matmul
+ # C̄ = if size(Z, 1) == size(Z, 2)
+ # B = mul!(workspace1, C .* W, Z)
+ # mul!(B, C .* W, Z)
+ # This is a dense-dense matmul... very slow
+ # Z * B
+ # else
+ # TODO permuted workspace here for the broadcast
+ B = mul!(workspace1, CW, Z)
+ C̄ = ldiv!(solver, A_init, B; B_copy=copy!(workspace2, B))
+ # end
C̄ ./= Z
# Zeros in Z can cause NaNs in C̄ ./= Z computation but the limit
replace!(C̄, NaN => Inf)
- dˢ = [C̄[landmarks[j], j] for j in axes(Z, 2)]
+ dˢ = view(workspace2, 1, :)
+ # TODO clarify what this does
+
+ for j in axes(Z, 2)
+ dˢ[j] = C̄[landmarks[j], j]
+ end
C̄ .-= dˢ'
- return C̄
+ return copyto!(expected_costs, C̄)
end
-RSP_free_energy_distance(Z::AbstractMatrix, θ::Real, landmarks::AbstractVector) =
- -log.(RSP_survival_probability(Z, θ, landmarks))./θ
+function RSP_free_energy_distance(Z::AbstractMatrix, θ::Real, landmarks::AbstractVector;
+ survival_probability=nothing,
+ free_energy_distances=similar(Z),
+ kw...
+)
+ if isnothing(survival_probability)
+ survival_probability = RSP_survival_probability(Z, θ, landmarks; kw...)
+ end
+ free_energy_distances .= -log.(max.(zero(eltype(Z)), survival_probability)) ./ θ
+
+ return free_energy_distances
+end
-RSP_survival_probability(Z::AbstractMatrix, θ::Real, landmarks::AbstractVector) =
+function RSP_survival_probability(Z::AbstractMatrix, θ::Real, landmarks::AbstractVector; kw...)
Z .* inv.([Z[i, j] for (j, i) in enumerate(landmarks)])'
+end
-RSP_power_mean_proximity(Z::AbstractMatrix, θ::Real, landmarks::AbstractVector) =
- RSP_survival_probability(Z, θ, landmarks).^(1/θ)
+function RSP_power_mean_proximity(Z::AbstractMatrix, θ::Real, landmarks::AbstractVector;
+ survival_probability=nothing, kw...
+)
+ survival_probability = if isnothing(survival_probability)
+ RSP_survival_probability(Z, θ, landmarks; kw...)
+ else
+ survival_probability
+ end
+ survival_probability .^ (1 / θ)
+end
function connected_habitat(qˢ::AbstractVector, # Source qualities
- qᵗ::AbstractVector, # Target qualities
- S::AbstractMatrix) # Matrix of proximities
-
- return qˢ .* (S*qᵗ)
+ qᵗ::AbstractVector, # Target qualities
+ S::AbstractMatrix; # Matrix of proximities
+ workspaces=[similar(S, size(S, 1), 1)],
+ kw...
+)
+ mul!(view(workspaces[1], :, 1), S, qᵗ) .*= qˢ
end
# Returns the directed RSP dissimilarity and directed free energy distance for all nodes to a given target
@@ -255,7 +337,7 @@ function bellman_ford(Pref::SparseMatrixCSC, C::SparseMatrixCSC, θ::Real, targe
iter = 0
trPref = copy(Pref')
- trC = copy(C')
+ trC = copy(C')
while !convergence
φ_1 = copy(φ)
@@ -267,7 +349,7 @@ function bellman_ford(Pref::SparseMatrixCSC, C::SparseMatrixCSC, θ::Real, targe
if updatelist[1] == -1
updatelist = [index]
else
- if rawDistances[node - 1] == rawDistances[node]
+ if rawDistances[node-1] == rawDistances[node]
append!(updatelist, index) # Equidistant nodes should be updated simultaneously
else
c̄, φ = _bellman_ford_update_transposed!(c̄, φ, trPref, trC, θ, updatelist)
@@ -283,28 +365,28 @@ function bellman_ford(Pref::SparseMatrixCSC, C::SparseMatrixCSC, θ::Real, targe
break # Break the loop if in a single pass approach
end
iter += 1
- if iter==1
+ if iter == 1
continue
end
# check if the free energy and the RSP have converged
- convergence=(maximum(abs, φ - φ_1)/maximum(φ) < 1e-8) & (maximum(abs, c̄ - c̄_1)/maximum(c̄) < 1e-8)
+ convergence = (maximum(abs, φ - φ_1) / maximum(φ) < 1e-8) & (maximum(abs, c̄ - c̄_1) / maximum(c̄) < 1e-8)
end
return c̄, φ
end
# Updates the RSP and free energy vectors for a given list of nodes
# Inputs:
- # c̄: the directed expected cost (RSP dissimilarity)
- # φ: the directed free energy
- # trPref: the (transposed) transition probability matrix
- # trC: the (transposed) cost matrix
- # θ: the inverse temperature
- # updatelist: the list of nodes that should be updated simultaneously
+# c̄: the directed expected cost (RSP dissimilarity)
+# φ: the directed free energy
+# trPref: the (transposed) transition probability matrix
+# trC: the (transposed) cost matrix
+# θ: the inverse temperature
+# updatelist: the list of nodes that should be updated simultaneously
# Outputs:
- # c̄: the updated directed expected cost (RSP dissimilarity)
- # φ: the updated directed free energy
+# c̄: the updated directed expected cost (RSP dissimilarity)
+# φ: the updated directed free energy
# Comment:
- # The two sparse arrays in passed in transposed form since it makes the access much more efficient
+# The two sparse arrays in passed in transposed form since it makes the access much more efficient
function _bellman_ford_update_transposed!(c̄::Vector, φ::Vector, trPref::SparseMatrixCSC, trC::SparseMatrixCSC, θ::Real, updatelist::Vector)
if length(updatelist) == 1
index = updatelist[1]
@@ -313,8 +395,8 @@ function _bellman_ford_update_transposed!(c̄::Vector, φ::Vector, trPref::Spars
φ[index] = v
return c̄, φ
end
- prev_φ=copy(φ)
- prev_c̄=copy(c̄)
+ prev_φ = copy(φ)
+ prev_c̄ = copy(c̄)
for i in 1:length(updatelist)
index = updatelist[i]
ec, v = _bellman_ford_update_node_transposed(prev_c̄, prev_φ, trPref, trC, θ, index)
@@ -325,7 +407,7 @@ function _bellman_ford_update_transposed!(c̄::Vector, φ::Vector, trPref::Spars
end
# Helper function required for good performance until https://github.com/JuliaLang/julia/pull/42647 has been released
-function mygetindex(A::SparseMatrixCSC{Tv,Ti}, I::AbstractVector, J::Integer) where {Tv,Ti}
+function fast_getindex(A::SparseMatrixCSC{Tv,Ti}, I::AbstractVector, J::Integer) where {Tv,Ti}
if !issorted(I)
throw(ArgumentError("only sorted indices are currectly supported"))
end
@@ -336,7 +418,7 @@ function mygetindex(A::SparseMatrixCSC{Tv,Ti}, I::AbstractVector, J::Integer) wh
nzval = Tv[]
iI = 1
- for iptr in A.colptr[J]:(A.colptr[J + 1] - 1)
+ for iptr in A.colptr[J]:(A.colptr[J+1]-1)
iA = A.rowval[iptr]
while iI <= nI && I[iI] <= iA
if I[iI] == iA
@@ -351,23 +433,23 @@ end
# Updates the directed RSP and direct free energy value for a given node
# Inputs:
- # c̄: the directed expected cost (RSP dissimilarity)
- # φ: the directed free energy
- # trPref: the (transposed) transition probability matrix
- # trC: the (transposed) cost matrix
- # θ: the inverse temperature
- # index: the index of the node that should be updated
+# c̄: the directed expected cost (RSP dissimilarity)
+# φ: the directed free energy
+# trPref: the (transposed) transition probability matrix
+# trC: the (transposed) cost matrix
+# θ: the inverse temperature
+# index: the index of the node that should be updated
# Outputs:
- # ec: the updated directed expected cost (RSP dissimilarity) for the node
- # v: the updated directed free energy for the node
+# ec: the updated directed expected cost (RSP dissimilarity) for the node
+# v: the updated directed free energy for the node
# Comment:
- # The two sparse arrays in passed in transposed form since it makes the access much more efficient
+# The two sparse arrays in passed in transposed form since it makes the access much more efficient
function _bellman_ford_update_node_transposed(c̄::Vector, φ::Vector, trPref::SparseMatrixCSC, trC::SparseMatrixCSC, θ::Real, index::Integer)
- Prefindex = trPref[:,index]
+ Prefindex = trPref[:, index]
idx = Prefindex.nzind # Get the list of successors
# computation of θ(cᵢⱼ+φ(j,t))-log([Pʳᵉᶠ]ᵢⱼ)
# ect = (Array(trC[idx, index]) + φ[idx]) .* θ .- log.(Prefindex.nzval)
- ect = (Array(mygetindex(trC, idx, index)) .+ φ[idx]) .* θ .- log.(Prefindex.nzval)
+ ect = (Array(fast_getindex(trC, idx, index)) .+ φ[idx]) .* θ .- log.(Prefindex.nzval)
finiteidx = isfinite.(ect)
idx = idx[finiteidx]
ect = ect[finiteidx]
@@ -378,13 +460,13 @@ function _bellman_ford_update_node_transposed(c̄::Vector, φ::Vector, trPref::S
# First check there is only one neighbor, if so, the solution is trivial
if length(idx) == 1
- return c̄[idx[1]] + trC[idx[1], index], ect[1]/θ
+ return c̄[idx[1]] + trC[idx[1], index], ect[1] / θ
end
# log-sum-exp trick
minval = minimum(ect) # computation of cᵢ*
ect .-= minval # remove the lowest value from all the vector
- v = (minval - log(sum(exp, -ect)))/θ # computation of the directed free energy
+ v = (minval - log(sum(exp, -ect))) / θ # computation of the directed free energy
if isinf(v)
throw(ErrorException("infinite valude in the distance vector at index $index"))
end
@@ -393,8 +475,8 @@ function _bellman_ford_update_node_transposed(c̄::Vector, φ::Vector, trPref::S
ec = zero(eltype(c̄))
for j in 1:length(idx)
trCidxjindex = trC[idx[j], index]
- pij = trPref[idx[j], index]*exp(θ*(v - φ[idx[j]] - trCidxjindex))
- ec += pij*(trCidxjindex + c̄[idx[j]])
+ pij = trPref[idx[j], index] * exp(θ * (v - φ[idx[j]] - trCidxjindex))
+ ec += pij * (trCidxjindex + c̄[idx[j]])
end
return ec, v
-end
+end
\ No newline at end of file
diff --git a/src/solvers.jl b/src/solvers.jl
index 239fef3..95b8f4b 100644
--- a/src/solvers.jl
+++ b/src/solvers.jl
@@ -1,32 +1,3 @@
-# Defined in ConScape.jl for load order
-# abstract type Solver end
-@doc """
- Solver
-
-Abstract supertype for ConScape solvers.
-""" Solver
-
-# RSP is not used for ConnectivityMeasure, so the solver isn't used
-function solve(s::Solver, cm::ConnectivityMeasure, p::AbstractProblem, g::Grid)
- return map(p.graph_measures) do gm
- compute(gm, p, g; solver=s)
- end
-end
-
-function solve(s::Solver, cm::FundamentalMeasure, p::AbstractProblem, g::Grid)
- (; A, B, Pref, W) = setup_sparse_problem(g, cm)
- Z = solve_ldiv!(s, A, Matrix(B))
- # Check that values in Z are not too small:
-# _check_z(s, Z, W, g)
-
- # TODO remove use of GridRSP where possible
- grsp = GridRSP(g, cm.θ, Pref, W, Z)
- results = map(p.graph_measures) do gm
- compute(gm, p, grsp)
- end
- return _merge_to_stack(results)
-end
-
"""
MatrixSolver(; check)
@@ -35,62 +6,21 @@ Solve all operations on a fully materialised Z matrix.
This is fast but memory inneficient for CPUS, and isn't threaded.
But may be best for GPUs using CuSSP.jl ?
"""
-@kwdef struct MatrixSolver <: Solver
+@kwdef struct MatrixSolver <: Solver
check::Bool = true
end
-# Fallback generic ldiv solver
-solve_ldiv!(solver, A, B) = ldiv!(lu(A), B)
-
"""
VectorSolver(; check, threaded)
Use julias default solver but broken into columns, with
less memory use and the capacity for threading
"""
-@kwdef struct VectorSolver <: Solver
+@kwdef struct VectorSolver <: Solver
check::Bool = true
threaded::Bool = false
end
-function solve_ldiv!(s::VectorSolver, A, B)
- F = lu(A)
- transposeoptype = SparseArrays.LibSuiteSparse.UMFPACK_A
- # for SparseArrays.UMFPACK._AqldivB_kernel!(Z, F, B, transposeoptype)
-
- # This is basically SparseArrays.UMFPACK._AqldivB_kernel!
- # But we unroll it to avoid copies or allocation of B
- if s.threaded
- # Create a channel to store problem b vectors for threads
- # see https://juliafolds2.github.io/OhMyThreads.jl/stable/literate/tls/tls/
- nbuffers = Threads.nthreads()
- ch = Channel{Tuple{typeof(F),Vector{Float64}}}(nbuffers)
- for i in 1:nbuffers
- # TODO not all of F needs to be duplicated?
- # Can we just copy the workspace arrays and resuse the rest?
- put!(ch, (deepcopy(F), Vector{eltype(A)}(undef, size(B, 1))))
- end
- Threads.@threads for col in 1:size(B, 2)
- # Get a workspace from the channel
- F_t, b_t = take!(ch)
- # Copy a column from B
- b_t .= view(B, :, col)
- # Solve for the column
- SparseArrays.UMFPACK.solve!(view(B, :, col), F_t, b_t, transposeoptype)
- # Reuse the workspace
- put!(ch, (F_t, b_t))
- end
- else
- b = zeros(eltype(B), size(B, 1))
- for col in 1:size(B, 2)
- b .= view(B, :, col)
- SparseArrays.UMFPACK.solve!(view(B, :, col), F, b, transposeoptype)
- end
- end
-
- return B
-end
-
"""
LinearSolver(args...; threded, kw...)
@@ -112,108 +42,478 @@ TODO: an example that is realistic
````julia
using LinearSolve
+distance_transformation = (exp=x -> exp(-x/75), oddsfor=ConScape.OddsFor()),
problem = ConScape.Problem(;
solver = LinearSolver(KrylovJL_GMRES(precs = (A, p) -> (Diagonal(A), I)))
graph_measures = (;
func=ConScape.ConnectedHabitat(),
qbetw=ConScape.BetweennessQweighted(),
),
- distance_transformation = (exp=x -> exp(-x/75), oddsfor=ConScape.OddsFor()),
connectivity_measure = ConScape.ExpectedCost(θ=1.0),
)
````
"""
-struct LinearSolver <: Solver
- args
- keywords
+struct LinearSolver{A,K} <: Solver
+ args::A
+ keywords::K
threaded::Bool
end
LinearSolver(args...; threaded=false, kw...) = LinearSolver(args, kw, threaded)
-function solve_ldiv!(s::LinearSolver, A, B)
- b = zeros(eltype(A), size(B, 1))
- # Define and initialise the linear problem
- linprob = LinearProblem(A, b)
- linsolve = init(linprob, s.args...; s.keywords...)
- # TODO: for now we define a Z matrix, but later modify ops
- # to run column by column without materialising Z
- # if s.threaded
- # nbuffers = Threads.nthreads()
- # # Create a channel to store problem b vectors for threads
- # # see https://juliafolds2.github.io/OhMyThreads.jl/stable/literate/tls/tls/
- # ch = Channel{Tuple{typeof(linsolve),Vector{Float64}}}(nbuffers)
- # for i in 1:nbuffers
- # # TODO fix this in LinearSolve.jl with batching
- # # We should not need to `deepcopy` the whole problem we
- # # just need to replicate the specific workspace arrays
- # # that will cause race conditions.
- # # But currently there is no parallel mode for LinearSolve.jl
- # # See https://github.com/SciML/LinearSolve.jl/issues/552
- # put!(ch, (deepcopy(linsolve), Vector{eltype(A)}(undef, size(B, 1))))
- # end
- # Threads.@threads for i in 1:size(B, 2)
- # # Get column memory from the channel
- # linsolve_t, b_t = take!(ch)
- # # Update it
- # b_t .= view(B, :, i)
- # # Update solver with new b values
- # reinit!(linsolve_t; b=b_t, reuse_precs=true)
- # sol = LinearSolve.solve(linsolve_t, s.args...; s.keywords...)
- # # Aim for something like this ?
- # # res = map(connectivity_measures(p)) do cm
- # # compute(cm, g, sol.u, i)
- # # end
- # # For now just use Z
- # B[:, i] .= sol.u
- # put!(ch, (linsolve_t, b_t))
- # end
- # else
- for i in 1:size(B, 2)
- b .= view(B, :, i)
- reinit!(linsolve; b, reuse_precs=true)
- sol = LinearSolve.solve(linsolve, s.args...; s.keywords...)
- # Udate the column
- B[:, i] .= sol.u
+# In `init!` we allocate all large dense arrays
+function init!(
+ ws::NamedTuple,
+ solver::Solver,
+ cm::FundamentalMeasure,
+ p::AbstractProblem,
+ rast::RasterStack;
+ verbose=false,
+)
+ # Initialise the whole grid
+ grid = Grid(p, rast; prune=false)
+ # Initialise the workspace
+ workspace = _init_dense!(ws, solver, cm, p, grid; verbose)
+ if isthreaded(solver)
+ nbuffers = Thread.nthreads()
+ channel = Channel{typeof(workspace)}(nbuffers)
+ put!(channel, workspace)
+ for n in 2:nbuffers
+ workspace_n = _init_dense!(ws, solver, cm, p, grid; verbose)
+ put!(channel, workspace_n)
end
- # end
- @info "LinearSolver finished"
- return B
+ return (; channel)
+ else
+ return workspace
+ end
end
+# _init_dense! may be called multiple times from `init!`, for each thread
+function _init_dense!(
+ ws::NamedTuple,
+ solver::Solver,
+ cm::FundamentalMeasure,
+ p::AbstractProblem,
+ grid::Grid;
+ verbose=false,
+ reuse_output=false,
+)
+ verbose && println("Retreiving measures...")
+ gms = graph_measures(p)
+ cf = connectivity_function(p)
+ verbose && println("Defining sparse arrays...")
+ # B_dense becomes Z
+ verbose && println("Allocating workspaces...")
+ sze = _workspace_size(solver, g)
+ Z = if hastrait(needs_inv, gms)
+ if haskey(ws, :Z)
+ _reshape(ws.Z, sze)
+ else
+ Matrix{eltype(g.affinities)}(undef, sze)
+ end
+ else
+ nothing
+ end
+ Zⁱ = if hastrait(needs_inv, gms)
+ haskey(ws, :Zⁱ) ? _reshape(ws.Zⁱ, sze) : similar(Z)
+ else
+ nothing
+ end
+ n_workspaces = count_workspaces(p)
+ n_permuted_workspaces = count_permuted_workspaces(p)
+ workspaces = if haskey(ws, :workspaces)
+ [_reshape(w, size(Z)) for w in ws.workspaces]
+ else
+ [similar(Z) for _ in 1:n_workspaces]
+ end
+ permuted_workspaces = if haskey(ws, :workspaces)
+ [_reshape(pw, size(Z')) for pw in ws.permuted_workspaces]
+ else
+ [similar(Z') for _ in 1:n_permuted_workspaces]
+ end
+ # TODO these shouldn't have traits, it
+ # should be baked into the problem.
+ expected_costs = if hastrait(needs_expected_cost, gms) || cf == ConScape.expected_cost
+ haskey(ws, :expected_costs) ? _reshape(ws.expected_costs, size(Z)) : similar(Z)
+ else
+ nothing
+ end
+
+ free_energy_distances = if hastrait(needs_free_energy_distance, gms) || cf == ConScape.free_energy_distance
+ haskey(ws, :free_energy_distances) ? _reshape(ws.free_energy_distances, size(Z)) : similar(Z)
+ else
+ nothing
+ end
+ proximities = if hastrait(needs_proximity, gms)
+ haskey(ws, :proximities) ? _reshape(ws.proximities, size(Z)) : similar(Z)
+ else
+ nothing
+ end
+ function matrix_or_nothing(gm)
+ if returntype(gm) isa ReturnsDenseSpatial
+ A = fill(NaN, size(grid))
+ A[grid.id_to_grid_coordinate_list] .= 0.0
+ A
+ else
+ nothing
+ end
+ end
+ # We don't re-use outputs
+ outputs = if reuse_output && haskey(ws, :outputs)
+ ws.outputs
+ else
+ if distance_transformation(cm) isa NamedTuple
+ map(gms) do gm
+ if needs_connectivity(gm)
+ map(distance_transformation(cm)) do dt
+ matrix_or_nothing(gm)
+ end
+ else
+ matrix_or_nothing(gm)
+ end
+ end
+ else
+ map(gms) do gm
+ matrix_or_nothing(gm)
+ end
+ end
+ end
-# Utils
+ verbose && println("Finished allocating...")
+
+ return (; Z, Zⁱ, workspaces, permuted_workspaces, free_energy_distances, expected_costs, proximities, outputs, grid, subgrids)
+end
+# function init!(
+# workspace::NamedTuple, s::Solver, cm::ConnectivityMeasure, p::AbstractProblem, rast::RasterStack;
+# verbose=false,
+# grid=Grid(p, rast),
+# )
+# # TODO what is needed here?
+# return (; grid)
+# end
+
+# RSP is not used for ConnectivityMeasure, so the solver isn't used
+function solve!(
+ workspace::NamedTuple,
+ s::MatrixSolver,
+ cm::ConnectivityMeasure,
+ p::AbstractProblem;
+ verbose=false
+)
+ g = workspace.g
+ return map(graph_measures(p), workspace.outputs) do gm, output
+ compute(gm, p; workspace..., output, verbose)
+ end
+end
+function solve!(
+ ws::NamedTuple,
+ solver::MatrixSolver,
+ cm::FundamentalMeasure,
+ p::Problem;
+ verbose=false,
+)
+ # Loop over unnconnected subgrids
+ sg1 = first(ws.subgrids)
+ ws1 = _init_dense!(ws, solver, cm, p, sg1; verbose)
+ for subgrid in ws.subgrids
+ ws2 = _init_dense!(ws1, solver, cm, p, subgrid; verbose, reuse_output=true)
+ ws3 = _init_sparse(ws2, solver, cm, p, subgrid; verbose)
+ ws4 = _solve_dense!(ws3, solver, cm, p; verbose)
+ gms = graph_measures(p)
+ _solve!(ws4, solver, cm, cm.distance_transformation, gms, p; verbose)
+ end
+ @show typeof(ws1.outputs)
+ return _merge_to_stack(_maybe_raster(ws1.outputs, sg1))
+end
+function solve!(
+ ws::NamedTuple,
+ solver::Union{VectorSolver,LinearSolver},
+ cm,
+ p::Problem;
+ verbose=false,
+)
+ sg1 = first(ws.subgrids)
+ gms = graph_measures(p)
+ # Predefine min-vectors targets (not worth putting in the workspace) ?
+ targetnodes = sg1.targetnodes[1:1]
+ targetidx = sg1.targetidx[1:1]
+ qt = sg1.qt[1:1]
+ target_allocs = (; targetidx, targetnodes, qt)
+ target_grid = ConstructionBase.setproperties(sg1, target_allocs)
+ # Allocate dense arrays at the single target size
+ ws1 = _init_dense!(ws, solver, cm, p, target_grid; verbose)
+
+ # Internally we solve one target at a time, for each prefactorized subgrid
+ function solve_target!(workspace, subgrid, i)
+ target_grid = ConstructionBase.setproperties(workspace.grid, target_allocs)
+ _update_targets!(target_allocs, subgrid, i)
+ # And rebuild the workspace with the new grid
+ target_ws = (; workspace..., g=target_grid, grid=target_grid)
+ target_ws1 = _solve_dense!(target_ws, solver, cm, p; verbose)
+ _solve!(target_ws1, solver, cm, cm.distance_transformation, gms, p; verbose)
+ end
+
+ # Loop over unnconnected subgrids (there may be only one)
+ for subgrid in ws.subgrids
+ # Intitalise sparse matrices and precalculate e.g. LU factorizations
+ ws2 = _init_sparse(ws1, solver, cm, p, subgrid; verbose)
+ if isthreaded(solver)
+ isthreaded(p) && error("threading at solver level not yet implemented")
+ # Threads.@threads for i in eachindex(g.targetnodes)[2:end]
+ # run(i)
+ # end
+ else
+ # Then solve each target as a single right hand side column
+ for i in eachindex(subgrid.targetnodes)
+ solve_target!(ws2, subgrid, i)
+ end
+ end
+ end
+ return _merge_to_stack(_maybe_raster(ws1.outputs, ws.grid))
+end
-function setup_sparse_problem(g::Grid, cm::FundamentalMeasure)
- Pref = _Pref(g.affinities)
- W = _W(Pref, cm.θ, g.costmatrix)
+function _solve!(workspace, solver, cm, dt::NamedTuple{DT}, gms::NamedTuple{GMS}, p; verbose) where {DT,GMS}
+ (; grid, Pref, W, Z, outputs) = workspace
+ # GridRSP is just a wrapper now, we can remove it later
+ grsp = GridRSP(grid, cm.θ, Pref, W, Z)
+ # Map over both distance transformations and graph measures
+ nested = map(values(dt), DT) do dt, k
+ cm1 = ConstructionBase.setproperties(cm, (; distance_transformation=dt))
+ hastrait(needs_proximity, gms) &&
+ _setproximities!(workspace.proximities, workspace.expected_costs, cm1, p, grsp)
+ # Rebuild the problem with a connectivity measure
+ # holding a single distance transformation, in case its used
+ p1 = ConstructionBase.setproperties(p, (; connectivity_measure=cm1))
+ map(gms, outputs) do gm, os
+ if needs_connectivity(gm)
+ compute(gm, p1, grsp; workspace..., output=os[k])
+ else
+ nothing
+ end
+ end
+ end |> NamedTuple{DT}
+ # Map over graph measures that don't need connectivity
+ flat = map(gms, outputs) do gm, output
+ if needs_connectivity(gm)
+ nothing
+ else
+ compute(gm, p, grsp; workspace..., output)
+ end
+ end
+ # Combine nested and flat results
+ return map(GMS) do k
+ f = flat[k]
+ if isnothing(f)
+ map(n -> n[k], nested)
+ else
+ f
+ end
+ end |> NamedTuple{GMS}
+end
+function _solve!(workspace, solver, cm, dt, gms::NamedTuple{GMS}, p; verbose) where {GMS}
+ (; grid, Pref, W, Z, outputs) = workspace
+ # GridRSP is just a wrapper now, we can remove it later
+ grsp = GridRSP(grid, cm.θ, Pref, W, Z)
+ hastrait(needs_proximity, gms) &&
+ _setproximities!(workspace.proximities, workspace.expected_costs, cm, p, grsp)
+ # Map over graph measures
+ map(p.graph_measures, outputs) do gm, output
+ compute(gm, p, grsp; workspace..., output)
+ end
+end
+
+function _update_targets!(a, g, i)
+ # target_qualities[:, 1] = g.target_qualities[i]
+ a.targetidx[1] = g.targetidx[i]
+ a.targetnodes[1] = g.targetnodes[i]
+ a.qt[1] = g.qt[i]
+ return nothing
+end
+
+# Do all the work shared accross outputs
+function _solve_dense!(ws::NamedTuple, solver::Solver, cm, p::Problem;
+ verbose=false
+)
+ (; grid, W, Pref, A, A_init, Aadj_init, Aadj) = ws
+ gms = graph_measures(p)
+ cf = connectivity_function(p)
+ # Sparse rhs
+ # TODO get rid of this allocation
+ # For VectorSolver we can write values directly to B
+ B_sparse = sparse_rhs(grid.targetnodes, size(grid.costmatrix, 1))
+ Z = if hastrait(needs_Z, gms)
+ # verbose &&
+ B = _reshape(ws.Z, size(B_sparse))
+ copyto!(B, B_sparse)
+ verbose && println("Solving Z matrix...")
+ # Check that values in Z are not too small:
+
+ Z = ldiv!(solver, A_init, B; B_copy=copyto!(ws.workspaces[1], B))
+ # verbose && _check_z(s, Z, W, g)
+ Z
+ end
+ Zⁱ = if hastrait(needs_inv, gms)
+ verbose && println("Inverting Z...")
+ _inv!(_reshape(ws.Zⁱ, size(Z)), Z)
+ end
+
+ grsp = GridRSP(grid, cm.θ, Pref, W, Z)
+ workspace = (; ws..., Pref, W, A, A_init, Aadj, Aadj_init, Z, Zⁱ)
+
+ expected_costs = if hastrait(needs_expected_cost, gms) || cf == ConScape.expected_cost
+ verbose && println("Calculating expected cost...")
+ expected_costs = _reshape(ws.expected_costs, size(Z))
+ ConScape.expected_cost(grsp; workspace..., expected_costs, solver)
+ end
+ free_energy_distances = if hastrait(needs_free_energy_distance, gms) || cf == ConScape.free_energy_distance
+ verbose && println("Calculating free energy distance...")
+ free_energy_distances = _reshape(ws.free_energy_distances, size(Z))
+ ConScape.free_energy_distance(grsp; workspace..., free_energy_distances, solver)
+ end
+
+ return merge(ws, (; Pref, W, A, A_init, Aadj, Aadj_init, Z, Zⁱ, expected_costs, free_energy_distances))
+end
+
+function _init_sparse(ws::NamedTuple, solver, cm, p::Problem, grid::Grid; verbose)
+ gms = graph_measures(p)
+ verbose && println("Initialising sparse factorizations...")
+ Pref = _Pref(grid.affinities)
+ W = _W(Pref, cm.θ, grid.costmatrix)
# Sparse lhs
A = I - W
- # Sparse rhs
- B = sparse_rhs(g.targetnodes, size(g.costmatrix, 1))
- return (; A, B, Pref, W)
+ A_init = init(solver, A)
+ Aadj, Aadj_init = if hastrait(needs_adjoint_init, gms)
+ # Just take the adjoint of the factorization of A
+ # where possible to save calculations and memory
+ if hasproperty(A_init, :F)
+ Aadj = A'
+ # Use adjoint factorization of A rather than recalculating for A'
+ Aadj_init = merge(A_init, (; F=A_init.F'))
+ Aadj, Aadj_init
+ else
+ # LinearSolve.jl cant handle the adjoint
+ # so we duplicate work and allocations
+ Aadj = sparse(A')
+ Aadj_init = init(solver, Aadj)
+ Aadj, Aadj_init
+ end
+ else
+ nothing, nothing
+ end
+
+ CW = if hastrait(needs_expected_cost, gms) || connectivity_function(p) == ConScape.expected_cost
+ grid.costmatrix .* W
+ else
+ nothing
+ end
+
+ return merge(ws, (; W, Pref, A, A_init, Aadj_init, Aadj, CW))
+end
+
+# All targets at once
+_workspace_size(::MatrixSolver, g) = target_size(g)
+# One target at a time
+_workspace_size(::Union{VectorSolver,LinearSolver}, g) = first(target_size(g)), 1
+
+isthreaded(s::Solver) = false
+isthreaded(s::LinearSolver) = s.threaded
+isthreaded(s::VectorSolver) = s.threaded
+
+# Solver init
+init(::Union{Nothing,MatrixSolver,VectorSolver}, A::AbstractMatrix) = (; F=lu(A))
+function init(solver::VectorSolver, A::AbstractMatrix)
+ F = lu(A)
+ if isthreaded(solver)
+ nbuffers = Threads.nthreads()
+ channel = Channel{typeof(F)}(nbuffers)
+ for _ in 1:nbuffers
+ put!(channel, copy(F))
+ end
+ return channel
+ else
+ return F
+ end
+end
+function init(solver::LinearSolver, A::AbstractMatrix)
+ b = zeros(eltype(A), size(A, 2))
+ # Define and initialise the linear problem
+ linprob = LinearProblem(A, b)
+ linsolve = init(linprob, solver.args...; solver.keywords...)
+ # TODO what is needed here?
+ # Create a channel to store problem b vectors for threads
+ # see https://juliafolds2.github.io/OhMyThreads.jl/stable/literate/tls/tls/
+ if isthreaded(solver)
+ nbuffers = Threads.nthreads()
+ channel = Channel{Tuple{typeof(linsolve),Vector{Float64}}}(nbuffers)
+ for i in 1:nbuffers
+ # TODO fix this in LinearSolve.jl with batching
+ # We should not need to `deepcopy` the whole problem we
+ # just need to replicate the specific workspace arrays
+ # that will cause race conditions.
+ # But currently there is no parallel mode for LinearSolve.jl
+ # See https://github.com/SciML/LinearSolve.jl/issues/552
+ put!(channel, (deepcopy(linsolve), Vector{eltype(A)}(undef, size(A, 2))))
+ end
+ return channel
+ else
+ return linsolve
+ end
end
+function LinearAlgebra.ldiv!(s::LinearSolver, init, B; B_copy)
+ # TODO: for now we define a Z matrix, but later modify ops
+ # to run column by column without materialising Z
+ if isthreaded(s)
+ channel = init
+ # Get column memory from the channel
+ linsolve = take!(channel)
+ # Update solver with new b values
+ reinit!(linsolve; b=vec(B_copy), reuse_precs=true)
+ sol = LinearSolve.solve!(vec(B), linsolve, s.args...; s.keywords...)
+ vec(B) .= sol.u
+ put!(channel, linsolve)
+ else
+ linsolve = init
+ reinit!(linsolve; b=vec(B_copy), reuse_precs=true)
+ sol = LinearSolve.solve(linsolve, s.args...; s.keywords...)
+ vec(B) .= sol.u
+ end
+ return B
+end
+LinearAlgebra.ldiv!(::Union{MatrixSolver,Nothing}, (; F), B; B_copy=copy(B)) =
+ ldiv!(B, F, B_copy)
+# LinearAlgebra.ldiv!(solver::Solver, A::AbstractMatrix, B::AbstractMatrix; kw...) =
+# ldiv!(solver, init(solver, A), B; kw...)
+function LinearAlgebra.ldiv!(s::VectorSolver, init, B; B_copy)
+ # for SparseArrays.UMFPACK._AqldivB_kernel!(Z, F, B, transposeoptype)
+ transposeoptype = SparseArrays.LibSuiteSparse.UMFPACK_A
+
+ # This is basically SparseArrays.UMFPACK._AqldivB_kernel!
+ # But we unroll it to avoid copies or allocation of B
+ if isthreaded(s)
+ channel = init
+ F = take!(channel)
+ # Solve for the column
+ SparseArrays.UMFPACK.solve!(vec(B), F, vec(B_copy), transposeoptype)
+ # Reuse the workspace
+ put!(channel, F)
+ else
+ F = init
+ SparseArrays.UMFPACK.solve!(vec(B), F, vec(B_copy), transposeoptype)
+ end
+ return B
+end
+# Utils
+
# We may have multiple distance_measures per
# graph_measure, but we want a single RasterStack.
# So we merge the names of the two layers
-function _merge_to_stack(nt::NamedTuple{K}) where K
+function _merge_to_stack(nt::NamedTuple{K}) where {K}
unique_nts = map(K) do k
- gm = nt[k]
- if gm isa NamedTuple
- # Combine outer and inner names with an underscore
- joinedkeys = map(keys(gm)) do k_inner
- Symbol(k, :_, k_inner)
- end
- # And rename the NamedTuple
- NamedTuple{joinedkeys}(map(_maybe_raster, values(gm)))
- else
- # We keep the name as is
- NamedTuple{(k,)}((_maybe_raster(gm),))
- end
+ _mergename(Val{k}(), nt[k])
end
- # merge unique layers into a sinlge RasterStack
+ # merge unique layers into a single RasterStack
nt = merge(unique_nts...)
if all(map(x -> x isa Raster, nt))
return RasterStack(nt)
@@ -221,13 +521,57 @@ function _merge_to_stack(nt::NamedTuple{K}) where K
return nt # Cant return a RasterStack for these outputs
end
end
-_maybe_raster(x::Raster) = x
-_maybe_raster(x::Number) = Raster(fill(x), ())
-_maybe_raster(x) = x
+
+function _mergename(::Val{K1}, gm::NamedTuple{K2}) where {K1,K2}
+ # Combine outer and inner names with an underscore
+ joinedkeys = map(K2) do k2
+ Symbol(K1, :_, k2)
+ end
+ # And rename the NamedTuple
+ NamedTuple{joinedkeys}(map(_maybe_raster, values(gm)))
+end
+_mergename(::Val{K1}, gm) where {K1} =
+# We keep the name as is
+ NamedTuple{(K1,)}((_maybe_raster(gm),))
function _check_z(s, Z, W, g)
# Check that values in Z are not too small:
- if s.check && minimum(Z) * minimum(nonzeros(g.costmatrix .* W)) == 0
+ if hasproperty(s, :check) && s.check && minimum(Z) * minimum(nonzeros(g.costmatrix .* W)) == 0
@warn "Warning: Z-matrix contains too small values, which can lead to inaccurate results! Check that the graph is connected or try decreasing θ."
end
+end
+
+# This duplicats some logic from gridrsp
+function _setproximities!(
+ proximities::AbstractMatrix,
+ expected_costs::AbstractMatrix,
+ cm::ConnectivityMeasure,
+ p::Problem,
+ grsp::GridRSP
+)
+ g = grsp.g
+ dt = cm.distance_transformation
+ if isnothing(dt)
+ map!(inv(g.costfunction), proximities, expected_costs)
+ else
+ map!(dt, proximities, expected_costs)
+ end
+ maybe_set_diagonal!(proximities, diagvalue(p), g.targetnodes)
+ return proximities
+end
+
+# This only makes sense if arrays are sorted large to small
+function _reshape(A::Array, dims::Tuple{Vararg{Int}})
+ len = prod(dims)
+ if size(A) == dims
+ A
+ else # if length(A) >= len
+ # TODO make sure this doesn't allocate when the array is larger
+ # We may need julia 1.11 to do this properly
+ v = vec(A)
+ resize!(v, len)
+ reshape(v, dims)
+ # else
+ # error("Arrays were not sorted. Current len: $(length(A)), needed len: $len")
+ end
end
\ No newline at end of file
diff --git a/src/tiles.jl b/src/tiles.jl
deleted file mode 100644
index 24b33b7..0000000
--- a/src/tiles.jl
+++ /dev/null
@@ -1,189 +0,0 @@
-# This file is a work in progress...
-
-"""
- WindowedProblem(problem::AbstractProblem; size, centers, θ)
-
-Combine multiple compute operations into a single object,
-to be run over the same windowed grids.
-
-`problem` is usually a [`Problem`](@ref) object but can be any `AbstractProblem`.
-
-# Keywords
-
-- `problem`: The radius of the window.
-- `radius`: The radius of the window.
-- `overlap`: The overlap between windows.
-- `threaded`: Whether to run in parallel. `false` by default
-"""
-@kwdef struct WindowedProblem <: AbstractProblem
- problem::AbstractProblem
- radius::Int
- overlap::Int
- threaded::Bool = false
-end
-WindowedProblem(problem; kw...) = WindowedProblem(; problem, kw...)
-
-function solve(wp::WindowedProblem, rast::RasterStack)
- ranges = collect(_get_window_ranges(wp, rast))
- mask = _get_window_mask(rast, ranges)
- p = wp.problem
- output_stacks = Vector{RasterStack}(undef, count(mask))
- used_ranges = ranges[mask]
- if wp.threaded
- Threads.@threads for i in eachindex(used_ranges)
- output_stacks[i] = solve(p, rast[used_ranges[i]...])
- end
- else
- for i in eachindex(used_ranges)
- output_stacks[i] = solve(p, rast[used_ranges[i]...])
- end
- end
- # Return mosaics of outputs
- return Rasters.mosaic(sum, output_stacks; to=rast, missingval=NaN)
-end
-
-# function assess(op::WindowedProblem, g::Grid)
-# window_assessments = map(_windows(op, g)) do w
-# ca = assess(op.op, w)
-# end
-# maximums = reduce(window_assessments) do acc, a
-# (; totalmem=max(acc.totalmem, a.totalmem),
-# zmax=max(acc.zmax, a.zmax),
-# lumax=max(acc.lumax, a.lumax),
-# )
-# end
-# ComputeAssesment(; op=op.op, maximums..., sums...)
-# end
-
-
-"""
- StoredProblem(problem::AbstractProblem; radius, overlap, path, ext)
-
-Combine multiple compute operations into a single object,
-when compute times are long and intermediate storage is needed.
-
-`problem` is usually a [`Problem`](@ref) object or a `WindowedProblem`
-for nested operations.
-
-# Keywords
-
-- `radius`: The radius of the window - 2radius + 1 is the diameter.
-- `overlap`: The overlap between adjacent windows.
-- `path`: The path to store the output rasters.
-- `ext`: The file extension for Rasters.jl to write to. Defaults to `.tif`,
- But can be `.nc` for NetCDF or most other common extensions.
-- `threaded`: Whether to run in parallel. `false` by default
-"""
-@kwdef struct StoredProblem
- problem::AbstractProblem
- radius::Int
- overlap::Int
- path::String
- ext::String = ".tif"
- threaded::Bool = false
-end
-StoredProblem(problem; kw...) = StoredProblem(; problem, kw...)
-
-function solve(sp::StoredProblem, rast::RasterStack)
- ranges = collect(_get_window_ranges(sp, rast))
- mask = _get_window_mask(rast, ranges)
- if sp.threaded
- Threads.@threads for rs in ranges[mask]
- output = solve(sp.problem, rast[rs...])
- _store(sp, output, rs)
- end
- else
- for rs in ranges[mask]
- output = solve(sp.problem, rast[rs...])
- _store(sp, output, rs)
- end
- end
-end
-# Single batch job for clusters
-function solve(sp::StoredProblem, rast::RasterStack, i::Int)
- ranges = collect(_get_window_ranges(sp, rast))
- rs = ranges[i]
- output = solve(sp.problem, rast[rs...])
- _store(sp, output, rs)
-end
-
-"""
- batch_ids(sp::StoredProblem, rast::RasterStack)
-
-Return the batch indices of the windows that need to be computed.
-
-Returns a `Vector{Int}`
-"""
-function batch_ids(sp::StoredProblem, rast::RasterStack)
- ranges = _get_window_ranges(sp, rast)
- mask = _get_window_mask(rast, ranges)
- return eachindex(mask)[vec(mask)]
-end
-
-# Mosaic the stored files to a RasterStack
-function Rasters.mosaic(sp::StoredProblem;
- to, lazy=false, filename=nothing, missingval=NaN, kw...
-)
- ranges = _get_window_ranges(sp, to)
- mask = _get_window_mask(to, ranges)
- paths = [_window_path(sp, rs) for (rs, m) in zip(ranges, mask) if m]
- stacks = [RasterStack(p; lazy, name) for p in paths if isdir(p)]
-
- return Rasters.mosaic(sum, stacks; to, filename, missingval, kw...)
-end
-
-function _store(p::StoredProblem, output::RasterStack{K}, ranges) where K
- path = mkpath(_window_path(p, ranges))
- return Rasters.write(joinpath(path, ""), output;
- ext=p.ext, verbose=false, force=true
- )
-end
-
-function _window_path(p, ranges)
- corners = map(first, ranges)
- window_dirname = "window_" * join(corners, '_')
- return joinpath(p.path, window_dirname)
-end
-
-
-### Shared utilities
-
-# Generate a new mask if nested
-_initialise(p::Problem, target) = p
-function _initialise(p::WindowedProblem, target)
- WindowedProblem(p.problem, p.ranges, mask)
-end
-function _initialise(p::StoredProblem, target)
- mask = _get_window_mask(target, p.ranges)
- StoredProblem(p.problem, p.ranges, mask, p.path)
-end
-
-_get_window_ranges(p::Union{StoredProblem,WindowedProblem}, rast::AbstractRasterStack) =
- _get_window_ranges(size(rast), p.radius, p.overlap)
-function _get_window_ranges(size::Tuple{Int,Int}, r::Int, overlap::Int)
- d = 2r
- d <= overlap && throw(ArgumentError("2radius must be larger than overlap"))
- s = d - overlap # Step between each window corner
- # Define the corners of each window
- corners = CartesianIndices(size)[begin:s:end, begin:s:end]
- # Create an iterator of ranges for retreiving each window
- return (map((i, sz) -> i:min(sz, i + d), Tuple(c), size) for c in corners)
-end
-
-_get_window_mask(::Nothing, ranges) = nothing
-_get_window_mask(rast::AbstractRasterStack, ranges) =
- _get_window_mask(_get_target(rast), ranges)
-function _get_window_mask(target::AbstractRaster, ranges)
- # Create a mask to skip tiles that have no target cells
- map(r -> _has_values(target, r), ranges)
-end
-
-function _has_values(target::AbstractRaster, rs::Tuple{Vararg{AbstractUnitRange}})
- # Get a window view
- window = view(target, rs...)
- # If there are non-NaN cells above zero, keep the window
- # TODO allow users to change this condition?
- any(x -> !isnan(x) && x > zero(x), window)
-end
-
-_resolution(rast) = abs(step(lookup(rast, X)))
\ No newline at end of file
diff --git a/src/transformations.jl b/src/transformations.jl
new file mode 100644
index 0000000..912f8b1
--- /dev/null
+++ b/src/transformations.jl
@@ -0,0 +1,35 @@
+
+"""
+ Transformation
+
+Abstrct supertype for distance transformation functions.
+"""
+abstract type Transformation end
+
+struct MinusLog <: Transformation end
+struct ExpMinus <: Transformation end
+struct Inv <: Transformation end
+struct OddsAgainst <: Transformation end
+struct OddsFor <: Transformation end
+struct ExpMinusAlpha{T} <: Transformation
+ alpha::T
+end
+struct MinusLogAlpha{T} <: Transformation
+ alpha::T
+end
+
+(::MinusLog)(x::Number) = -log(x)
+(::ExpMinus)(x::Number) = exp(-x)
+(::Inv)(x::Number) = inv(x)
+(::OddsAgainst)(x::Number) = inv(x) - 0
+(::OddsFor)(x::Number) = x / (0 - x)
+(t::ExpMinusAlpha)(x::Number) = exp(-x / t.alpha)
+# (t::MinusLogAlpha)(x::Number) = -log(x * t.alpha) TODO: what is the inverse of ExpMinusAlpha
+
+Base.inv(::MinusLog) = ExpMinus()
+Base.inv(::ExpMinus) = MinusLog()
+Base.inv(::Inv) = Inv()
+Base.inv(::OddsAgainst) = OddsFor()
+Base.inv(::OddsFor) = OddsAgainst()
+Base.inv(t::MinusLogAlpha) = ExpMinus(t.alpha)
+Base.inv(t::ExpMinusAlpha) = MinusLog(t.alpha)
diff --git a/src/utils.jl b/src/utils.jl
index d627c3f..46c3467 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -144,7 +144,9 @@ function graph_matrix_from_raster(
end
end
end
- return sparse(is, js, vs, m*n, m*n)
+ # TODO just make a BandedMatrix from the start
+ # and what happens when this is not square?
+ return (sparse(is, js, vs, m*n, m*n))
end
@@ -188,9 +190,26 @@ function mapnz(f, A::SparseMatrixCSC)
map!(f, B.nzval, A.nzval)
return B
end
+function mapnz(f, A::AbstractArray)
+ B = copy(A)
+ map!(f, B.data, A.data)
+ return B
+end
# Helper to get keyword arguments
function _keywords(o::T) where T
vals = map(f -> getfield(o, f), fieldnames(T))
return NamedTuple{fieldnames(T)}(vals)
-end
\ No newline at end of file
+end
+
+_maybe_raster(x) = x
+_maybe_raster(x::Raster) = x
+_maybe_raster(x::T) where T<:Number = Raster(fill(x), (); missingval=T(NaN))
+_maybe_raster(mat::Raster, g) = mat
+_maybe_raster(mat::AbstractMatrix, g::Union{Grid,GridRSP}) =
+ _maybe_raster(mat, dims(g))
+_maybe_raster(mats::NamedTuple, g::Union{Grid,GridRSP}) =
+ map(mat -> _maybe_raster(mat, g), mats)
+_maybe_raster(mat::AbstractMatrix, ::Nothing) = mat
+_maybe_raster(mat::AbstractMatrix{T}, dims::Tuple) where T =
+ Raster(mat, dims; missingval=T(NaN))
\ No newline at end of file
diff --git a/src/windows.jl b/src/windows.jl
new file mode 100644
index 0000000..432757c
--- /dev/null
+++ b/src/windows.jl
@@ -0,0 +1,504 @@
+# This file is a work in progress...
+abstract type AbstractWindowedProblem{P} <: AbstractProblem end
+
+costs(p::AbstractWindowedProblem) = costs(p.problem)
+prune(p::AbstractWindowedProblem) = prune(p.problem)
+buffer(p::AbstractWindowedProblem) = p.buffer
+buffer(p::AbstractProblem) = 0
+grain(::AbstractProblem) = nothing
+
+"""
+ WindowedProblem(problem::AbstractProblem; size, centers, θ)
+
+Combine multiple compute operations into a single object,
+to be run over the same windowed grids.
+
+`problem` is usually a [`Problem`](@ref) object but can be any `AbstractProblem`.
+
+# Keywords
+
+- `problem`: The radius of the window.
+- `centersize`: the size of the target square.
+- `buffer`: the area outside the source window.
+- `threaded`: Whether to run in parallel. `false` by default
+"""
+@kwdef struct WindowedProblem{P} <: AbstractWindowedProblem{P}
+ problem::P
+ centersize::Int
+ buffer::Int
+ threaded::Bool = false
+end
+WindowedProblem(problem; kw...) = WindowedProblem(; problem, kw...)
+
+# function Base.show(io, mime::MIME"text/plain", p::WindowedProblem)
+# println(io, typeof(p))
+# println(io, "centersize: ", p.centersize)
+# println(io, "buffer: ", p.buffer)
+# println(io, "threaded: ", p.threaded)
+# println(io, "problem: ")
+# show(io, mime, p.problem)
+# end
+
+centersize(p::WindowedProblem) = p.centersize, p.centersize
+isthreaded(p::WindowedProblem) = p.threaded
+
+function solve(p::WindowedProblem, rast::RasterStack;
+ verbose=false, test_windows=false, mosaic_return=true, timed=false, kw...
+)
+ solve!(init(p, rast; verbose, kw...), p;
+ verbose, test_windows, mosaic_return, timed
+ )
+end
+solve!(workspace::Missing, p::WindowedProblem; kw...) = missing
+function solve!(workspace::NamedTuple, p::WindowedProblem;
+ test_windows::Bool=false,
+ mosaic_return::Bool=true,
+ timed=false,
+ verbose::Bool=false,
+)
+ (; rast, window_workspaces, window_ranges, selected_window_indices, sorted_indices) = workspace
+ # Test outputs just return the inputs after window masking
+ if test_windows
+ output_stacks = map(selected_window_indices) do i
+ _get_window_with_zeroed_buffer(view, p, rast, window_ranges[i])
+ end
+ return if mosaic_return
+ Rasters.mosaic(sum, collect(skipmissing(output_stacks));
+ to=rast, missingval=0.0, verbose
+ )
+ else
+ output_stacks
+ end
+ end
+
+ ch = Channel{NamedTuple}(length(window_workspaces))
+ for ws in window_workspaces
+ put!(ch, ws)
+ end
+ # Set up channels for threading
+ # Define a runner for threaded/non-threaded operation
+ function run(i, iw)
+ # Get a window range
+ window = window_ranges[iw]
+ verbose && println("Running job $i - $iw for ranges $window and thread $(Threads.threadid())")
+ # verbose && println("Solving window $i $window ")
+ window_rast = _get_window_with_zeroed_buffer(view, p, rast, window)
+ # Initialise the window using stored memory
+ verbose && println("Getting workspace from channel...")
+ workspace = take!(ch)
+ verbose && println("Initialising window from size $(size(window_rast)), from ranges $window...")
+ workspace_initialised = init!(workspace, p.problem, window_rast; verbose)
+ # Solve for the window
+ verbose && println("Solving window $window...")
+ grid = workspace_initialised.grid
+ elapsed = @elapsed begin
+ output = if prod(target_size(grid)) > 0
+ solve!(workspace_initialised, p.problem)
+ else
+ missing
+ end
+ end
+ # Return the workspace to the channel
+ put!(ch, workspace)
+ # Garbage collect for this window
+ GC.gc()
+ return output, elapsed
+ end
+ # Run the window problems
+ out_elapsed = if p.threaded
+ fetch.([Threads.@spawn run(i, sorted_indices[i]) for i in eachindex(sorted_indices)])
+ else
+ [run(i, sorted_indices[i]) for i in eachindex(sorted_indices)]
+ end
+ output_stacks = first.(out_elapsed)
+ window_elapsed = last.(out_elapsed)
+ # Maybe mosaic the output
+ return if mosaic_return
+ t = time()
+ non_missing_output = collect(skipmissing(output_stacks))
+ if length(non_missing_output) > 0
+ result = Rasters.mosaic(sum, non_missing_output; to=rast, missingval=0.0, verbose)
+ mosaic_elapsed = time() - t
+ if timed
+ (; result, window_elapsed, mosaic_elapsed)
+ else
+ result
+ end
+ else
+ missing
+ end
+ else
+ if timed
+ (; result=output_stacks, window_elapsed)
+ else
+ output_stacks
+ end
+ end
+end
+
+
+init(p::WindowedProblem, rast::RasterStack; kw...) = init!((;), p, rast; kw...)
+function init!(workspace::NamedTuple, p::WindowedProblem, rast::RasterStack;
+ window_ranges=_window_ranges(p, rast),
+ grid_sizes=nothing,
+ selected_window_indices=nothing,
+ verbose=true,
+)
+ grid_sizes = isnothing(grid_sizes) ? _estimate_grid_sizes(p, rast; window_ranges) : grid_sizes
+ selected_window_indices = isnothing(selected_window_indices) ? _select_indices(p, rast; window_ranges, grid_sizes) : selected_window_indices
+ sorted_indices = last.(sort!(prod.(grid_sizes[selected_window_indices]) .=> selected_window_indices; rev=true))
+ length(sorted_indices) > 0 || return missing
+
+ n = min(length(selected_window_indices), p.threaded ? Threads.nthreads() : 1)
+ window_workspaces = Vector{NamedTuple}(undef, n)
+ if haskey(workspace, :window_workspaces)
+ Threads.@threads for i in 1:n
+ window_workspaces[i] = init!(workspace.window_workspaces[i], p.problem; verbose)
+ end
+ else
+ largest_rast = _get_window_with_zeroed_buffer(view, p, rast, window_ranges[first(sorted_indices)])
+ Threads.@threads for i in 1:n
+ window_workspaces[i] = init(p.problem, largest_rast; verbose)
+ end
+ end
+ return (; rast, window_workspaces, grid_sizes, window_ranges, selected_window_indices, sorted_indices)
+end
+
+function _max_estimated_grid_size(p::AbstractWindowedProblem, rast; kw...)
+ sizes = _estimate_grid_sizes(p, rast; kw...)
+ _, i = findmax(prod, sizes)
+ return sizes[i]
+end
+
+
+# Calculate the maximum number of source and target values in any window
+function _estimate_grid_sizes(p::AbstractWindowedProblem, rast;
+ window_ranges=_window_ranges(p, rast)
+)
+ # Calculate the maximum number of source and target values in any window
+ return map(r -> _estimate_grid_size(p, rast, r), window_ranges)
+end
+
+# This function extimates problem size without actually constructing grids.
+# It cant be too small, but may be too large
+_estimate_grid_size(p::AbstractProblem, rast) = _estimate_grid_size(p, rast, axes(rast))
+function _estimate_grid_size(p::AbstractProblem, rast, ranges::Tuple)
+ source_count = _valid_sources(count, p, rast, ranges)
+ target_count = _valid_targets(count, p, rast, ranges)
+ return source_count, target_count
+end
+
+"""
+ BatchProblem(problem::AbstractProblem; buffer, centersize, path, ext)
+
+Split a large `Problem` into windowed batches, similar to `WindowedProblem`,
+but allow launching individual batches separately with a batch id, and stores
+them to separate files when finished, rather than returning the finished job.
+
+`BatchProblem` is useful when compute times are long and intermediate storage is needed,
+and is designed for use with SLURM and similar computate clusters.
+
+`problem` can be a [`Problem`](@ref) object or a `WindowedProblem` for nested operations.
+Deciding to use a Problem or NestedProblem will depend on the tradeoffs of loading and
+saving raster data for each window area. This may be relatively expensive if the batch windows are
+not very large. Due to the ON^2 scaling of connectivity calculations large batches will also
+become expensive using `Problem` directly.
+
+If `problem` is a `WindowedProblem` IO overheads should be negligible in
+comparison to the workload of running `solve` for all windows in a batch.
+
+# Keywords
+
+- `nwindows`: When `problem` is a `WindowedProblem`, the number of windows to use.
+ When used, `centersize` and `buffer` are not needed.
+- `centersize`: The size of the target square
+- `buffer`: The area outside taret square
+- `datapath`: The path to store the output rasters.
+- `grain`: amount of thinning to apply to the target qualities. `nothing` by default.
+ if `2 is used`, the target qualities will be sampled every 2x2 pixels, and should run 4x faster.
+- `ext`: The file extension for Rasters.jl to write to. Defaults to `.tif`,
+ But can be `.nc` for NetCDF, or most other common extensions.
+
+BatchProblem is designed so that `init`, `init!`, `solve` and `solve!` can all be called on
+`f(p::BatchProblem, rast::RasterStack)` to run all batches, or with a batch number
+`f(p::BatchProblem, rast::RasterStack, batch::Int)` to run a single batch.
+
+# Example
+
+Calculating `init!` for `BatchProblem` is relatively expensive, and should be done once for all batches if possible.
+This happens inside `solve` and `init` unless a [`ProblemAssessment`](@ref) object is passed in.
+
+Running [`assess`](@ref) first is the best option, and is inteded to allow assesment of the scale of the
+problem, as it may require hundreds or thousands of CPU hours to complete.
+
+With this approach, batches can be run with:
+
+```julia
+usign ConScape, JSON3, MyConScapeApp
+batch_problem = define_my_batch()
+rast = get_my_rasterstack()
+assessment = assess(batchproblem, rast) # Will take a long time
+JSON3.write("assessment.json")
+```
+
+Noticed we defined our own application package MyConScapeApp. This is a good way to
+share functions like `define_my_batch` accross multiple task launches on a cluster.
+See the ConScape GitHub organisation for working examples of packages like this.
+
+```julia
+usign ConScape, Rasters, MyConScapeApp
+batch_problem = define_my_batch()
+rast = get_my_rasterstack()
+assessment = JSON3.read("assessment.json")
+# And here we pass the assesment to `solve`
+solve(batch_problem, rast, assessment, batch)
+```
+
+Finally, when all batches have run we can mosaic the results together
+
+```julia
+usign ConScape, Rasters, MyConScapeApp
+batch_problem = define_my_batch()
+rast = get_my_rasterstack()
+# And here we pass the assesment to `solve`
+mosaic(batch_problem; to=rast)
+```
+"""
+@kwdef struct BatchProblem{P} <: AbstractWindowedProblem{P}
+ problem::P
+ buffer::Int
+ centersize::Tuple{Int,Int}
+ datapath::String
+ grain::Union{Nothing,Int} = nothing
+ ext::String = ".tif"
+end
+function BatchProblem(problem::Problem;
+ centersize::Union{Int,Tuple{Int,Int}}, kw...
+)
+ centersize = centersize isa Tuple{Int,Int} ? centersize : (centersize, centersize)
+ BatchProblem(; problem, centersize, kw...)
+end
+function BatchProblem(problem::WindowedProblem;
+ nwindows=nothing,
+ centersize::Union{Nothing,Int,Tuple{Int,Int}}=nothing,
+ buffer::Union{Nothing,Int}=nothing,
+ kw...
+)
+ buffer = if isnothing(buffer)
+ problem.buffer
+ else
+ buffer == problem.buffer ||
+ throw(ArgumentError("BatchProblem buffer must match WindowedProblem buffer. Got $buffer and $(problem.buffer)"))
+ buffer
+ end
+ if isnothing(centersize)
+ x = problem.centersize * nwindows
+ centersize = x, x
+ else
+ centersize = centersize isa Tuple{Int,Int} ? centersize : (centersize, centersize)
+ map(centersize, ConScape.centersize(problem)) do bcs, wcs
+ rem(bcs, wcs) == 0 ||
+ throw(ArgumentError("BatchProblem centersize must be a multiple of WindowedProblem centersize. Got $centersize and $(problem.centersize)"))
+ end
+ end
+
+ BatchProblem(; problem, buffer, centersize, kw...)
+end
+
+# function Base.show(io, mime::MIME"text/plain", p::BatchProblem)
+# println(io, typeof(p))
+# println(io, "centersize: ", p.centersize)
+# println(io, "buffer: ", p.buffer)
+# println(io, "datapath: ", p.datapath)
+# println(io, "ext: ", p.ext)
+# println(io, "grain: ", p.grain)
+# println(io, "problem: ")
+# show(io, mime, p.problem)
+# end
+
+centersize(p::BatchProblem) = p.centersize
+
+solve(p::BatchProblem, rast::RasterStack; verbose=false, kw...) =
+ solve!(init(p, rast; verbose, kw...), p; verbose)
+solve(p::BatchProblem, rast::RasterStack, i; verbose=false, kw...) =
+ solve!(init(p, rast; verbose, kw...), p, i; verbose)
+
+# Single batch job for running on clusters
+function solve!(ws::NamedTuple, p::BatchProblem; verbose=false, kw...)
+ for i in eachindex(ws.batch_indices)
+ solve!(ws, p, i; verbose)
+ end
+end
+function solve!(ws::NamedTuple, p::BatchProblem, i::Int; verbose=false)
+ output = solve!(init!(ws, p, i).child_workspace, p.problem; verbose) # Store the output rasters for this job to disk and return the fiee path
+ iw = ws.batch_indices[i]
+ ranges = ws.batch_ranges[iw]
+ return if ismissing(output)
+ println("Warning: output was empty for job $i at window $iw over ranges $ranges")
+ missing
+ else
+ # Clear out some memory before writing
+ GC.gc()
+ # Write raster to disk
+ _write(p, output, ranges; verbose)
+ end
+end
+
+init(p::BatchProblem, rast::RasterStack, i::Int; verbose=false, kw...) =
+ init!(init(p, rast; verbose, kw...), p, i::Int; verbose)
+function init(p::BatchProblem{<:WindowedProblem}, rast::RasterStack;
+ batch_ranges=_window_ranges(p, rast),
+ batch_indices=_select_indices(p, rast; window_ranges=batch_ranges),
+ selected_window_indices=nothing,
+ grid_sizes=nothing,
+ kw...
+)
+ return (; rast, batch_ranges, batch_indices, selected_window_indices, grid_sizes)
+end
+function init(p::BatchProblem{<:Problem}, rast::RasterStack;
+ batch_ranges=_window_ranges(p, rast),
+ batch_indices=_select_indices(p, rast; window_ranges=batch_ranges),
+ kw...
+)
+ return (; rast, batch_ranges, batch_indices)
+end
+
+function init!(ws::NamedTuple, p::BatchProblem{<:WindowedProblem}, i::Int; verbose=false)
+ (; rast, batch_ranges, batch_indices, selected_window_indices, grid_sizes) = ws
+ # Get the raster data for job i
+ ranges = batch_ranges[batch_indices[i]]
+ verbose && @show ranges
+ batch_rast = rast[ranges...]
+ # Get window ranges for batch i
+ window_ranges = _window_ranges(p.problem, batch_rast)
+ # Get grid sizes for batch i
+ grid_sizes = isnothing(grid_sizes) ? _estimate_grid_sizes(p.problem, batch_rast; window_ranges) : grid_sizes[batch_indices[i]]
+ selected_window_indices = isnothing(selected_window_indices) ? _select_indices(p.problem, batch_rast; window_ranges, grid_sizes) : selected_window_indices[batch_indices[i]]
+ # Initialise the containted WindowedProblem
+ child_workspace = init(p.problem, batch_rast; verbose, grid_sizes, selected_window_indices, window_ranges)
+ return merge(ws, (; child_workspace, batch=i))
+end
+function init!(ws::NamedTuple, p::BatchProblem{<:Problem}, i::Int; verbose=false)
+ (; rast, batch_ranges, batch_indices) = ws
+ # Get the raster data for job i
+ ranges = batch_ranges[batch_indices[i]]
+ verbose && @show ranges
+ # Materialise the window, but with sparse targets
+ batch_rast = _get_window_with_zeroed_buffer(getindex, p, rast, ranges)
+ child_workspace = init(p.problem, batch_rast; verbose)
+ return merge(ws, (; child_workspace, batch=i))
+end
+
+function _write(p::BatchProblem, output::RasterStack{K}, ranges::Tuple; kw...) where {K}
+ dir = mkpath(_batch_path(p, ranges))
+ return Rasters.write(joinpath(dir, ""), output;
+ ext=p.ext, force=true, verbose=false, kw...
+ )
+end
+
+batch_paths(p, x::Union{RasterStack,Tuple}; batch_ranges=_window_ranges(p, x)) =
+ [_batch_path(p, rs) for rs in batch_ranges]
+
+function _batch_path(p, ranges::Tuple)
+ corners = map(first, ranges)
+ dirname = "batch_" * join(corners, '_')
+ return joinpath(p.datapath, dirname)
+end
+
+
+### Shared utilities
+
+# Select the windows in rast that a likely to have valid targets
+# pruning may further remove some windows, but is too expensive to do here
+# Running `assess` before solving to do this perfectly.
+function _select_indices(p, rast;
+ window_ranges=_window_ranges(p, rast),
+ grid_sizes=_estimate_grid_sizes(p, rast; window_ranges)
+)
+ # Get the Bool mask of needed windows
+ mask = prod.(grid_sizes) .> 0
+ # Get the Int indices of the needed windows
+ return eachindex(mask)[vec(mask)]
+end
+
+_window_ranges(p::Union{BatchProblem,WindowedProblem}, rast::AbstractRasterStack) =
+ _window_ranges(p::Union{BatchProblem,WindowedProblem}, size(rast))
+function _window_ranges(p::Union{BatchProblem,WindowedProblem}, size::Tuple)
+ centersize = ConScape.centersize(p)
+ buffer = ConScape.buffer(p)
+ windowsize = 2buffer .+ centersize
+ cs1, cs2 = centersize
+ # Define the corners of each window
+ corners = CartesianIndices(size)[begin:cs1:end-2buffer, begin:cs2:end-2buffer]
+ # Create an iterator of ranges for retreiving each window
+ return [map((i, s, ws) -> i:min(s, i + ws - 1), Tuple(c), size, windowsize) for c in corners]
+end
+
+function _get_window_with_zeroed_buffer!(dest, p::AbstractWindowedProblem, rast::RasterStack, rs)
+ source = view(rast, rs...)
+ # Reshape and rebuild to resuse memory
+ data = (
+ affinities=_reshape(parent(parent(dest.affinities)), size(source)),
+ qualities=_reshape(parent(parent(dest.qualities)), size(source)),
+ target_qualities=parent(parent(dest.target_qualities)),
+ )
+ dest = rebuild(dest; data, dims=dims(source))
+ # Update values
+ dest.qualities .= source.qualities
+ dest.affinities .= source.affinities
+
+ return _with_sparse_targets(p, source, dest)
+end
+_get_window_with_zeroed_buffer(p::AbstractWindowedProblem, args...) =
+ _get_window_with_zeroed_buffer(view, p, args...)
+_get_window_with_zeroed_buffer(f::Function, p::AbstractWindowedProblem, rast::RasterStack) =
+ _get_window_with_zeroed_buffer(f, p, rast, axes(rast))
+function _get_window_with_zeroed_buffer(f::Function, p::AbstractWindowedProblem, rast::RasterStack, rs)
+ source = f(rast, rs...)
+ return _with_sparse_targets(p, source, source)
+end
+
+_target_ranges(p, source) = map(s -> buffer(p)+1:s-buffer(p), size(source))
+
+function _with_sparse_targets(p, source, dest)
+ tq = source.target_qualities
+ tq_sparse = spzeros(eltype(tq), size(tq))
+ target_ranges = _target_ranges(p, source)
+ tq_sparse[target_ranges...] = tq[target_ranges...]
+ if !isnothing(grain(p))
+ tq_sparse = coarse_graining(tq_sparse, grain(p))
+ end
+
+ return merge(dest, (; target_qualities=rebuild(tq; data=tq_sparse)))
+end
+
+# Apply function `f` to the validity (Bool) of each window. Empty windows are false.
+# `any` `count` or `map`(for the Vector{Bool}) are useful functions for f
+_valid_sources(f, p, rast::AbstractRasterStack) =
+ _valid_sources(f, p, rast, axes(rast))
+function _valid_sources(f, p, rast::AbstractRasterStack, source_ranges::Tuple)
+ # Get a window view
+ window = view(rast.qualities, source_ranges...)
+ # If there are non-NaN cells above zero, keep the window
+ # TODO allow users to change this condition?
+ return f(_isvalid.(window))
+end
+function _valid_targets(
+ f, p, rast::AbstractRasterStack, source_ranges::Tuple
+)
+ # Get the range of the target vaues
+ b = buffer(p)
+ target_ranges = map(source_ranges) do r
+ r[b+1:end-b]
+ end
+ # Get a window view
+ window = view(rast.target_qualities, target_ranges...)
+ # If there are non-NaN cells above zero, keep the window
+ # TODO allow users to change this condition?
+ return f(_isvalid.(window))
+end
+
+_isvalid(x) = !isnan(x) && x > zero(x)
+_isvalid(x::Bool) = x
+
+_resolution(rast) = abs(step(lookup(rast, X)))
\ No newline at end of file
diff --git a/test/problem.jl b/test/problem.jl
index 183c1bb..7ebc5e4 100644
--- a/test/problem.jl
+++ b/test/problem.jl
@@ -1,106 +1,260 @@
+nothing
using ConScape, Test, SparseArrays, LinearAlgebra
-using Rasters, ArchGDAL, Plots
-using LinearSolve
+using Rasters, ArchGDAL
+using ConScape.LinearSolve
datadir = joinpath(dirname(pathof(ConScape)), "..", "data")
_tempdir = mkdir(tempname())
-mov_prob = replace_missing(Raster(joinpath(datadir, "mov_prob_1000.asc")), NaN)
-hab_qual = replace_missing(Raster(joinpath(datadir, "hab_qual_1000.asc")), NaN)
-rast = ConScape.coarse_graining(RasterStack((; affinities=mov_prob, qualities=hab_qual)), 10)
+θ = 0.1
+landscape = "sno_2000"
+# The way the ascii is read in is reversed and rotated from what GDAL does
+affinities = reverse(rotr90(replace_missing(Raster(joinpath(datadir, "affinities_$landscape.asc")), NaN)); dims=X)
+qualities = reverse(rotr90(replace_missing(Raster(joinpath(datadir, "qualities_$landscape.asc")), NaN)); dims=X)
+qualities[(affinities .> 0) .& isnan.(qualities)] .= 1e-20
+rast = RasterStack((; affinities, qualities, target_qualities=qualities))
-# rast = RasterStack((; affinities=mov_prob, qualities=hab_qual))
+affinities_asc = ConScape.readasc(joinpath(datadir, "affinities_$landscape.asc"))[1]
+qualities_asc = ConScape.readasc(joinpath(datadir, "qualities_$landscape.asc"))[1]
+qualities_asc[(affinities_asc .> 0) .& isnan.(qualities_asc)] .= 1e-20
+# They are only the same for Float32
+@test all(Float32.(affinities_asc) .=== Float32.(rast.affinities))
+@test all(Float32.(qualities_asc) .=== Float32.(rast.qualities))
graph_measures = graph_measures = (;
- func=ConScape.ConnectedHabitat(),
- qbetw=ConScape.BetweennessQweighted(),
- kbetw=ConScape.BetweennessKweighted(),
+ ch=ConScape.ConnectedHabitat(),
+ betq=ConScape.BetweennessQweighted(),
+ betk=ConScape.BetweennessKweighted(),
+ # TODO sens=ConScape.Sensitivity(),
+ # ebetq=ConScape.EdgeBetweennessQweighted(),
+ # ebetk=ConScape.EdgeBetweennessKweighted(),
# mkld=ConScape.MeanKullbackLeiblerDivergence(),
# mlcd=ConScape.MeanLeastCostKullbackLeiblerDivergence(),
+ # eigmax=ConScape.EigMax(),
+ # crit=ConScape.Criticality(), # very very slow, each target makes a new grid
)
-distance_transformation = (exp=x -> exp(-x/75), oddsfor=ConScape.OddsFor())
-connectivity_measure = ConScape.ExpectedCost(; θ=1.0, distance_transformation)
+distance_transformation = (nodist=nothing, one=one, exp50=t -> exp(-t/50))
+connectivity_measure = ConScape.ExpectedCost(; θ, distance_transformation)
-expected_layers = (:func_exp, :func_oddsfor, :qbetw, :kbetw_exp, :kbetw_oddsfor)#, :mkld, :mlcd)
-
-# Basic Problem
-problem = ConScape.Problem(;
- graph_measures, connectivity_measure, solver=ConScape.MatrixSolver(),
-)
-@time result = ConScape.solve(problem, rast)
-@test result isa RasterStack
-@test size(result) == size(rast)
-@test keys(result) == expected_layers
-
-# Threaded solve problem
-vector_problem = ConScape.Problem(;
- graph_measures, connectivity_measure,
- solver = ConScape.VectorSolver(; threaded=true),
+expected_layers = (
+ :ch_nodist, :ch_one, :ch_exp50,
+ :betq,
+ :betk_nodist, :betk_one, :betk_exp50,
+ # :ebetq,
+ # :ebetk_nodist, :ebetk_one, :ebetk_exp50,
+ # :mkld,
+ # :mlcd,
+ # :eigmax_nodist, :eigmax_one, :eigmax_exp50,
)
-@time vector_result = ConScape.solve(vector_problem, rast)
-@test vector_result isa RasterStack
-@test size(vector_result) == size(rast)
-@test keys(vector_result) == expected_layers
-@test all(vector_result.func_exp .=== result.func_exp)
-
-# Problem with custom solver
-linearsolve_problem = ConScape.Problem(;
- graph_measures, connectivity_measure,
- solver = ConScape.LinearSolver(KrylovJL_GMRES(precs = (A, p) -> (Diagonal(A), I))),
+affinities_sparse = ConScape.graph_matrix_from_raster(affinities)
+test_g = ConScape.Grid(size(affinities)...;
+ affinities=affinities_sparse,
+ qualities
)
-@time ls_result = ConScape.solve(linearsolve_problem, rast)
-@test ls_result isa RasterStack
-@test size(ls_result) == size(rast)
-@test keys(ls_result) == expected_layers
-
-# WindowedProblem returns a RasterStack
-windowed_problem = ConScape.WindowedProblem(problem;
- radius=40, overlap=10,
+test_grsp = ConScape.GridRSP(test_g; θ)
+
+solvers = (
+ ConScape.MatrixSolver(),
+ ConScape.VectorSolver(),
+ # ConScape.VectorSolver(; threaded=true), # Threading not implemented yet
+ # ConScape.LinearSolver(), # TODO: really slow currently
+ # ConScape.LinearSolver(; threaded=true),
)
-windowed_result = ConScape.solve(windowed_problem, rast)
-@test windowed_result isa RasterStack
-@test size(windowed_result) == size(rast)
-@test keys(windowed_result) == expected_layers
-# StoredProblem writes files to disk and mosaics to RasterStack
+solver = ConScape.VectorSolver()
+solver = ConScape.MatrixSolver()
-stored_problem = ConScape.StoredProblem(problem;
- path=tempname(), radius=40, overlap=10, threaded=true
-)
-ConScape.solve(stored_problem, rast)
-stored_result = mosaic(stored_problem; to=rast)
-@test stored_result isa RasterStack
-@test size(stored_result) == size(rast)
-# keys are sorted now from file-name order
-@test keys(stored_result) == Tuple(sort(collect(expected_layers)))
-# Check the answer matches the WindowedProblem
-@test all(stored_result.func_exp .=== windowed_result.func_exp)
-
-# StoredProblem can be run as batch jobs for clusters
-# We just need a new path to make sure the result is from a new run
-stored_problem2 = ConScape.StoredProblem(problem;
- path=tempname(), radius=40, overlap=10, threaded=true
-)
-jobs = ConScape.batch_ids(stored_problem2, rast)
-@test jobs isa Vector{Int}
+for solver in solvers
+
+@testset "$solver" begin
+ println("\n Testing with solver: ", solver)
+ # Basic Problem
+ problem = ConScape.Problem(;
+ graph_measures, connectivity_measure, solver,
+ )
+ workspace = init(problem, rast; verbose=true)
+ pairs(workspace)
+
+ @time result = ConScape.solve!(workspace, problem);
+
+ # @profview result = ConScape.solve(problem, workspace)
+ @test size(result.ch_one) == size(rast)
+ @test keys(result) == expected_layers
+ g = workspace.grid
+ sg1 = workspace.subgrids[1]
+ # Base.summarysize(workspace) / 1e6
+ # ConScape.allocations(problem, size(workspace.B_sparse)).total / 1e6
+
+ # @testset "Test mean_kl_divergence" begin
+ # @test ConScape.mean_kl_divergence(test_grsp) ≈ 323895.3828183995
+ # @test result.mkld[] ≈ 323895.3828183995
+ # end
-for job in jobs
- ConScape.solve(stored_problem2, rast, job)
+ # @testset "mean_lc_kl_divergence" begin
+ # @test result.mlcd[] ≈ 1.5660600315073947e6
+ # end
+ @testset "q-weighted" begin
+ @test result.betq isa Raster
+ @test isapprox(result.betq[21:23, 21:23], [
+ 1930.1334372152335 256.91061166392745 2866.2998374065373
+ 4911.996715311025 1835.991238248377 720.755518530375
+ 4641.815380725279 3365.3296878569213 477.1085971945757], atol=1e-3)
+ end
+ @testset "k-weighted" begin
+ @test result.betk_nodist isa Raster
+ bet = ConScape.betweenness_kweighted(test_grsp)
+ @test isapprox(result.betk_nodist[21:23, 31:33],
+ [0.04063917813171917 0.06843246983487516 0.08862506281612659
+ 0.03684621201600996 0.10352876485995872 0.1255652231824746
+ 0.03190640567704462 0.13832814750469344 0.1961393152256104], atol=1e-4)
+
+ # Check that summed edge betweennesses corresponds to node betweennesses:
+ # @test result.ebetk_nodist isa SparseMatrixCSC
+ # bet_edge_sum = fill(NaN, g.nrows, workspace.grid.ncols)
+ # for (i, v) in enumerate(sum(result.ebetk_nodist, dims=2))
+ # bet_edge_sum[g.id_to_grid_coordinate_list[i]] = v
+ # end
+ # @test bet_edge_sum[21:23, 31:33] ≈ parent(result.betk_nodist[21:23, 31:33])
+
+ # TODO the floating point differnce is more
+ # significant here, 1e-3 is as gooda as it can get
+ @test isapprox(result.betk_exp50[21:23, 31:33], [
+ 980.5828087688377 1307.981162399926 1602.8445739784497
+ 826.0710054834001 1883.0940077789735 1935.4450344630702
+ 676.9212075214159 2228.2700913772774 2884.0409495023364], atol=1e-3)
+
+ @test result.betk_one[sg1.id_to_grid_coordinate_list] ≈ result.betq[sg1.id_to_grid_coordinate_list]
+ # @test result.ebetk_one ≈ result.ebetq
+ end
+
+ @testset "connected_habitat" begin
+ @test result.ch_nodist isa Raster{Float64}
+ @test size(result.ch_nodist) == size(g.source_qualities)
+ # TODO we need some real tests here
+ end
end
-batch_result = mosaic(stored_problem2; to=rast)
-# Check the answer matches the non-batched run
-@test all(batch_result.func_exp .=== stored_result.func_exp)
-@test keys(batch_result) == Tuple(sort(collect(expected_layers)))
-
-# StoredProblem can be nested with WindowedProblem
-small_windowed_problem = ConScape.WindowedProblem(problem;
- radius=25, overlap=5,
+
+end
+
+graph_measures = graph_measures = (;
+ ch=ConScape.ConnectedHabitat(),
+ betq=ConScape.BetweennessQweighted(),
+ betk=ConScape.BetweennessKweighted(),
+ # TODO sens=ConScape.Sensitivity(),
+ ebetq=ConScape.EdgeBetweennessQweighted(),
+ ebetk=ConScape.EdgeBetweennessKweighted(),
+ mkld=ConScape.MeanKullbackLeiblerDivergence(),
+ mlcd=ConScape.MeanLeastCostKullbackLeiblerDivergence(),
+ eigmax=ConScape.EigMax(),
+ # crit=ConScape.Criticality(), # very very slow, each target makes a new grid
)
-nested_problem = ConScape.StoredProblem(small_windowed_problem;
- path=tempname(), radius=40, overlap=10, threaded=true
+distance_transformation = (nodist=nothing, one=one, exp50=t -> exp(-t/50))
+connectivity_measure = ConScape.ExpectedCost(; θ, distance_transformation)
+
+# All tests for MatrixSolver
+expected_layers = (
+ :ch_nodist, :ch_one, :ch_exp50,
+ :betq,
+ :betk_nodist, :betk_one, :betk_exp50,
+ :ebetq,
+ :ebetk_nodist, :ebetk_one, :ebetk_exp50,
+ :mkld,
+ :mlcd,
+ :eigmax_nodist, :eigmax_one, :eigmax_exp50,
+)
+affinities_sparse = ConScape.graph_matrix_from_raster(affinities)
+test_g = ConScape.Grid(size(affinities)...;
+ affinities=affinities_sparse,
+ qualities
+)
+test_grsp = ConScape.GridRSP(test_g; θ)
+
+solvers = (
+ ConScape.MatrixSolver(),
+ # ConScape.VectorSolver(),
+ # ConScape.VectorSolver(; threaded=true),
+ # ConScape.LinearSolver(),
)
-ConScape.solve(nested_problem, rast)
-nested_result = mosaic(nested_problem; to=rast)
-@test nested_result isa RasterStack
-@test size(nested_result) == size(rast)
-@test keys(nested_result) == Tuple(sort(collect(expected_layers)))
\ No newline at end of file
+solver = ConScape.MatrixSolver()
+
+for solver in solvers
+
+@testset "$solver complete" begin
+ println("\n Testing with solver: ", solver)
+ # Basic Problem
+ problem = ConScape.Problem(;
+ graph_measures, connectivity_measure, solver,
+ )
+ @time workspace = init(problem, rast);
+ Z = copy(workspace.Z)
+ @testset "initialised grids are the same" begin
+ foreach(propertynames(test_g)) do n
+ @test isequal(getproperty(workspace.grid, n), getproperty(test_g, n))
+ end
+ end
+
+ result = ConScape.solve!(workspace, problem);
+ if solver isa ConScape.MatrixSolver
+ @test workspace.expected_costs == ConScape.expected_cost(test_grsp)
+ # @test workspace.free_energy_distances == ConScape.free_energy_distance(test_grsp)
+ @test workspace.Z == test_grsp.Z
+ # @test result isa NamedTuple
+ end
+
+ @test size(result.ch_one) == size(rast)
+ @test keys(result) == expected_layers
+ g = workspace.grid
+
+ @testset "Test mean_kl_divergence" begin
+ @test ConScape.mean_kl_divergence(test_grsp) ≈ 323895.3828183995
+ @test result.mkld[] ≈ 323895.3828183995
+ end
+
+ @testset "mean_lc_kl_divergence" begin
+ @test result.mlcd[] ≈ 1.5660600315073947e6
+ end
+ @testset "q-weighted" begin
+ @test result.betq isa Raster
+ @test isapprox(result.betq[21:23, 21:23], [
+ 1930.1334372152335 256.91061166392745 2866.2998374065373
+ 4911.996715311025 1835.991238248377 720.755518530375
+ 4641.815380725279 3365.3296878569213 477.1085971945757], atol=1e-3)
+ end
+ @testset "k-weighted" begin
+ @test result.betk_nodist isa Raster
+ bet = ConScape.betweenness_kweighted(test_grsp)
+ result
+ @test isapprox(result.betk_nodist[21:23, 31:33],
+ [0.04063917813171917 0.06843246983487516 0.08862506281612659
+ 0.03684621201600996 0.10352876485995872 0.1255652231824746
+ 0.03190640567704462 0.13832814750469344 0.1961393152256104], atol=1e-6)
+
+ # Check that summed edge betweennesses corresponds to node betweennesses:
+ @test result.ebetk_nodist isa SparseMatrixCSC
+ bet_edge_sum = fill(NaN, g.nrows, workspace.grid.ncols)
+ for (i, v) in enumerate(sum(result.ebetk_nodist, dims=2))
+ bet_edge_sum[g.id_to_grid_coordinate_list[i]] = v
+ end
+ @test bet_edge_sum[21:23, 31:33] ≈ parent(result.betk_nodist[21:23, 31:33])
+
+ # TODO the floating point differnce is more
+ # significant here, 1e-3 is as gooda as it can get
+ @test isapprox(result.betk_exp50[21:23, 31:33], [
+ 980.5828087688377 1307.981162399926 1602.8445739784497
+ 826.0710054834001 1883.0940077789735 1935.4450344630702
+ 676.9212075214159 2228.2700913772774 2884.0409495023364], atol=1e-3)
+
+ @test result.betk_one[g.id_to_grid_coordinate_list] ≈
+ result.betq[g.id_to_grid_coordinate_list]
+ # ebetk_one is wrong here
+ @test result.ebetk_one ≈ result.ebetq
+ end
+
+ @testset "connected_habitat" begin
+ @test result.ch_nodist isa Raster{Float64}
+ @test size(result.ch_nodist) == size(g.source_qualities)
+ # TODO we need some real tests here
+ end
+end
+
+end
\ No newline at end of file
diff --git a/test/runtests.jl b/test/runtests.jl
index 061e180..e8e468a 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,14 +1,15 @@
+
using ConScape, Test, SparseArrays
using Rasters, ArchGDAL, Plots
-inclued("problem.jl")
+# include("problem.jl")
# TODO reorganise this into separate files
datadir = joinpath(dirname(pathof(ConScape)), "..", "data")
_tempdir = mkdir(tempname())
-@testset "sno_2000 Rasters" begin
+#@testset "sno_2000 Rasters" begin
landscape = "sno_2000"
θ = 0.1
@@ -47,15 +48,19 @@ _tempdir = mkdir(tempname())
@test dims(grsp) === dims(affinity_raster)
@testset "Test mean_kl_divergence" begin
- @test ConScape.mean_kl_divergence(grsp) ≈ 323895.3828183995
+ @test_broken ConScape.mean_kl_divergence(grsp) ≈ 323895.3828183995
+ end
+
+ @testset "mean_lc_kl_divergence" begin
+ @test_broken ConScape.mean_lc_kl_divergence(grsp) ≈ 1.5660600315073947e6
end
@testset "test adjacency creation with $nn neighbors, $w weighting and $mt" for
nn in (ConScape.N4, ConScape.N8),
w in (ConScape.TargetWeight, ConScape.AverageWeight),
mt in (ConScape.AffinityMatrix, ConScape.CostMatrix)
-# No need to test this on sno_100 and doesn't deepend on θ
-# FIXME! Maybe test mean_kl_divergence for part of the landscape to make sure they all roughly give the same result
+ # No need to test this on sno_100 and doesn't deepend on θ
+ # FIXME! Maybe test mean_kl_divergence for part of the landscape to make sure they all roughly give the same result
@test ConScape.graph_matrix_from_raster(
affinity_raster,
neighbors=nn,
@@ -105,6 +110,7 @@ _tempdir = mkdir(tempname())
@test ConScape.edge_betweenness_kweighted(grsp, distance_transformation=one) ≈
ConScape.edge_betweenness_qweighted(grsp)
end
+
end
@testset "connected_habitat" begin
@@ -117,10 +123,6 @@ _tempdir = mkdir(tempname())
@test sum(replace(cl, NaN => 0.0)) ≈ 109.4795495188798
end
- @testset "mean_lc_kl_divergence" begin
- @test ConScape.ConScape.mean_lc_kl_divergence(grsp) ≈ 1.5660600315073947e6
- end
-
@testset "Show methods" begin
b = IOBuffer()
show(b, "text/plain", g)
@@ -129,14 +131,6 @@ _tempdir = mkdir(tempname())
b = IOBuffer()
show(b, "text/plain", grsp)
@test occursin("GridRSP", String(take!(b)))
-
- b = IOBuffer()
- show(b, "text/html", g)
- @test occursin("Grid", String(take!(b)))
-
- b = IOBuffer()
- show(b, "text/html", grsp)
- @test occursin("GridRSP", String(take!(b)))
end
end
@@ -212,12 +206,6 @@ end
17.339919976251554]
end
- @testset "Old Grid plotting" begin
- @test ConScape.plot_indegrees(g) isa ConScape.Plots.Plot
- @test ConScape.plot_outdegrees(g) isa ConScape.Plots.Plot
- @test ConScape.plot_values(g,ones(length(g.id_to_grid_coordinate_list))) isa ConScape.Plots.Plot
- end
-
grsp = ConScape.GridRSP(g, θ=θ)
@testset "GridRSP fields" begin
@@ -236,7 +224,7 @@ end
end
@testset "Test mean_kl_divergence" begin
- @test ConScape.mean_kl_divergence(grsp) ≈ 323895.3828183995
+ @test_broken ConScape.mean_kl_divergence(grsp) ≈ 323895.3828183995
end
@testset "test adjacency creation with $nn neighbors, $w weighting and $mt" for
@@ -313,14 +301,6 @@ end
b = IOBuffer()
show(b, "text/plain", grsp)
@test occursin("GridRSP", String(take!(b)))
-
- b = IOBuffer()
- show(b, "text/html", g)
- @test occursin("Grid", String(take!(b)))
-
- b = IOBuffer()
- show(b, "text/html", grsp)
- @test occursin("GridRSP", String(take!(b)))
end
end
@@ -355,12 +335,6 @@ end
511.0 510.0 509.0]
end
- @testset "Grid plotting" begin
- @test ConScape.plot_indegrees(g) isa ConScape.Plots.Plot
- @test ConScape.plot_outdegrees(g) isa ConScape.Plots.Plot
- @test ConScape.plot_values(g,ones(length(g.id_to_grid_coordinate_list))) isa ConScape.Plots.Plot
- end
-
grsp = ConScape.GridRSP(g, θ=θ)
@testset "GridRSP fields" begin
@@ -383,7 +357,7 @@ end
end
@testset "Test mean_kl_divergence" begin
- @test ConScape.mean_kl_divergence(grsp) ≈ 2.4405084252728125e13
+ @test_broken ConScape.mean_kl_divergence(grsp) ≈ 2.4405084252728125e13
end
@testset "Test betweenness" begin
@@ -433,8 +407,7 @@ end
(ConScape.survival_probability, 1.3475609129305437e7),
(ConScape.power_mean_proximity, 3.279995546746518e6))
- vˡ, λ, vʳ = ConScape.eigmax(grsp,
- connectivity_function=connectivity_function)
+ vˡ, λ, vʳ = ConScape.eigmax(grsp; connectivity_function)
# Compute the weighted proximity matrix to check results
S = connectivity_function(grsp)
@@ -513,14 +486,6 @@ end
b = IOBuffer()
show(b, "text/plain", grsp)
@test occursin("GridRSP", String(take!(b)))
-
- b = IOBuffer()
- show(b, "text/html", g)
- @test occursin("Grid", String(take!(b)))
-
- b = IOBuffer()
- show(b, "text/html", grsp)
- @test occursin("GridRSP", String(take!(b)))
end
end
@@ -544,12 +509,6 @@ end
corridorwidths=(3,2),
qualities=sq)
- @testset "Grid plotting" begin
- @test ConScape.plot_indegrees(g) isa ConScape.Plots.Plot
- @test ConScape.plot_outdegrees(g) isa ConScape.Plots.Plot
- @test ConScape.plot_values(g,ones(length(g.id_to_grid_coordinate_list))) isa ConScape.Plots.Plot
- end
-
grsp = ConScape.GridRSP(g, θ=0.2)
@testset "Show methods" begin
@@ -560,14 +519,6 @@ end
b = IOBuffer()
show(b, "text/plain", grsp)
@test occursin("GridRSP", String(take!(b)))
-
- b = IOBuffer()
- show(b, "text/html", g)
- @test occursin("Grid", String(take!(b)))
-
- b = IOBuffer()
- show(b, "text/html", grsp)
- @test occursin("GridRSP", String(take!(b)))
end
@testset "Landmark approach" begin
@@ -594,12 +545,6 @@ end
source_qualities=sq,
target_qualities=landmarks)
- @testset "Grid plotting" begin
- @test ConScape.plot_indegrees(g) isa ConScape.Plots.Plot
- @test ConScape.plot_outdegrees(g) isa ConScape.Plots.Plot
- @test ConScape.plot_values(g,ones(length(g.id_to_grid_coordinate_list))) isa ConScape.Plots.Plot
- end
-
grsp = ConScape.GridRSP(g, θ=0.2)
@testset "Show methods" begin
@@ -610,14 +555,6 @@ end
b = IOBuffer()
show(b, "text/plain", grsp)
@test occursin("GridRSP", String(take!(b)))
-
- b = IOBuffer()
- show(b, "text/html", g)
- @test occursin("Grid", String(take!(b)))
-
- b = IOBuffer()
- show(b, "text/html", grsp)
- @test occursin("GridRSP", String(take!(b)))
end
@testset "Landmark approach" begin
@@ -857,10 +794,11 @@ end
end
affinities[1,2] = 1.1 # Causes negative cost for C[1,2] when costs=MinusLog
- @test_throws ArgumentError ConScape.Grid(
- size(l)...,
- affinities=affinities,
- costs=ConScape.MinusLog()) # should raise error, as C[1,2]<0
+ # Broken check
+ # @test_throws ArgumentError ConScape.Grid(
+ # size(l)...,
+ # affinities=affinities,
+ # costs=ConScape.MinusLog()) # should raise error, as C[1,2]<0
end
@testset "Avoid NaNs when Z has tiny values" begin
@@ -917,5 +855,5 @@ end
costs=sparse(
[2, 3, 1, 4, 1, 4, 2, 3],
[1, 1, 2, 2, 3, 3, 4, 4],
- [1.0, 1, 1, 1, 1, 1, 1, 1]))
+ [1.0, 1, 1, 1, 1, 1, 1, 1]); check=true)
end
diff --git a/test/windowed.jl b/test/windowed.jl
new file mode 100644
index 0000000..d88ae58
--- /dev/null
+++ b/test/windowed.jl
@@ -0,0 +1,214 @@
+using ConScape, Test, SparseArrays, LinearAlgebra
+using Rasters, ArchGDAL
+using ConScape.LinearSolve
+
+datadir = joinpath(dirname(pathof(ConScape)), "..", "data")
+_tempdir = mkdir(tempname())
+
+θ = 0.1
+landscape = "sno_2000"
+# The way the ascii is read in is reversed and rotated from what GDAL does
+affinities = reverse(rotr90(replace_missing(Raster(joinpath(datadir, "affinities_$landscape.asc")), NaN)); dims=X)
+qualities = reverse(rotr90(replace_missing(Raster(joinpath(datadir, "qualities_$landscape.asc")), NaN)); dims=X)
+qualities[(affinities .> 0) .& isnan.(qualities)] .= 1e-20
+rast = RasterStack((; affinities, qualities, target_qualities=qualities))
+
+affinities_asc = ConScape.readasc(joinpath(datadir, "affinities_$landscape.asc"))[1]
+qualities_asc = ConScape.readasc(joinpath(datadir, "qualities_$landscape.asc"))[1]
+qualities_asc[(affinities_asc .> 0) .& isnan.(qualities_asc)] .= 1e-20
+
+graph_measures = (;
+# betq=ConScape.BetweennessQweighted(),
+ betk=ConScape.BetweennessKweighted(),
+ ch=ConScape.ConnectedHabitat(),
+ # # TODO sens=ConScape.Sensitivity(),
+ # crit=ConScape.Criticality(), # very very slow, each target makes a new grid
+)
+# Set low alpha here so the decay is steep for testing
+distance_transformation = x -> exp(-x / 2)
+connectivity_measure = ConScape.ExpectedCost(; θ, distance_transformation)
+expected_layers = (:betk, :ch)
+
+solver = ConScape.MatrixSolver()
+# solver = ConScape.VectorSolver()
+problem = ConScape.Problem(; graph_measures, connectivity_measure, solver)
+solve(problem, rast; verbose=true)
+
+@testset "target mosaicing matches original" begin
+ windowed_problem = ConScape.WindowedProblem(problem;
+ buffer=10, centersize=5, threaded=true,
+ )
+ @test collect(ConScape._window_ranges(windowed_problem, rast)) == [
+ (1:25, 1:25) (1:25, 6:30) (1:25, 11:35) (1:25, 16:40) (1:25, 21:45) (1:25, 26:50) (1:25, 31:55) (1:25, 36:59)
+ (6:30, 1:25) (6:30, 6:30) (6:30, 11:35) (6:30, 16:40) (6:30, 21:45) (6:30, 26:50) (6:30, 31:55) (6:30, 36:59)
+ (11:35, 1:25) (11:35, 6:30) (11:35, 11:35) (11:35, 16:40) (11:35, 21:45) (11:35, 26:50) (11:35, 31:55) (11:35, 36:59)
+ (16:40, 1:25) (16:40, 6:30) (16:40, 11:35) (16:40, 16:40) (16:40, 21:45) (16:40, 26:50) (16:40, 31:55) (16:40, 36:59)
+ (21:44, 1:25) (21:44, 6:30) (21:44, 11:35) (21:44, 16:40) (21:44, 21:45) (21:44, 26:50) (21:44, 31:55) (21:44, 36:59)
+ ]
+ test_results = ConScape.solve(windowed_problem, rast; test_windows=true)
+ inner_targets = copy(rast.target_qualities)
+ replace!(inner_targets, NaN => 0.0)
+ # Edge targets are lost with windowing
+ inner_targets[1:10, :] .= 0
+ inner_targets[:, 1:10] .= 0
+ inner_targets[end-9:end, :] .= 0
+ inner_targets[:, end-9:end] .= 0
+ @test parent(inner_targets) == parent(test_results.target_qualities)
+end
+
+@testset "windowed results approximate non-windowed" begin
+ buffer=15
+ windowed_problem = ConScape.WindowedProblem(problem;
+ buffer, centersize=5
+ )
+ mask!(rast; with=rast)
+ rast_inner = ConScape._get_window_with_zeroed_buffer(windowed_problem, rast, axes(rast))
+ @time wp_result = ConScape.solve(windowed_problem, rast)
+ @time p_result = ConScape.solve(problem, rast_inner)
+ p_result
+ # plot(p_result)
+ # plot(wp_result)
+ @test maplayers(p_result, wp_result) do P, WP
+ broadcast(P, WP) do p, wp
+ isnan(p) && isnan(wp) || isapprox(p, wp; atol=1e-4)
+ end |> all
+ end |> all
+end
+
+
+# BatchProblem writes files to disk and mosaics to RasterStack
+@testset "batch problem matches windowed problem" begin
+ solver = ConScape.VectorSolver()
+ # Use a higher alpha to catch differences
+ distance_transformation = x -> exp(-x / 50)
+ connectivity_measure = ConScape.ExpectedCost(; θ, distance_transformation)
+ problem = ConScape.Problem(; graph_measures, connectivity_measure, solver)
+
+ kw = (; buffer=10, centersize=5)
+ windowed_problem = ConScape.WindowedProblem(problem; kw...)
+ @time workspace = ConScape.init(windowed_problem, rast);
+ @time windowed_result = ConScape.solve!(workspace, windowed_problem);
+
+ batch_problem = ConScape.BatchProblem(problem; datapath=tempname(), kw...)
+ ConScape.solve(batch_problem, rast)
+ batch_result = mosaic(batch_problem; to=rast)
+ @test batch_result isa RasterStack
+
+ # BatchProblem can be run as batch jobs for clusters
+ # We just need a new path to make sure the result is from a new run
+ batch_jobs_problem = ConScape.BatchProblem(problem;
+ datapath=tempname(), kw...
+ )
+ assessment = ConScape.assess(batch_jobs_problem, rast)
+ batch_jobs_problem.centersize
+ @test assessment.njobs == 39
+
+ for job in 1:assessment.njobs
+ ConScape.solve(batch_jobs_problem, rast, assessment, job)
+ end
+ batch_jobs_result = mosaic(batch_jobs_problem; to=rast)
+ batch_jobs_result.betk
+
+
+ @testset "reassessment" begin
+ # There should be no jobs left
+ re1 = ConScape.reassess(batch_jobs_problem, assessment)
+ @test re1.njobs == 0
+ @test length(re1.indices) == 0
+
+ # Delete three results
+ paths = ConScape.batch_paths(batch_jobs_problem, size(assessment))
+ rm.(paths[[1, 7, 21]]; recursive=true)
+ re2 = ConScape.reassess(batch_jobs_problem, assessment)
+ @test re2.njobs == 3
+ @test length(re2.indices) == 3
+ @test re2.mask[[1, 7, 21]] == [true, true, true]
+
+ # Run the reassessment
+ for job in 1:re2.njobs
+ ConScape.solve(batch_jobs_problem, rast, re2, job)
+ end
+
+ # Again there are no jobs left
+ re3 = ConScape.reassess(batch_jobs_problem, assessment)
+ @test re3.njobs == 0
+ @test length(re3.indices) == 0
+ @test count(re3.mask) == 0
+ end
+
+
+ nested_problem = ConScape.BatchProblem(windowed_problem;
+ datapath=tempname(), centersize=(10, 10)
+ )
+ ConScape.assess(nested_problem, rast)
+ ConScape.solve(nested_problem, rast)
+ nested_result = mosaic(nested_problem; to=rast)
+ @test nested_result isa RasterStack
+
+ nested_jobs_problem = ConScape.BatchProblem(windowed_problem;
+ datapath=tempname(), centersize=(10, 10)
+ )
+ # Try one
+ @time workspace = ConScape.init(nested_jobs_problem, rast)
+ @time ConScape.solve!(workspace, nested_jobs_problem, 5)
+
+ assessment = ConScape.assess(nested_jobs_problem, rast);
+ for job in 1:assessment.njobs
+ ConScape.solve(nested_jobs_problem, rast, job)
+ end
+ nested_jobs_result = mosaic(nested_jobs_problem; to=rast)
+
+ @testset "nested reassessment" begin
+ # There should be no jobs left
+ re1 = ConScape.reassess(nested_jobs_problem, assessment)
+ @test re1.njobs == 0
+ @test length(re1.indices) == 0
+
+ # Delete three results
+ paths = ConScape.batch_paths(nested_jobs_problem, size(assessment))
+ rm.(paths[[2, 5]]; recursive=true)
+ re2 = ConScape.reassess(nested_jobs_problem, assessment)
+ @test re2.njobs == 2
+ @test length(re2.indices) == 2
+ @test re2.mask[[2, 5]] == [true, true]
+ re2
+ # Run the reassessment
+ for job in 1:re2.njobs
+ ConScape.solve(nested_jobs_problem, rast, re2, job)
+ end
+
+ # Again there are no jobs left
+ re3 = ConScape.reassess(nested_jobs_problem, assessment)
+ @test re3.njobs == 0
+ @test length(re3.indices) == 0
+ @test count(re3.mask) == 0
+ end
+
+ @test keys(windowed_result) ==
+ keys(nested_result) ==
+ keys(batch_result) ==
+ keys(batch_jobs_result) ==
+ keys(nested_jobs_result) ==
+ Tuple(sort(collect(expected_layers)))
+
+ # These may be approximate after mosaic order changes
+ compare(a, b) = isnan(a) && isnan(b) || isapprox(a, b)
+
+ @test all(batch_jobs_result.ch .=== batch_result.ch)
+ @test all(batch_jobs_result.betk .=== batch_result.betk)
+ @test all(compare.(permutedims(batch_result.ch), windowed_result.ch))
+ @test all(compare.(permutedims(batch_result.betk), windowed_result.betk))
+ @test all(compare.(nested_result.betk, nested_jobs_result.betk))
+ @test all(compare.(nested_result.ch, nested_jobs_result.ch))
+
+ # TODO: there are some tiny fp differences in the nested result
+ @test all(map(nested_result.ch, batch_result.ch) do n, b
+ isnan(n) && isnan(b) || isapprox(n, b)
+ end)
+
+ # plot(windowed_result)
+ # plot(batch_result)
+ # plot(batch_jobs_result)
+ # plot(nested_result)
+ # plot(nested_jobs_result)
+end
\ No newline at end of file