From 74929fcf9e2fe0521356c59a7ed0581b3095bb8d Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Sat, 9 Nov 2024 12:25:23 -0500 Subject: [PATCH 01/55] set up pr From ca6f71652e44293de11b534482c659e8fca16221 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Sat, 9 Nov 2024 12:27:47 -0500 Subject: [PATCH 02/55] refactor modules + set up graph lib --- src/lib.rs | 4 ++++ src/lowtime_graph.rs | 22 ++++++++++++++++++++++ src/phillips_dessouky.rs | 26 +++++++------------------- src/utils.rs | 15 +++++++++++++++ 4 files changed, 48 insertions(+), 19 deletions(-) create mode 100644 src/lowtime_graph.rs create mode 100644 src/utils.rs diff --git a/src/lib.rs b/src/lib.rs index ae23248..058cebf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,11 @@ use pyo3::prelude::*; +mod utils; mod phillips_dessouky; +mod lowtime_graph; + use phillips_dessouky::PhillipsDessouky; +use lowtime_graph::LowtimeGraph; #[pymodule] fn _lowtime_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs new file mode 100644 index 0000000..4628646 --- /dev/null +++ b/src/lowtime_graph.rs @@ -0,0 +1,22 @@ +use pyo3::prelude::*; + +use ordered_float::OrderedFloat; + +use std::time::Instant; +use log::info; + +use crate::utils; + + +#[pyclass] +pub struct LowtimeGraph { +} + +#[pymethods] +impl LowtimeGraph { + #[new] + fn new( + ) -> PyResult { + Ok(LowtimeGraph {}) + } +} diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index 896b46f..29e4451 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -10,6 +10,8 @@ use pathfinding::directed::edmonds_karp::{ use std::time::Instant; use log::info; +use crate::utils; + #[pyclass] pub struct PhillipsDessouky { @@ -42,7 +44,7 @@ impl PhillipsDessouky { ((*from, *to), OrderedFloat(*cap)) }).collect(); let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow scaling to OrderedFloat time: {:.10}s", PhillipsDessouky::profile_duration(profiling_start, profiling_end)); + info!("PROFILING Rust_PhillipsDessouky::max_flow scaling to OrderedFloat time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); let profiling_start = Instant::now(); let (flows, _max_flow, _min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( @@ -52,29 +54,15 @@ impl PhillipsDessouky { edges_edmonds_karp, ); let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", PhillipsDessouky::profile_duration(profiling_start, profiling_end)); + info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); let profiling_start = Instant::now(); - let flows_f64: Vec<((u32, u32), f64)> = flows.into_iter().map(|(edge, flow)| { - (edge, flow.into_inner()) + let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { + ((*from, *to), flow.into_inner()) }).collect(); let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow reformat OrderedFloat to f64 time: {:.10}s", PhillipsDessouky::profile_duration(profiling_start, profiling_end)); + info!("PROFILING Rust_PhillipsDessouky::max_flow reformat OrderedFloat to f64 time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); Ok(flows_f64) } } - -// Private to Rust, not exposed to Python -impl PhillipsDessouky { - fn profile_duration(start: Instant, end: Instant) -> f64 { - let duration = end.duration_since(start); - let seconds = duration.as_secs(); - let subsec_nanos = duration.subsec_nanos(); - - let fractional_seconds = subsec_nanos as f64 / 1_000_000_000.0; - let total_seconds = seconds as f64 + fractional_seconds; - - return total_seconds; - } -} diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..d22a298 --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,15 @@ +use std::time::{ + Instant, + Duration, +}; + +pub fn profile_duration(start: Instant, end: Instant) -> f64 { + let duration: Duration = end.duration_since(start); + let seconds = duration.as_secs(); + let subsec_nanos = duration.subsec_nanos(); + + let fractional_seconds = subsec_nanos as f64 / 1_000_000_000.0; + let total_seconds = seconds as f64 + fractional_seconds; + + return total_seconds; +} From 293579d738d5f2dba45ab7404cac02e3633591a5 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 11 Nov 2024 11:59:21 -0500 Subject: [PATCH 03/55] graph updates --- src/lowtime_graph.rs | 49 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index 4628646..e473738 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -1,5 +1,6 @@ use pyo3::prelude::*; +use std::collections::HashMap; use ordered_float::OrderedFloat; use std::time::Instant; @@ -8,15 +9,51 @@ use log::info; use crate::utils; -#[pyclass] pub struct LowtimeGraph { + edges: HashMap>, + preds: HashMap, } -#[pymethods] impl LowtimeGraph { - #[new] - fn new( - ) -> PyResult { - Ok(LowtimeGraph {}) + pub fn new() -> Self { + LowtimeGraph { + edges: HashMap::new(), + preds: HashMap::new(), + } } + + pub fn get_nodes(&self) -> Vec { + let mut nodes: Vec = self.edges.keys().cloned().collect(); + nodes.sort(); + nodes + } + + pub fn add_edge(&mut self, from: u32, to: u32, op: Operation) -> () { + self.edges + .entry(from) + .or_insert_with(HashMap::new) + .insert(to, op); + self.preds.insert(to, from); + } + + pub fn get_op(&self, from: u32, to: u32) -> Option<&Operation> { + self.edges + .get(&from) + .and_then(|ops| ops.get(&to)) + } + + pub fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut Operation> { + self.edges + .get_mut(&from) + .and_then(|ops| ops.get_mut(&to)) + } + + +} + +struct Operation { + capacity: OrderedFloat, + flow: OrderedFloat, + ub: OrderedFloat, + lb: OrderedFloat, } From b115396ab93c339869293511e526d32d774a9ecd Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 11 Nov 2024 12:49:25 -0500 Subject: [PATCH 04/55] operation and costmodel --- src/lib.rs | 7 +++-- src/lowtime_graph.rs | 28 +++++++----------- src/operation.rs | 67 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 21 deletions(-) create mode 100644 src/operation.rs diff --git a/src/lib.rs b/src/lib.rs index 058cebf..e55e5e5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,12 @@ use pyo3::prelude::*; -mod utils; -mod phillips_dessouky; mod lowtime_graph; +mod operation; +mod phillips_dessouky; +mod utils; use phillips_dessouky::PhillipsDessouky; -use lowtime_graph::LowtimeGraph; + #[pymodule] fn _lowtime_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index e473738..e8e85ad 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -1,20 +1,21 @@ -use pyo3::prelude::*; - use std::collections::HashMap; -use ordered_float::OrderedFloat; use std::time::Instant; use log::info; +use crate::operation::{Operation, CostModel}; use crate::utils; -pub struct LowtimeGraph { - edges: HashMap>, +pub struct LowtimeGraph { + edges: HashMap>>, preds: HashMap, } -impl LowtimeGraph { +impl LowtimeGraph +where + C: CostModel +{ pub fn new() -> Self { LowtimeGraph { edges: HashMap::new(), @@ -28,7 +29,7 @@ impl LowtimeGraph { nodes } - pub fn add_edge(&mut self, from: u32, to: u32, op: Operation) -> () { + pub fn add_edge(&mut self, from: u32, to: u32, op: Operation) -> () { self.edges .entry(from) .or_insert_with(HashMap::new) @@ -36,24 +37,15 @@ impl LowtimeGraph { self.preds.insert(to, from); } - pub fn get_op(&self, from: u32, to: u32) -> Option<&Operation> { + pub fn get_op(&self, from: u32, to: u32) -> Option<&Operation> { self.edges .get(&from) .and_then(|ops| ops.get(&to)) } - pub fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut Operation> { + pub fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut Operation> { self.edges .get_mut(&from) .and_then(|ops| ops.get_mut(&to)) } - - -} - -struct Operation { - capacity: OrderedFloat, - flow: OrderedFloat, - ub: OrderedFloat, - lb: OrderedFloat, } diff --git a/src/operation.rs b/src/operation.rs new file mode 100644 index 0000000..c4dc4e5 --- /dev/null +++ b/src/operation.rs @@ -0,0 +1,67 @@ +use ordered_float::OrderedFloat; + + +pub trait CostModel { + fn get_cost(&self, duration: u32) -> f64; +} + +pub struct Operation { + capacity: OrderedFloat, + flow: OrderedFloat, + ub: OrderedFloat, + lb: OrderedFloat, + cost_model: C, +} + +impl Operation +where + C: CostModel, +{ + fn new(capacity: f64, flow: f64, ub: f64, lb: f64, cost_model: C) -> Self { + Operation { + capacity: OrderedFloat(capacity), + flow: OrderedFloat(flow), + ub: OrderedFloat(ub), + lb: OrderedFloat(lb), + cost_model, + } + } + + fn get_capacity(&self) -> OrderedFloat { + self.capacity + } + + fn get_flow(&self) -> OrderedFloat { + self.flow + } + + fn get_ub(&self) -> OrderedFloat { + self.ub + } + + fn get_lb(&self) -> OrderedFloat { + self.lb + } + + fn get_cost(&self, duration: u32) -> f64 { + self.cost_model.get_cost(duration) + } +} + +struct ExponentialModel { + a: f64, + b: f64, + c: f64, +} + +impl ExponentialModel { + fn new(a: f64, b: f64, c: f64) -> Self { + ExponentialModel {a, b, c} + } +} + +impl CostModel for ExponentialModel { + fn get_cost(&self, duration: u32) -> f64 { + self.a * f64::exp(self.b * duration as f64) + self.c + } +} \ No newline at end of file From 8e7b7873a7b05e86838596111fb096c34726c911 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 11 Nov 2024 12:50:51 -0500 Subject: [PATCH 05/55] note on caching --- src/operation.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/operation.rs b/src/operation.rs index c4dc4e5..784cf77 100644 --- a/src/operation.rs +++ b/src/operation.rs @@ -52,6 +52,8 @@ struct ExponentialModel { a: f64, b: f64, c: f64, + // TODO(ohjun): consider impact of manual caching + // cache: HashMap } impl ExponentialModel { From 3bd85de1933a5d58d3d719df1d8a41b43f82e6e8 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 11 Nov 2024 13:14:23 -0500 Subject: [PATCH 06/55] rename to ids --- src/lowtime_graph.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index e8e85ad..d0b3980 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -23,10 +23,10 @@ where } } - pub fn get_nodes(&self) -> Vec { - let mut nodes: Vec = self.edges.keys().cloned().collect(); - nodes.sort(); - nodes + pub fn get_node_ids(&self) -> Vec { + let mut node_ids: Vec = self.edges.keys().cloned().collect(); + node_ids.sort(); + node_ids } pub fn add_edge(&mut self, from: u32, to: u32, op: Operation) -> () { From 86844c8e904424100fc5cbf53d1da7f723882ee9 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 11 Nov 2024 13:41:19 -0500 Subject: [PATCH 07/55] edit operation fields --- src/operation.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/operation.rs b/src/operation.rs index 784cf77..b26e08a 100644 --- a/src/operation.rs +++ b/src/operation.rs @@ -8,8 +8,10 @@ pub trait CostModel { pub struct Operation { capacity: OrderedFloat, flow: OrderedFloat, - ub: OrderedFloat, - lb: OrderedFloat, + // TODO(ohjun): should we have a separate struct for node values above and operation values? + ub: u32, + lb: u32, + duration: u32, cost_model: C, } @@ -17,12 +19,13 @@ impl Operation where C: CostModel, { - fn new(capacity: f64, flow: f64, ub: f64, lb: f64, cost_model: C) -> Self { + fn new(capacity: f64, flow: f64, ub: u32, lb: u32, duration: u32, cost_model: C) -> Self { Operation { capacity: OrderedFloat(capacity), flow: OrderedFloat(flow), - ub: OrderedFloat(ub), - lb: OrderedFloat(lb), + ub, + lb, + duration, cost_model, } } @@ -35,14 +38,18 @@ where self.flow } - fn get_ub(&self) -> OrderedFloat { + fn get_ub(&self) -> u32 { self.ub } - fn get_lb(&self) -> OrderedFloat { + fn get_lb(&self) -> u32 { self.lb } + fn get_duration(&self) -> u32 { + self.duration + } + fn get_cost(&self, duration: u32) -> f64 { self.cost_model.get_cost(duration) } From b5911fc1422678364e3a19f30d9e7235edebc18b Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 11 Nov 2024 13:42:52 -0500 Subject: [PATCH 08/55] floats instead of int --- src/operation.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/operation.rs b/src/operation.rs index b26e08a..d122c7a 100644 --- a/src/operation.rs +++ b/src/operation.rs @@ -8,9 +8,9 @@ pub trait CostModel { pub struct Operation { capacity: OrderedFloat, flow: OrderedFloat, + ub: OrderedFloat, + lb: OrderedFloat, // TODO(ohjun): should we have a separate struct for node values above and operation values? - ub: u32, - lb: u32, duration: u32, cost_model: C, } @@ -19,12 +19,12 @@ impl Operation where C: CostModel, { - fn new(capacity: f64, flow: f64, ub: u32, lb: u32, duration: u32, cost_model: C) -> Self { + fn new(capacity: f64, flow: f64, ub: f64, lb: f64, duration: u32, cost_model: C) -> Self { Operation { capacity: OrderedFloat(capacity), flow: OrderedFloat(flow), - ub, - lb, + ub: OrderedFloat(ub), + lb: OrderedFloat(lb), duration, cost_model, } @@ -38,11 +38,11 @@ where self.flow } - fn get_ub(&self) -> u32 { + fn get_ub(&self) -> OrderedFloat { self.ub } - fn get_lb(&self) -> u32 { + fn get_lb(&self) -> OrderedFloat { self.lb } From 7a575c9a72950a4817f6d37d2ff80a3e3ced0061 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Thu, 14 Nov 2024 14:09:17 -0500 Subject: [PATCH 09/55] not working; cost model issues, making generic might be hard with interop --- lowtime/_lowtime_rs.pyi | 2 ++ src/cost_model.rs | 60 +++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/lowtime_graph.rs | 32 +++++++++++------ src/operation.rs | 36 +++---------------- src/phillips_dessouky.bak.rs | 70 ++++++++++++++++++++++++++++++++++++ src/phillips_dessouky.rs | 64 +++++++++++++++++---------------- 7 files changed, 193 insertions(+), 72 deletions(-) create mode 100644 src/cost_model.rs create mode 100644 src/phillips_dessouky.bak.rs diff --git a/lowtime/_lowtime_rs.pyi b/lowtime/_lowtime_rs.pyi index f5b7270..e9b7e0a 100644 --- a/lowtime/_lowtime_rs.pyi +++ b/lowtime/_lowtime_rs.pyi @@ -11,3 +11,5 @@ class PhillipsDessouky: edges_raw: list[tuple[tuple[int, int], float]], ) -> None: ... def max_flow(self) -> list[tuple[tuple[int, int], float]]: ... + +# TODO(ohjun): add CostModel interface diff --git a/src/cost_model.rs b/src/cost_model.rs new file mode 100644 index 0000000..9eac491 --- /dev/null +++ b/src/cost_model.rs @@ -0,0 +1,60 @@ +use pyo3::prelude::*; + +use std::collections::HashMap; + + +pub trait ModelCost { + fn get_cost(&self, duration: u32) -> f64; +} + +#[pyclass] +pub struct CostModel { + cache: HashMap, + inner: Box, +} + +#[pymethods] +impl CostModel { + pub fn new(inner: Box) -> PyResult { + Ok(CostModel { inner, cache: HashMap::new() }) + } + + pub fn get_cost(&mut self, duration: u32) -> f64 { + // match self.cache.entry(duration) { + // HashMap::Entry::Occupied => { + // self.cache.get(&duration)?.clone() + // } + // HashMap::Entry::Empty => { + // let cost = self.inner.get_cost(duration); + // self.cache.insert(duration, cost); + // cost + // } + // } + match self.cache.get(&duration) { + Some(cost) => *cost, + None => { + let cost = self.inner.get_cost(duration); + self.cache.insert(duration, cost); + cost + } + } + } +} + +struct ExponentialModel { + a: f64, + b: f64, + c: f64, +} + +impl ExponentialModel { + fn new(a: f64, b: f64, c: f64) -> Self { + ExponentialModel {a, b, c} + } +} + +impl ModelCost for ExponentialModel { + fn get_cost(&self, duration: u32) -> f64 { + self.a * f64::exp(self.b * duration as f64) + self.c + } +} diff --git a/src/lib.rs b/src/lib.rs index e55e5e5..77633af 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ use pyo3::prelude::*; +mod cost_model; mod lowtime_graph; mod operation; mod phillips_dessouky; diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index d0b3980..0c644c6 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -3,19 +3,16 @@ use std::collections::HashMap; use std::time::Instant; use log::info; -use crate::operation::{Operation, CostModel}; +use crate::operation::Operation; use crate::utils; -pub struct LowtimeGraph { - edges: HashMap>>, +pub struct LowtimeGraph { + edges: HashMap>, preds: HashMap, } -impl LowtimeGraph -where - C: CostModel -{ +impl LowtimeGraph { pub fn new() -> Self { LowtimeGraph { edges: HashMap::new(), @@ -23,13 +20,28 @@ where } } + pub fn of_python( + node_ids: Vec, + source_node_id: u32, + sink_node_id: u32, + edges: Vec<((u32, u32), f64)>, + ) -> Self { + let graph = LowtimeGraph::new(); + // edges.iter().map(|((from, to), capacity)| { + // let cost_model = C::new(0, 0, 0); + // let op = Operation::new(capacity, 0.0, 0.0, 0.0, 0, cost_model); + // graph.add_edge(from, to, op) + // }); + graph + } + pub fn get_node_ids(&self) -> Vec { let mut node_ids: Vec = self.edges.keys().cloned().collect(); node_ids.sort(); node_ids } - pub fn add_edge(&mut self, from: u32, to: u32, op: Operation) -> () { + pub fn add_edge(&mut self, from: u32, to: u32, op: Operation) -> () { self.edges .entry(from) .or_insert_with(HashMap::new) @@ -37,13 +49,13 @@ where self.preds.insert(to, from); } - pub fn get_op(&self, from: u32, to: u32) -> Option<&Operation> { + pub fn get_op(&self, from: u32, to: u32) -> Option<&Operation> { self.edges .get(&from) .and_then(|ops| ops.get(&to)) } - pub fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut Operation> { + pub fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut Operation> { self.edges .get_mut(&from) .and_then(|ops| ops.get_mut(&to)) diff --git a/src/operation.rs b/src/operation.rs index d122c7a..fadb56c 100644 --- a/src/operation.rs +++ b/src/operation.rs @@ -1,24 +1,18 @@ use ordered_float::OrderedFloat; +use crate::cost_model::CostModel; -pub trait CostModel { - fn get_cost(&self, duration: u32) -> f64; -} -pub struct Operation { +pub struct Operation { capacity: OrderedFloat, flow: OrderedFloat, ub: OrderedFloat, lb: OrderedFloat, - // TODO(ohjun): should we have a separate struct for node values above and operation values? duration: u32, - cost_model: C, + cost_model: CostModel, } -impl Operation -where - C: CostModel, -{ +impl Operation { fn new(capacity: f64, flow: f64, ub: f64, lb: f64, duration: u32, cost_model: C) -> Self { Operation { capacity: OrderedFloat(capacity), @@ -50,27 +44,7 @@ where self.duration } - fn get_cost(&self, duration: u32) -> f64 { + fn get_cost(&mut self, duration: u32) -> f64 { self.cost_model.get_cost(duration) } } - -struct ExponentialModel { - a: f64, - b: f64, - c: f64, - // TODO(ohjun): consider impact of manual caching - // cache: HashMap -} - -impl ExponentialModel { - fn new(a: f64, b: f64, c: f64) -> Self { - ExponentialModel {a, b, c} - } -} - -impl CostModel for ExponentialModel { - fn get_cost(&self, duration: u32) -> f64 { - self.a * f64::exp(self.b * duration as f64) + self.c - } -} \ No newline at end of file diff --git a/src/phillips_dessouky.bak.rs b/src/phillips_dessouky.bak.rs new file mode 100644 index 0000000..bb5b1b2 --- /dev/null +++ b/src/phillips_dessouky.bak.rs @@ -0,0 +1,70 @@ +use pyo3::prelude::*; + +use ordered_float::OrderedFloat; +use pathfinding::directed::edmonds_karp::{ + SparseCapacity, + Edge, + edmonds_karp +}; + +use std::time::Instant; +use log::info; + +use crate::lowtime_graph::LowtimeGraph; +use crate::operation::{Operation, CostModel}; +use crate::utils; + + +#[pyclass] +pub struct PhillipsDessouky { + node_ids: Vec, + source_node_id: u32, + sink_node_id: u32, + edges_raw: Vec<((u32, u32), f64)>, +} + +#[pymethods] +impl PhillipsDessouky { + #[new] + fn new( + node_ids: Vec, + source_node_id: u32, + sink_node_id: u32, + edges_raw: Vec<((u32, u32), f64)>, + ) -> PyResult { + Ok(PhillipsDessouky { + node_ids, + source_node_id, + sink_node_id, + edges_raw, + }) + } + + fn max_flow(&self) -> PyResult> { + let profiling_start = Instant::now(); + let edges_edmonds_karp: Vec>> = self.edges_raw.iter().map(|((from, to), cap)| { + ((*from, *to), OrderedFloat(*cap)) + }).collect(); + let profiling_end = Instant::now(); + info!("PROFILING Rust_PhillipsDessouky::max_flow scaling to OrderedFloat time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); + + let profiling_start = Instant::now(); + let (flows, _max_flow, _min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( + &self.node_ids, + &self.source_node_id, + &self.sink_node_id, + edges_edmonds_karp, + ); + let profiling_end = Instant::now(); + info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); + + let profiling_start = Instant::now(); + let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { + ((*from, *to), flow.into_inner()) + }).collect(); + let profiling_end = Instant::now(); + info!("PROFILING Rust_PhillipsDessouky::max_flow reformat OrderedFloat to f64 time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); + + Ok(flows_f64) + } +} diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index 29e4451..42c48ce 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -10,15 +10,13 @@ use pathfinding::directed::edmonds_karp::{ use std::time::Instant; use log::info; +use crate::lowtime_graph::LowtimeGraph; use crate::utils; #[pyclass] pub struct PhillipsDessouky { - node_ids: Vec, - source_node_id: u32, - sink_node_id: u32, - edges_raw: Vec<((u32, u32), f64)>, + graph: LowtimeGraph, } #[pymethods] @@ -28,41 +26,45 @@ impl PhillipsDessouky { node_ids: Vec, source_node_id: u32, sink_node_id: u32, - edges_raw: Vec<((u32, u32), f64)>, + edges: Vec<((u32, u32), f64)>, ) -> PyResult { Ok(PhillipsDessouky { - node_ids, - source_node_id, - sink_node_id, - edges_raw, + graph: LowtimeGraph::of_python( + node_ids, + source_node_id, + sink_node_id, + edges, + ) }) } fn max_flow(&self) -> PyResult> { - let profiling_start = Instant::now(); - let edges_edmonds_karp: Vec>> = self.edges_raw.iter().map(|((from, to), cap)| { - ((*from, *to), OrderedFloat(*cap)) - }).collect(); - let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow scaling to OrderedFloat time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); + // TODO(ohjun): temporary for compile check + Ok(Vec::new()) + // let profiling_start = Instant::now(); + // let edges_edmonds_karp: Vec>> = self.edges_raw.iter().map(|((from, to), cap)| { + // ((*from, *to), OrderedFloat(*cap)) + // }).collect(); + // let profiling_end = Instant::now(); + // info!("PROFILING Rust_PhillipsDessouky::max_flow scaling to OrderedFloat time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - let profiling_start = Instant::now(); - let (flows, _max_flow, _min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( - &self.node_ids, - &self.source_node_id, - &self.sink_node_id, - edges_edmonds_karp, - ); - let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); + // let profiling_start = Instant::now(); + // let (flows, _max_flow, _min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( + // &self.node_ids, + // &self.source_node_id, + // &self.sink_node_id, + // edges_edmonds_karp, + // ); + // let profiling_end = Instant::now(); + // info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - let profiling_start = Instant::now(); - let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { - ((*from, *to), flow.into_inner()) - }).collect(); - let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow reformat OrderedFloat to f64 time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); + // let profiling_start = Instant::now(); + // let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { + // ((*from, *to), flow.into_inner()) + // }).collect(); + // let profiling_end = Instant::now(); + // info!("PROFILING Rust_PhillipsDessouky::max_flow reformat OrderedFloat to f64 time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - Ok(flows_f64) + // Ok(flows_f64) } } From cd5c43b8dcc4ab043d0e61ccac0211393718105a Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Thu, 14 Nov 2024 15:01:57 -0500 Subject: [PATCH 10/55] wip cost model, issues with creating generic constructor --- src/cost_model.rs | 43 +++++++++++++++++++++++++++++-------------- src/operation.rs | 2 +- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/cost_model.rs b/src/cost_model.rs index 9eac491..2979a0f 100644 --- a/src/cost_model.rs +++ b/src/cost_model.rs @@ -9,27 +9,42 @@ pub trait ModelCost { #[pyclass] pub struct CostModel { + inner: Box, cache: HashMap, - inner: Box, } #[pymethods] impl CostModel { - pub fn new(inner: Box) -> PyResult { - Ok(CostModel { inner, cache: HashMap::new() }) - } + // Issues with traits of input type + // #[new] + // pub fn new(model: Box) -> PyResult { + // Ok(CostModel { + // inner: model, + // cache: HashMap::new(), + // }) + // } + + // Python functions cannot have generic type parameters + // #[new] + // pub fn new(model: C) -> PyResult + // where + // C: ModelCost + Send + // { + // Ok(CostModel { + // inner: Box::new(model), + // cache: HashMap::new(), + // }) + // } + + // Some error message about f64 not implementing some convoluted trait + // pub fn new_exponential(a: f64, b: f64, c: f64) -> PyResult { + // Ok(CostModel { + // inner: Box::new(ExponentialModel::new(a, b, c)), + // cache: HashMap::new(), + // }) + // } pub fn get_cost(&mut self, duration: u32) -> f64 { - // match self.cache.entry(duration) { - // HashMap::Entry::Occupied => { - // self.cache.get(&duration)?.clone() - // } - // HashMap::Entry::Empty => { - // let cost = self.inner.get_cost(duration); - // self.cache.insert(duration, cost); - // cost - // } - // } match self.cache.get(&duration) { Some(cost) => *cost, None => { diff --git a/src/operation.rs b/src/operation.rs index fadb56c..4c5a07c 100644 --- a/src/operation.rs +++ b/src/operation.rs @@ -13,7 +13,7 @@ pub struct Operation { } impl Operation { - fn new(capacity: f64, flow: f64, ub: f64, lb: f64, duration: u32, cost_model: C) -> Self { + fn new(capacity: f64, flow: f64, ub: f64, lb: f64, duration: u32, cost_model: CostModel) -> Self { Operation { capacity: OrderedFloat(capacity), flow: OrderedFloat(flow), From 7a0901b3ff64cfcf7d1c65c51990535ce432d125 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 18 Nov 2024 19:18:37 -0500 Subject: [PATCH 11/55] working: minimal change with new PhillipsDessouky setup --- lowtime/_lowtime_rs.pyi | 8 ++- lowtime/solver.py | 41 ++++++++++-- src/lib.rs | 2 +- src/lowtime_graph.rs | 141 +++++++++++++++++++++++++++++++++++---- src/operation.rs | 71 ++++++++++---------- src/phillips_dessouky.rs | 34 ++-------- 6 files changed, 211 insertions(+), 86 deletions(-) diff --git a/lowtime/_lowtime_rs.pyi b/lowtime/_lowtime_rs.pyi index e9b7e0a..a593eca 100644 --- a/lowtime/_lowtime_rs.pyi +++ b/lowtime/_lowtime_rs.pyi @@ -10,6 +10,12 @@ class PhillipsDessouky: sink_node_id: int, edges_raw: list[tuple[tuple[int, int], float]], ) -> None: ... - def max_flow(self) -> list[tuple[tuple[int, int], float]]: ... + def max_flow(self) -> list[ + tuple[ + tuple[int, int], + tuple[float, float, float, float], + tuple[bool, int, int, int, int, int, int, int] | None, + ] + ]: ... # TODO(ohjun): add CostModel interface diff --git a/lowtime/solver.py b/lowtime/solver.py index ae034aa..7f07921 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -438,13 +438,44 @@ def find_min_cut(self, dag: nx.DiGraph) -> tuple[set[int], set[int]]: def format_rust_inputs( dag: nx.DiGraph, ) -> tuple[ - nx.classes.reportviews.NodeView, list[tuple[tuple[int, int], float]] + nx.classes.reportviews.NodeView, + list[ + tuple[ + tuple[int, int], + tuple[float, float, float, float], + tuple[bool, int, int, int, int, int, int, int] | None, + ] + ], ]: nodes = dag.nodes - edges = [ - ((u, v), cap) - for (u, v), cap in nx.get_edge_attributes(dag, "capacity").items() - ] + edges = [] + for from_, to_, edge_attrs in dag.edges(data=True): + op_details = ( + None + if self.attr_name not in edge_attrs + else ( + edge_attrs[self.attr_name].is_dummy, + edge_attrs[self.attr_name].duration, + edge_attrs[self.attr_name].max_duration, + edge_attrs[self.attr_name].min_duration, + edge_attrs[self.attr_name].earliest_start, + edge_attrs[self.attr_name].latest_start, + edge_attrs[self.attr_name].earliest_finish, + edge_attrs[self.attr_name].latest_finish, + ) + ) + edges.append( + ( + (from_, to_), + ( + edge_attrs["capacity"], + edge_attrs.get("flow", 0), + edge_attrs.get("ub", 0), + edge_attrs.get("lb", 0), + ), + op_details, + ) + ) return nodes, edges # Helper function for Rust interop diff --git a/src/lib.rs b/src/lib.rs index 77633af..a80a3c4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,6 @@ use pyo3::prelude::*; -mod cost_model; +// mod cost_model; mod lowtime_graph; mod operation; mod phillips_dessouky; diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index 0c644c6..bae4575 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -1,5 +1,12 @@ use std::collections::HashMap; +use ordered_float::OrderedFloat; +use pathfinding::directed::edmonds_karp::{ + SparseCapacity, + Edge, + edmonds_karp +}; + use std::time::Instant; use log::info; @@ -8,13 +15,19 @@ use crate::utils; pub struct LowtimeGraph { - edges: HashMap>, + node_ids: Vec, + source_node_id: u32, + sink_node_id: u32, + edges: HashMap>, preds: HashMap, } impl LowtimeGraph { pub fn new() -> Self { LowtimeGraph { + node_ids: Vec::new(), + source_node_id: u32::MAX, + sink_node_id: u32::MAX, edges: HashMap::new(), preds: HashMap::new(), } @@ -24,14 +37,39 @@ impl LowtimeGraph { node_ids: Vec, source_node_id: u32, sink_node_id: u32, - edges: Vec<((u32, u32), f64)>, + edges_raw: Vec<((u32, u32), (f64, f64, f64, f64), Option<(bool, u64, u64, u64, u64, u64, u64, u64)>)>, ) -> Self { - let graph = LowtimeGraph::new(); - // edges.iter().map(|((from, to), capacity)| { - // let cost_model = C::new(0, 0, 0); - // let op = Operation::new(capacity, 0.0, 0.0, 0.0, 0, cost_model); - // graph.add_edge(from, to, op) - // }); + let mut graph = LowtimeGraph::new(); + // TODO(ohjun): this is very bad. will make better after checking that minimal change version works. + graph.node_ids = node_ids; + graph.source_node_id = source_node_id; + graph.sink_node_id = sink_node_id; + edges_raw.iter().for_each(|( + (from, to), + (capacity, flow, ub, lb), + op_details, + )| { + let op = op_details.map(|( + is_dummy, + duration, + max_duration, + min_duration, + earliest_start, + latest_start, + earliest_finish, + latest_finish, + )| Operation::new( + is_dummy, + duration, + max_duration, + min_duration, + earliest_start, + latest_start, + earliest_finish, + latest_finish + )); + graph.add_edge(*from, *to, LowtimeEdge::new(op, *capacity, *flow, *ub, *lb)) + }); graph } @@ -41,23 +79,98 @@ impl LowtimeGraph { node_ids } - pub fn add_edge(&mut self, from: u32, to: u32, op: Operation) -> () { + pub fn add_edge(&mut self, from: u32, to: u32, edge: LowtimeEdge) -> () { self.edges .entry(from) .or_insert_with(HashMap::new) - .insert(to, op); + .insert(to, edge); self.preds.insert(to, from); } - pub fn get_op(&self, from: u32, to: u32) -> Option<&Operation> { + pub fn get_edge(&self, from: u32, to: u32) -> Option<&LowtimeEdge> { self.edges .get(&from) - .and_then(|ops| ops.get(&to)) + .and_then(|to_edges| to_edges.get(&to)) } - pub fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut Operation> { + pub fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut LowtimeEdge> { self.edges .get_mut(&from) - .and_then(|ops| ops.get_mut(&to)) + .and_then(|to_edges| to_edges.get_mut(&to)) + } + + pub fn max_flow(&self) -> Vec<((u32, u32), f64)> { + let profiling_start = Instant::now(); + let edges_edmonds_karp: Vec>> = self.edges.iter().flat_map(|(from, inner)| + inner.iter().map(|(to, edge)| ((*from, *to), edge.get_capacity())) + ).collect(); + let profiling_end = Instant::now(); + info!("PROFILING Rust_PhillipsDessouky::max_flow scaling to OrderedFloat time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); + + let profiling_start = Instant::now(); + let (flows, _max_flow, _min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( + &self.node_ids, + &self.source_node_id, + &self.sink_node_id, + edges_edmonds_karp, + ); + let profiling_end = Instant::now(); + info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); + + let profiling_start = Instant::now(); + let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { + ((*from, *to), flow.into_inner()) + }).collect(); + let profiling_end = Instant::now(); + info!("PROFILING Rust_PhillipsDessouky::max_flow reformat OrderedFloat to f64 time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); + + flows_f64 } } + +pub struct LowtimeEdge { + op: Option, + capacity: OrderedFloat, + flow: OrderedFloat, + ub: OrderedFloat, + lb: OrderedFloat, +} + +impl LowtimeEdge { + fn new( + op: Option, + capacity: f64, + flow: f64, + ub: f64, + lb: f64, + ) -> Self { + LowtimeEdge { + op, + capacity: OrderedFloat(capacity), + flow: OrderedFloat(flow), + ub: OrderedFloat(ub), + lb: OrderedFloat(lb), + } + } + + fn get_op(&self) -> Option<&Operation> { + self.op.as_ref() + } + + fn get_capacity(&self) -> OrderedFloat { + self.capacity + } + + fn get_flow(&self) -> OrderedFloat { + self.flow + } + + fn get_ub(&self) -> OrderedFloat { + self.ub + } + + fn get_lb(&self) -> OrderedFloat { + self.lb + } + +} \ No newline at end of file diff --git a/src/operation.rs b/src/operation.rs index 4c5a07c..9d14f36 100644 --- a/src/operation.rs +++ b/src/operation.rs @@ -1,50 +1,51 @@ -use ordered_float::OrderedFloat; - -use crate::cost_model::CostModel; +// use crate::cost_model::CostModel; pub struct Operation { - capacity: OrderedFloat, - flow: OrderedFloat, - ub: OrderedFloat, - lb: OrderedFloat, - duration: u32, - cost_model: CostModel, + is_dummy: bool, + + duration: u64, + max_duration: u64, + min_duration: u64, + + earliest_start: u64, + latest_start: u64, + earliest_finish: u64, + latest_finish: u64, + + // cost_model: CostModel, } impl Operation { - fn new(capacity: f64, flow: f64, ub: f64, lb: f64, duration: u32, cost_model: CostModel) -> Self { + pub fn new( + is_dummy: bool, + duration: u64, + max_duration: u64, + min_duration: u64, + earliest_start: u64, + latest_start: u64, + earliest_finish: u64, + latest_finish: u64, + // cost_model: CostModel, + ) -> Self { Operation { - capacity: OrderedFloat(capacity), - flow: OrderedFloat(flow), - ub: OrderedFloat(ub), - lb: OrderedFloat(lb), + is_dummy, duration, - cost_model, + max_duration, + min_duration, + earliest_start, + latest_start, + earliest_finish, + latest_finish, + // cost_model, } } - fn get_capacity(&self) -> OrderedFloat { - self.capacity - } - - fn get_flow(&self) -> OrderedFloat { - self.flow - } - - fn get_ub(&self) -> OrderedFloat { - self.ub - } - - fn get_lb(&self) -> OrderedFloat { - self.lb - } - - fn get_duration(&self) -> u32 { + pub fn get_duration(&self) -> u64 { self.duration } - fn get_cost(&mut self, duration: u32) -> f64 { - self.cost_model.get_cost(duration) - } + // fn get_cost(&mut self, duration: u32) -> f64 { + // self.cost_model.get_cost(duration) + // } } diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index 42c48ce..8a50308 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -26,45 +26,19 @@ impl PhillipsDessouky { node_ids: Vec, source_node_id: u32, sink_node_id: u32, - edges: Vec<((u32, u32), f64)>, + edges_raw: Vec<((u32, u32), (f64, f64, f64, f64), Option<(bool, u64, u64, u64, u64, u64, u64, u64)>)>, ) -> PyResult { Ok(PhillipsDessouky { graph: LowtimeGraph::of_python( node_ids, source_node_id, sink_node_id, - edges, + edges_raw, ) }) } - fn max_flow(&self) -> PyResult> { - // TODO(ohjun): temporary for compile check - Ok(Vec::new()) - // let profiling_start = Instant::now(); - // let edges_edmonds_karp: Vec>> = self.edges_raw.iter().map(|((from, to), cap)| { - // ((*from, *to), OrderedFloat(*cap)) - // }).collect(); - // let profiling_end = Instant::now(); - // info!("PROFILING Rust_PhillipsDessouky::max_flow scaling to OrderedFloat time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - - // let profiling_start = Instant::now(); - // let (flows, _max_flow, _min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( - // &self.node_ids, - // &self.source_node_id, - // &self.sink_node_id, - // edges_edmonds_karp, - // ); - // let profiling_end = Instant::now(); - // info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - - // let profiling_start = Instant::now(); - // let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { - // ((*from, *to), flow.into_inner()) - // }).collect(); - // let profiling_end = Instant::now(); - // info!("PROFILING Rust_PhillipsDessouky::max_flow reformat OrderedFloat to f64 time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - - // Ok(flows_f64) + fn max_flow(&self) -> Vec<((u32, u32), f64)> { + self.graph.max_flow() } } From 6342abb96cd31575d286343023ba1d4d37001e99 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 18 Nov 2024 19:25:39 -0500 Subject: [PATCH 12/55] working: fix type hint --- lowtime/_lowtime_rs.pyi | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lowtime/_lowtime_rs.pyi b/lowtime/_lowtime_rs.pyi index a593eca..1658960 100644 --- a/lowtime/_lowtime_rs.pyi +++ b/lowtime/_lowtime_rs.pyi @@ -8,14 +8,14 @@ class PhillipsDessouky: node_ids: list[int] | nx.classes.reportviews.NodeView, source_node_id: int, sink_node_id: int, - edges_raw: list[tuple[tuple[int, int], float]], + edges_raw: list[ + tuple[ + tuple[int, int], + tuple[float, float, float, float], + tuple[bool, int, int, int, int, int, int, int] | None, + ] + ], ) -> None: ... - def max_flow(self) -> list[ - tuple[ - tuple[int, int], - tuple[float, float, float, float], - tuple[bool, int, int, int, int, int, int, int] | None, - ] - ]: ... + def max_flow(self) -> list[tuple[int, int], float]: ... # TODO(ohjun): add CostModel interface From 4d231877858305fbc2b7c097c1e8c82587d77528 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 18 Nov 2024 19:30:32 -0500 Subject: [PATCH 13/55] working: actually fix type hint --- lowtime/_lowtime_rs.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lowtime/_lowtime_rs.pyi b/lowtime/_lowtime_rs.pyi index 1658960..f3e395e 100644 --- a/lowtime/_lowtime_rs.pyi +++ b/lowtime/_lowtime_rs.pyi @@ -16,6 +16,6 @@ class PhillipsDessouky: ] ], ) -> None: ... - def max_flow(self) -> list[tuple[int, int], float]: ... + def max_flow(self) -> list[tuple[tuple[int, int], float]]: ... # TODO(ohjun): add CostModel interface From 83efa137f17360a22f115111afe0e485f382ac8e Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 19 Nov 2024 13:51:41 -0500 Subject: [PATCH 14/55] remove backup file since working --- src/phillips_dessouky.bak.rs | 70 ------------------------------------ 1 file changed, 70 deletions(-) delete mode 100644 src/phillips_dessouky.bak.rs diff --git a/src/phillips_dessouky.bak.rs b/src/phillips_dessouky.bak.rs deleted file mode 100644 index bb5b1b2..0000000 --- a/src/phillips_dessouky.bak.rs +++ /dev/null @@ -1,70 +0,0 @@ -use pyo3::prelude::*; - -use ordered_float::OrderedFloat; -use pathfinding::directed::edmonds_karp::{ - SparseCapacity, - Edge, - edmonds_karp -}; - -use std::time::Instant; -use log::info; - -use crate::lowtime_graph::LowtimeGraph; -use crate::operation::{Operation, CostModel}; -use crate::utils; - - -#[pyclass] -pub struct PhillipsDessouky { - node_ids: Vec, - source_node_id: u32, - sink_node_id: u32, - edges_raw: Vec<((u32, u32), f64)>, -} - -#[pymethods] -impl PhillipsDessouky { - #[new] - fn new( - node_ids: Vec, - source_node_id: u32, - sink_node_id: u32, - edges_raw: Vec<((u32, u32), f64)>, - ) -> PyResult { - Ok(PhillipsDessouky { - node_ids, - source_node_id, - sink_node_id, - edges_raw, - }) - } - - fn max_flow(&self) -> PyResult> { - let profiling_start = Instant::now(); - let edges_edmonds_karp: Vec>> = self.edges_raw.iter().map(|((from, to), cap)| { - ((*from, *to), OrderedFloat(*cap)) - }).collect(); - let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow scaling to OrderedFloat time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - - let profiling_start = Instant::now(); - let (flows, _max_flow, _min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( - &self.node_ids, - &self.source_node_id, - &self.sink_node_id, - edges_edmonds_karp, - ); - let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - - let profiling_start = Instant::now(); - let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { - ((*from, *to), flow.into_inner()) - }).collect(); - let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow reformat OrderedFloat to f64 time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - - Ok(flows_f64) - } -} From ed4fbf2abacc6f14a7370d7197f51ba0c56d65ff Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 19 Nov 2024 14:21:23 -0500 Subject: [PATCH 15/55] restructure max_flow temporarily, use with_capacity+extend for performance --- lowtime/_lowtime_rs.pyi | 2 +- lowtime/solver.py | 6 ++- src/lowtime_graph.rs | 79 +++++++++++++++++++++------------------- src/phillips_dessouky.rs | 24 +++++++++++- 4 files changed, 69 insertions(+), 42 deletions(-) diff --git a/lowtime/_lowtime_rs.pyi b/lowtime/_lowtime_rs.pyi index f3e395e..b5e32f8 100644 --- a/lowtime/_lowtime_rs.pyi +++ b/lowtime/_lowtime_rs.pyi @@ -16,6 +16,6 @@ class PhillipsDessouky: ] ], ) -> None: ... - def max_flow(self) -> list[tuple[tuple[int, int], float]]: ... + def max_flow_bak(self) -> list[tuple[tuple[int, int], float]]: ... # TODO(ohjun): add CostModel interface diff --git a/lowtime/solver.py b/lowtime/solver.py index 7f07921..3a6ed00 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -508,7 +508,8 @@ def reformat_rust_flow_to_dict( profiling_data_transfer, ) - rust_flow_vec = rust_dag.max_flow() + # ohjun: FIRST MAX FLOW + rust_flow_vec = rust_dag.max_flow_bak() flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) profiling_max_flow = time.time() - profiling_max_flow @@ -602,7 +603,8 @@ def reformat_rust_flow_to_dict( profiling_data_transfer, ) - rust_flow_vec = rust_dag.max_flow() + # ohjun: SECOND MAX FLOW + rust_flow_vec = rust_dag.max_flow_bak() flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) profiling_max_flow = time.time() - profiling_max_flow diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index bae4575..f355789 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -16,20 +16,22 @@ use crate::utils; pub struct LowtimeGraph { node_ids: Vec, - source_node_id: u32, - sink_node_id: u32, + source_node_id: Option, + sink_node_id: Option, edges: HashMap>, preds: HashMap, + num_edges: usize, } impl LowtimeGraph { pub fn new() -> Self { LowtimeGraph { node_ids: Vec::new(), - source_node_id: u32::MAX, - sink_node_id: u32::MAX, + source_node_id: None, + sink_node_id: None, edges: HashMap::new(), preds: HashMap::new(), + num_edges: 0, } } @@ -40,10 +42,10 @@ impl LowtimeGraph { edges_raw: Vec<((u32, u32), (f64, f64, f64, f64), Option<(bool, u64, u64, u64, u64, u64, u64, u64)>)>, ) -> Self { let mut graph = LowtimeGraph::new(); - // TODO(ohjun): this is very bad. will make better after checking that minimal change version works. graph.node_ids = node_ids; - graph.source_node_id = source_node_id; - graph.sink_node_id = sink_node_id; + graph.source_node_id = Some(source_node_id); + graph.sink_node_id = Some(sink_node_id); + edges_raw.iter().for_each(|( (from, to), (capacity, flow, ub, lb), @@ -73,58 +75,61 @@ impl LowtimeGraph { graph } - pub fn get_node_ids(&self) -> Vec { + pub fn max_flow_bak(&self) -> Vec<((u32, u32), f64)> { + let edges_edmonds_karp = self.get_ek_preprocessed_edges(); + + let profiling_start = Instant::now(); + let (flows, _max_flow, _min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( + &self.node_ids, + &self.source_node_id.unwrap(), + &self.sink_node_id.unwrap(), + edges_edmonds_karp, + ); + let profiling_end = Instant::now(); + info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); + + let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { + ((*from, *to), flow.into_inner()) + }).collect(); + + flows_f64 + } + + fn get_node_ids(&self) -> Vec { let mut node_ids: Vec = self.edges.keys().cloned().collect(); node_ids.sort(); node_ids } - pub fn add_edge(&mut self, from: u32, to: u32, edge: LowtimeEdge) -> () { + fn add_edge(&mut self, from: u32, to: u32, edge: LowtimeEdge) -> () { self.edges .entry(from) .or_insert_with(HashMap::new) .insert(to, edge); self.preds.insert(to, from); + self.num_edges += 1; } - pub fn get_edge(&self, from: u32, to: u32) -> Option<&LowtimeEdge> { + fn get_edge(&self, from: u32, to: u32) -> Option<&LowtimeEdge> { self.edges .get(&from) .and_then(|to_edges| to_edges.get(&to)) } - pub fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut LowtimeEdge> { + fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut LowtimeEdge> { self.edges .get_mut(&from) .and_then(|to_edges| to_edges.get_mut(&to)) } - pub fn max_flow(&self) -> Vec<((u32, u32), f64)> { - let profiling_start = Instant::now(); - let edges_edmonds_karp: Vec>> = self.edges.iter().flat_map(|(from, inner)| - inner.iter().map(|(to, edge)| ((*from, *to), edge.get_capacity())) - ).collect(); - let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow scaling to OrderedFloat time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - - let profiling_start = Instant::now(); - let (flows, _max_flow, _min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( - &self.node_ids, - &self.source_node_id, - &self.sink_node_id, - edges_edmonds_karp, - ); - let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - - let profiling_start = Instant::now(); - let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { - ((*from, *to), flow.into_inner()) - }).collect(); - let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow reformat OrderedFloat to f64 time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - - flows_f64 + fn get_ek_preprocessed_edges(&self, ) -> Vec>> { + let mut processed_edges = Vec::with_capacity(self.num_edges); + processed_edges.extend( + self.edges.iter().flat_map(|(from, inner)| + inner.iter().map(|(to, edge)| + ((*from, *to), edge.get_capacity()) + ))); + processed_edges } } diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index 8a50308..c814fae 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -1,5 +1,7 @@ use pyo3::prelude::*; +use std::collections::HashSet; + use ordered_float::OrderedFloat; use pathfinding::directed::edmonds_karp::{ SparseCapacity, @@ -38,7 +40,25 @@ impl PhillipsDessouky { }) } - fn max_flow(&self) -> Vec<((u32, u32), f64)> { - self.graph.max_flow() + // TODO(ohjun): this is backup, remove when obsolete + fn max_flow_bak(&self) -> Vec<((u32, u32), f64)> { + self.graph.max_flow_bak() } + + // TODO(ohjun): iteratively implement/verify in _wip until done, + // then replace with this function + // fn find_min_cut(&self) -> (HashSet, HashSet) { + // } + + // fn find_min_cut_wip(&self) -> Vec<((u32, u32), f64)> { + + // } +} + +// not exposed to Python +impl PhillipsDessouky { + + // fn max_flow(&self) -> Vec<((u32, u32), f64)> { + // self.graph.max_flow() + // } } From ddacc3f41e1edf44e79c9cb46a055274e35208b9 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 20 Nov 2024 14:26:27 -0500 Subject: [PATCH 16/55] working: rustify up to first max flow in find_min_cut --- lowtime/_lowtime_rs.pyi | 2 +- lowtime/solver.goal-state.py | 465 ++++++++++++++++ lowtime/{solver.py => solver.orig.py} | 10 +- lowtime/solver.testing.py | 757 ++++++++++++++++++++++++++ src/lowtime_graph.rs | 151 +++-- src/operation.rs | 1 + src/phillips_dessouky.rs | 137 ++++- 7 files changed, 1477 insertions(+), 46 deletions(-) create mode 100644 lowtime/solver.goal-state.py rename lowtime/{solver.py => solver.orig.py} (98%) create mode 100644 lowtime/solver.testing.py diff --git a/lowtime/_lowtime_rs.pyi b/lowtime/_lowtime_rs.pyi index b5e32f8..e8bbc2d 100644 --- a/lowtime/_lowtime_rs.pyi +++ b/lowtime/_lowtime_rs.pyi @@ -16,6 +16,6 @@ class PhillipsDessouky: ] ], ) -> None: ... - def max_flow_bak(self) -> list[tuple[tuple[int, int], float]]: ... + def find_min_cut_wip(self) -> list[tuple[tuple[int, int], float]]: ... # TODO(ohjun): add CostModel interface diff --git a/lowtime/solver.goal-state.py b/lowtime/solver.goal-state.py new file mode 100644 index 0000000..304b3a4 --- /dev/null +++ b/lowtime/solver.goal-state.py @@ -0,0 +1,465 @@ +# Copyright (C) 2023 Jae-Won Chung +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" + + +from __future__ import annotations + +import time +import sys +import logging +from collections import deque +from collections.abc import Generator + +import networkx as nx +from attrs import define, field + +from lowtime.operation import Operation +from lowtime.graph_utils import ( + aon_dag_to_aoa_dag, + aoa_to_critical_dag, + get_critical_aoa_dag_total_time, + get_total_cost, +) +from lowtime.exceptions import LowtimeFlowError +from lowtime import _lowtime_rs + +FP_ERROR = 1e-6 + +logger = logging.getLogger(__name__) + + +@define +class IterationResult: + """Holds results after one PD iteration. + + Attributes: + iteration: The number of optimization iterations experienced by the DAG. + cost_change: The increase in cost from reducing the DAG's + quantized execution time by 1. + cost: The current total cost of the DAG. + quant_time: The current quantized total execution time of the DAG. + unit_time: The unit time used for time quantization. + real_time: The current real (de-quantized) total execution time of the DAG. + """ + + iteration: int + cost_change: float + cost: float + quant_time: int + unit_time: float + real_time: float = field(init=False) + + def __attrs_post_init__(self) -> None: + """Set `real_time` after initialization.""" + self.real_time = self.quant_time * self.unit_time + + +class PhillipsDessouky: + """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" + + def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: + """Initialize the Phillips-Dessouky solver. + + Assumptions: + - The graph is a Directed Acyclic Graph (DAG). + - Operations are annotated on nodes with name `attr_name`. + - There is only one source (entry) node. The source node is annotated as + `dag.graph["source_node"]`. + - There is only one sink (exit) node. The sink node is annotated as + `dag.graph["sink_node"]`. + - The `unit_time` attribute of every operation is the same. + + Args: + dag: A networkx DiGraph object that represents the computation DAG. + The aforementioned assumptions should hold for the DAG. + attr_name: The name of the attribute on nodes that holds the operation + object. Defaults to "op". + """ + self.attr_name = attr_name + + # Run checks on the DAG and cache some properties. + # Check: It's a DAG. + if not nx.is_directed_acyclic_graph(dag): + raise ValueError("The graph should be a Directed Acyclic Graph.") + + # Check: Only one source node that matches annotation. + if (source_node := dag.graph.get("source_node")) is None: + raise ValueError("The graph should have a `source_node` attribute.") + source_node_candidates = [] + for node_id, in_degree in dag.in_degree(): + if in_degree == 0: + source_node_candidates.append(node_id) + if len(source_node_candidates) == 0: + raise ValueError( + "Found zero nodes with in-degree 0. Cannot determine source node." + ) + if len(source_node_candidates) > 1: + raise ValueError( + f"Expecting only one source node, found {source_node_candidates}." + ) + if (detected_source_node := source_node_candidates[0]) != source_node: + raise ValueError( + f"Detected source node ({detected_source_node}) does not match " + f"the annotated source node ({source_node})." + ) + + # Check: Only one sink node that matches annotation. + if (sink_node := dag.graph.get("sink_node")) is None: + raise ValueError("The graph should have a `sink_node` attribute.") + sink_node_candidates = [] + for node_id, out_degree in dag.out_degree(): + if out_degree == 0: + sink_node_candidates.append(node_id) + if len(sink_node_candidates) == 0: + raise ValueError( + "Found zero nodes with out-degree 0. Cannot determine sink node." + ) + if len(sink_node_candidates) > 1: + raise ValueError( + f"Expecting only one sink node, found {sink_node_candidates}." + ) + if (detected_sink_node := sink_node_candidates[0]) != sink_node: + raise ValueError( + f"Detected sink node ({detected_sink_node}) does not match " + f"the annotated sink node ({sink_node})." + ) + + # Check: The `unit_time` attributes of every operation should be the same. + unit_time_candidates = set[float]() + for _, node_attr in dag.nodes(data=True): + if self.attr_name in node_attr: + op: Operation = node_attr[self.attr_name] + if op.is_dummy: + continue + unit_time_candidates.update( + option.unit_time for option in op.spec.options.options + ) + if len(unit_time_candidates) == 0: + raise ValueError( + "Found zero operations in the graph. Make sure you " + f"added operations as node attributes with key `{self.attr_name}`.", + ) + if len(unit_time_candidates) > 1: + raise ValueError( + f"Expecting the same `unit_time` across all operations, " + f"found {unit_time_candidates}." + ) + + self.aon_dag = dag + self.unit_time = unit_time_candidates.pop() + + def run(self) -> Generator[IterationResult, None, None]: + """Run the algorithm and yield a DAG after each iteration. + + The solver will not deepcopy operations on the DAG but rather in-place modify + them for speed. The caller should deepcopy the operations or the DAG if needed + before running the next iteration. + + Upon yield, it is guaranteed that the earliest/latest start/finish time values + of all operations are up to date w.r.t. the `duration` of each operation. + """ + logger.info("Starting Phillips-Dessouky solver.") + profiling_setup = time.time() + + # Convert the original activity-on-node DAG to activity-on-arc DAG form. + # AOA DAGs are purely internal. All public input and output of this class + # should be in AON form. + aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) + + # Estimate the minimum execution time of the DAG by setting every operation + # to run at its minimum duration. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.min_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + min_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected minimum quantized execution time: %d", min_time) + + # Estimated the maximum execution time of the DAG by setting every operation + # to run at its maximum duration. This is also our initial start point. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.max_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + max_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected maximum quantized execution time: %d", max_time) + + num_iters = max_time - min_time + 1 + logger.info("Expected number of PD iterations: %d", num_iters) + + profiling_setup = time.time() - profiling_setup + logger.info( + "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup + ) + + # Iteratively reduce the execution time of the DAG. + for iteration in range(sys.maxsize): + logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) + profiling_iter = time.time() + + # At this point, `critical_dag` always exists and is what we want. + # For the first iteration, the critical DAG is computed before the for + # loop in order to estimate the number of iterations. For subsequent + # iterations, the critcal DAG is computed after each iteration in + # in order to construct `IterationResult`. + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Critical DAG:") + logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) + logger.debug("Number of edges: %d", critical_dag.number_of_edges()) + non_dummy_ops = [ + attr[self.attr_name] + for _, _, attr in critical_dag.edges(data=True) + if not attr[self.attr_name].is_dummy + ] + logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) + logger.debug( + "Sum of non-dummy durations: %d", + sum(op.duration for op in non_dummy_ops), + ) + + profiling_annotate = time.time() + self.annotate_capacities(critical_dag) + profiling_annotate = time.time() - profiling_annotate + logger.info( + "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", + profiling_annotate, + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Capacity DAG:") + logger.debug( + "Total lb value: %f", + sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), + ) + logger.debug( + "Total ub value: %f", + sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), + ) + + try: + profiling_min_cut = time.time() + s_set, t_set = self.find_min_cut(critical_dag) + profiling_min_cut = time.time() - profiling_min_cut + logger.info( + "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", + profiling_min_cut, + ) + except LowtimeFlowError as e: + logger.info("Could not find minimum cut: %s", e.message) + logger.info("Terminating PD iteration.") + break + + profiling_reduce = time.time() + cost_change = self.reduce_durations(critical_dag, s_set, t_set) + profiling_reduce = time.time() - profiling_reduce + logger.info( + "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", + profiling_reduce, + ) + + if cost_change == float("inf") or abs(cost_change) < FP_ERROR: + logger.info("No further time reduction possible.") + logger.info("Terminating PD iteration.") + break + + # Earliest/latest start/finish times on operations also annotated here. + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + + # We directly modify operation attributes in the DAG, so after we + # ran one iteration, the AON DAG holds updated attributes. + result = IterationResult( + iteration=iteration + 1, + cost_change=cost_change, + cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), + quant_time=get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ), + unit_time=self.unit_time, + ) + logger.info("%s", result) + profiling_iter = time.time() - profiling_iter + logger.info( + "PROFILING PhillipsDessouky::run single iteration time: %.10fs", + profiling_iter, + ) + yield result + + def reduce_durations( + self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] + ) -> float: + """Modify operation durations to reduce the DAG's execution time by 1.""" + speed_up_edges: list[Operation] = [] + for node_id in s_set: + for child_id in list(dag.successors(node_id)): + if child_id in t_set: + op: Operation = dag[node_id][child_id][self.attr_name] + speed_up_edges.append(op) + + slow_down_edges: list[Operation] = [] + for node_id in t_set: + for child_id in list(dag.successors(node_id)): + if child_id in s_set: + op: Operation = dag[node_id][child_id][self.attr_name] + slow_down_edges.append(op) + + if not speed_up_edges: + logger.info("No speed up candidate operations.") + return 0.0 + + cost_change = 0.0 + + # Reduce the duration of edges (speed up) by quant_time 1. + for op in speed_up_edges: + if op.is_dummy: + logger.info("Cannot speed up dummy operation.") + return float("inf") + if op.duration - 1 < op.min_duration: + logger.info("Operation %s has reached the limit of speed up", op) + return float("inf") + cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) + op_before_str = str(op) + op.duration -= 1 + logger.info("Sped up %s to %s", op_before_str, op) + + # Increase the duration of edges (slow down) by quant_time 1. + for op in slow_down_edges: + # Dummy edges can always be slowed down. + if op.is_dummy: + logger.info("Slowed down DummyOperation (didn't really slowdown).") + continue + elif op.duration + 1 > op.max_duration: + logger.info("Operation %s has reached the limit of slow down", op) + return float("inf") + cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) + before_op_str = str(op) + op.duration += 1 + logger.info("Slowed down %s to %s", before_op_str, op) + + return cost_change + + def find_min_cut(self, dag: nx.DiGraph) -> tuple[set[int], set[int]]: + """Find the min cut of the DAG annotated with lower/upper bound flow capacities. + + Assumptions: + - The capacity DAG is in AOA form. + - The capacity DAG has been annotated with `lb` and `ub` attributes on edges, + representing the lower and upper bounds of the flow on the edge. + + Returns: + A tuple of (s_set, t_set) where s_set is the set of nodes on the source side + of the min cut and t_set is the set of nodes on the sink side of the min cut. + Returns None if no feasible flow exists. + + Raises: + LowtimeFlowError: When no feasible flow exists. + """ + # Helper function for Rust interop + def format_rust_inputs( + dag: nx.DiGraph, + ) -> tuple[ + nx.classes.reportviews.NodeView, + list[ + tuple[ + tuple[int, int], + tuple[float, float, float, float], + tuple[bool, int, int, int, int, int, int, int] | None, + ] + ], + ]: + nodes = dag.nodes + edges = [] + for from_, to_, edge_attrs in dag.edges(data=True): + op_details = ( + None + if self.attr_name not in edge_attrs + else ( + edge_attrs[self.attr_name].is_dummy, + edge_attrs[self.attr_name].duration, + edge_attrs[self.attr_name].max_duration, + edge_attrs[self.attr_name].min_duration, + edge_attrs[self.attr_name].earliest_start, + edge_attrs[self.attr_name].latest_start, + edge_attrs[self.attr_name].earliest_finish, + edge_attrs[self.attr_name].latest_finish, + ) + ) + edges.append( + ( + (from_, to_), + ( + edge_attrs.get("capacity", 0), # TODO(ohjun): suss + edge_attrs.get("flow", 0), + edge_attrs.get("ub", 0), + edge_attrs.get("lb", 0), + ), + op_details, + ) + ) + return nodes, edges + + # Construct Rust runner with input (critical) dag + nodes, edges = format_rust_inputs(dag) + rust_runner = _lowtime_rs.PhillipsDessouky( + nodes, + dag.graph["source_node"], + dag.graph["sink_node"], + edges + ) + s_set, t_set = rust_runner.find_min_cut() + + return s_set, t_set + + def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: + """In-place annotate the critical DAG with flow capacities.""" + # XXX(JW): Is this always large enough? + # It is necessary to monitor the `cost_change` value in `IterationResult` + # and make sure they are way smaller than this value. Even if the cost + # change is close or larger than this, users can scale down their cost + # value in `ExecutionOption`s. + inf = 10000.0 + for _, _, edge_attr in critical_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + duration = op.duration + # Dummy operations don't constrain the flow. + if op.is_dummy: # noqa: SIM114 + lb, ub = 0.0, inf + # Cannot be sped up or down. + elif duration == op.min_duration == op.max_duration: + lb, ub = 0.0, inf + # Cannot be sped up. + elif duration - 1 < op.min_duration: + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = inf + # Cannot be slowed down. + elif duration + 1 > op.max_duration: + lb = 0.0 + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + else: + # In case the cost model is almost linear, give this edge some room. + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR + + # XXX(JW): This roundiing may not be necessary. + edge_attr["lb"] = lb // FP_ERROR * FP_ERROR + edge_attr["ub"] = ub // FP_ERROR * FP_ERROR diff --git a/lowtime/solver.py b/lowtime/solver.orig.py similarity index 98% rename from lowtime/solver.py rename to lowtime/solver.orig.py index 3a6ed00..2bcefef 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.orig.py @@ -509,8 +509,14 @@ def reformat_rust_flow_to_dict( ) # ohjun: FIRST MAX FLOW - rust_flow_vec = rust_dag.max_flow_bak() + rust_flow_vec = rust_dag.max_flow_depr() flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) + def print_dict(d): + for from_, inner in d.items(): + for to_, flow in inner.items(): + logger.info(f'{from_} -> {to_}: {flow}') + logger.info('correct flow_dict:') + print_dict(flow_dict) profiling_max_flow = time.time() - profiling_max_flow logger.info( @@ -604,7 +610,7 @@ def reformat_rust_flow_to_dict( ) # ohjun: SECOND MAX FLOW - rust_flow_vec = rust_dag.max_flow_bak() + rust_flow_vec = rust_dag.max_flow_depr() flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) profiling_max_flow = time.time() - profiling_max_flow diff --git a/lowtime/solver.testing.py b/lowtime/solver.testing.py new file mode 100644 index 0000000..c061c37 --- /dev/null +++ b/lowtime/solver.testing.py @@ -0,0 +1,757 @@ +# Copyright (C) 2023 Jae-Won Chung +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" + + +from __future__ import annotations + +import time +import sys +import logging +from collections import deque +from collections.abc import Generator + +import networkx as nx +from attrs import define, field + +from lowtime.operation import Operation +from lowtime.graph_utils import ( + aon_dag_to_aoa_dag, + aoa_to_critical_dag, + get_critical_aoa_dag_total_time, + get_total_cost, +) +from lowtime.exceptions import LowtimeFlowError +from lowtime import _lowtime_rs + +FP_ERROR = 1e-6 + +logger = logging.getLogger(__name__) + + +@define +class IterationResult: + """Holds results after one PD iteration. + + Attributes: + iteration: The number of optimization iterations experienced by the DAG. + cost_change: The increase in cost from reducing the DAG's + quantized execution time by 1. + cost: The current total cost of the DAG. + quant_time: The current quantized total execution time of the DAG. + unit_time: The unit time used for time quantization. + real_time: The current real (de-quantized) total execution time of the DAG. + """ + + iteration: int + cost_change: float + cost: float + quant_time: int + unit_time: float + real_time: float = field(init=False) + + def __attrs_post_init__(self) -> None: + """Set `real_time` after initialization.""" + self.real_time = self.quant_time * self.unit_time + + +class PhillipsDessouky: + """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" + + def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: + """Initialize the Phillips-Dessouky solver. + + Assumptions: + - The graph is a Directed Acyclic Graph (DAG). + - Operations are annotated on nodes with name `attr_name`. + - There is only one source (entry) node. The source node is annotated as + `dag.graph["source_node"]`. + - There is only one sink (exit) node. The sink node is annotated as + `dag.graph["sink_node"]`. + - The `unit_time` attribute of every operation is the same. + + Args: + dag: A networkx DiGraph object that represents the computation DAG. + The aforementioned assumptions should hold for the DAG. + attr_name: The name of the attribute on nodes that holds the operation + object. Defaults to "op". + """ + self.attr_name = attr_name + + # Run checks on the DAG and cache some properties. + # Check: It's a DAG. + if not nx.is_directed_acyclic_graph(dag): + raise ValueError("The graph should be a Directed Acyclic Graph.") + + # Check: Only one source node that matches annotation. + if (source_node := dag.graph.get("source_node")) is None: + raise ValueError("The graph should have a `source_node` attribute.") + source_node_candidates = [] + for node_id, in_degree in dag.in_degree(): + if in_degree == 0: + source_node_candidates.append(node_id) + if len(source_node_candidates) == 0: + raise ValueError( + "Found zero nodes with in-degree 0. Cannot determine source node." + ) + if len(source_node_candidates) > 1: + raise ValueError( + f"Expecting only one source node, found {source_node_candidates}." + ) + if (detected_source_node := source_node_candidates[0]) != source_node: + raise ValueError( + f"Detected source node ({detected_source_node}) does not match " + f"the annotated source node ({source_node})." + ) + + # Check: Only one sink node that matches annotation. + if (sink_node := dag.graph.get("sink_node")) is None: + raise ValueError("The graph should have a `sink_node` attribute.") + sink_node_candidates = [] + for node_id, out_degree in dag.out_degree(): + if out_degree == 0: + sink_node_candidates.append(node_id) + if len(sink_node_candidates) == 0: + raise ValueError( + "Found zero nodes with out-degree 0. Cannot determine sink node." + ) + if len(sink_node_candidates) > 1: + raise ValueError( + f"Expecting only one sink node, found {sink_node_candidates}." + ) + if (detected_sink_node := sink_node_candidates[0]) != sink_node: + raise ValueError( + f"Detected sink node ({detected_sink_node}) does not match " + f"the annotated sink node ({sink_node})." + ) + + # Check: The `unit_time` attributes of every operation should be the same. + unit_time_candidates = set[float]() + for _, node_attr in dag.nodes(data=True): + if self.attr_name in node_attr: + op: Operation = node_attr[self.attr_name] + if op.is_dummy: + continue + unit_time_candidates.update( + option.unit_time for option in op.spec.options.options + ) + if len(unit_time_candidates) == 0: + raise ValueError( + "Found zero operations in the graph. Make sure you " + f"added operations as node attributes with key `{self.attr_name}`.", + ) + if len(unit_time_candidates) > 1: + raise ValueError( + f"Expecting the same `unit_time` across all operations, " + f"found {unit_time_candidates}." + ) + + self.aon_dag = dag + self.unit_time = unit_time_candidates.pop() + + def run(self) -> Generator[IterationResult, None, None]: + """Run the algorithm and yield a DAG after each iteration. + + The solver will not deepcopy operations on the DAG but rather in-place modify + them for speed. The caller should deepcopy the operations or the DAG if needed + before running the next iteration. + + Upon yield, it is guaranteed that the earliest/latest start/finish time values + of all operations are up to date w.r.t. the `duration` of each operation. + """ + logger.info("Starting Phillips-Dessouky solver.") + profiling_setup = time.time() + + # Convert the original activity-on-node DAG to activity-on-arc DAG form. + # AOA DAGs are purely internal. All public input and output of this class + # should be in AON form. + aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) + + # Estimate the minimum execution time of the DAG by setting every operation + # to run at its minimum duration. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.min_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + min_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected minimum quantized execution time: %d", min_time) + + # Estimated the maximum execution time of the DAG by setting every operation + # to run at its maximum duration. This is also our initial start point. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.max_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + max_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected maximum quantized execution time: %d", max_time) + + num_iters = max_time - min_time + 1 + logger.info("Expected number of PD iterations: %d", num_iters) + + profiling_setup = time.time() - profiling_setup + logger.info( + "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup + ) + + # Iteratively reduce the execution time of the DAG. + for iteration in range(sys.maxsize): + logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) + profiling_iter = time.time() + + # At this point, `critical_dag` always exists and is what we want. + # For the first iteration, the critical DAG is computed before the for + # loop in order to estimate the number of iterations. For subsequent + # iterations, the critcal DAG is computed after each iteration in + # in order to construct `IterationResult`. + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Critical DAG:") + logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) + logger.debug("Number of edges: %d", critical_dag.number_of_edges()) + non_dummy_ops = [ + attr[self.attr_name] + for _, _, attr in critical_dag.edges(data=True) + if not attr[self.attr_name].is_dummy + ] + logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) + logger.debug( + "Sum of non-dummy durations: %d", + sum(op.duration for op in non_dummy_ops), + ) + + profiling_annotate = time.time() + self.annotate_capacities(critical_dag) + profiling_annotate = time.time() - profiling_annotate + logger.info( + "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", + profiling_annotate, + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Capacity DAG:") + logger.debug( + "Total lb value: %f", + sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), + ) + logger.debug( + "Total ub value: %f", + sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), + ) + + try: + profiling_min_cut = time.time() + s_set, t_set = self.find_min_cut(critical_dag) + profiling_min_cut = time.time() - profiling_min_cut + logger.info( + "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", + profiling_min_cut, + ) + except LowtimeFlowError as e: + logger.info("Could not find minimum cut: %s", e.message) + logger.info("Terminating PD iteration.") + break + + profiling_reduce = time.time() + cost_change = self.reduce_durations(critical_dag, s_set, t_set) + profiling_reduce = time.time() - profiling_reduce + logger.info( + "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", + profiling_reduce, + ) + + if cost_change == float("inf") or abs(cost_change) < FP_ERROR: + logger.info("No further time reduction possible.") + logger.info("Terminating PD iteration.") + break + + # Earliest/latest start/finish times on operations also annotated here. + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + + # We directly modify operation attributes in the DAG, so after we + # ran one iteration, the AON DAG holds updated attributes. + result = IterationResult( + iteration=iteration + 1, + cost_change=cost_change, + cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), + quant_time=get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ), + unit_time=self.unit_time, + ) + logger.info("%s", result) + profiling_iter = time.time() - profiling_iter + logger.info( + "PROFILING PhillipsDessouky::run single iteration time: %.10fs", + profiling_iter, + ) + yield result + + def reduce_durations( + self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] + ) -> float: + """Modify operation durations to reduce the DAG's execution time by 1.""" + speed_up_edges: list[Operation] = [] + for node_id in s_set: + for child_id in list(dag.successors(node_id)): + if child_id in t_set: + op: Operation = dag[node_id][child_id][self.attr_name] + speed_up_edges.append(op) + + slow_down_edges: list[Operation] = [] + for node_id in t_set: + for child_id in list(dag.successors(node_id)): + if child_id in s_set: + op: Operation = dag[node_id][child_id][self.attr_name] + slow_down_edges.append(op) + + if not speed_up_edges: + logger.info("No speed up candidate operations.") + return 0.0 + + cost_change = 0.0 + + # Reduce the duration of edges (speed up) by quant_time 1. + for op in speed_up_edges: + if op.is_dummy: + logger.info("Cannot speed up dummy operation.") + return float("inf") + if op.duration - 1 < op.min_duration: + logger.info("Operation %s has reached the limit of speed up", op) + return float("inf") + cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) + op_before_str = str(op) + op.duration -= 1 + logger.info("Sped up %s to %s", op_before_str, op) + + # Increase the duration of edges (slow down) by quant_time 1. + for op in slow_down_edges: + # Dummy edges can always be slowed down. + if op.is_dummy: + logger.info("Slowed down DummyOperation (didn't really slowdown).") + continue + elif op.duration + 1 > op.max_duration: + logger.info("Operation %s has reached the limit of slow down", op) + return float("inf") + cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) + before_op_str = str(op) + op.duration += 1 + logger.info("Slowed down %s to %s", before_op_str, op) + + return cost_change + + def find_min_cut(self, dag: nx.DiGraph) -> tuple[set[int], set[int]]: + """Find the min cut of the DAG annotated with lower/upper bound flow capacities. + + Assumptions: + - The capacity DAG is in AOA form. + - The capacity DAG has been annotated with `lb` and `ub` attributes on edges, + representing the lower and upper bounds of the flow on the edge. + + Returns: + A tuple of (s_set, t_set) where s_set is the set of nodes on the source side + of the min cut and t_set is the set of nodes on the sink side of the min cut. + Returns None if no feasible flow exists. + + Raises: + LowtimeFlowError: When no feasible flow exists. + """ + # Helper function for Rust interop + def format_rust_inputs( + dag: nx.DiGraph, + ) -> tuple[ + nx.classes.reportviews.NodeView, + list[ + tuple[ + tuple[int, int], + tuple[float, float, float, float], + tuple[bool, int, int, int, int, int, int, int] | None, + ] + ], + ]: + nodes = dag.nodes + edges = [] + for from_, to_, edge_attrs in dag.edges(data=True): + op_details = ( + None + if self.attr_name not in edge_attrs + else ( + edge_attrs[self.attr_name].is_dummy, + edge_attrs[self.attr_name].duration, + edge_attrs[self.attr_name].max_duration, + edge_attrs[self.attr_name].min_duration, + edge_attrs[self.attr_name].earliest_start, + edge_attrs[self.attr_name].latest_start, + edge_attrs[self.attr_name].earliest_finish, + edge_attrs[self.attr_name].latest_finish, + ) + ) + edges.append( + ( + (from_, to_), + ( + edge_attrs.get("capacity", 0), + edge_attrs.get("flow", 0), + edge_attrs.get("ub", 0), + edge_attrs.get("lb", 0), + ), + op_details, + ) + ) + return nodes, edges + + ########### NEW ########### + # Construct Rust runner with input (critical) dag + ohjun_nodes, ohjun_edges = format_rust_inputs(dag) + # print(ohjun_nodes) + # print(ohjun_edges) + ohjun_rust_runner = _lowtime_rs.PhillipsDessouky( + ohjun_nodes, + dag.graph["source_node"], + dag.graph["sink_node"], + ohjun_edges + ) + ########### NEW ########### + + profiling_min_cut_setup = time.time() + source_node = dag.graph["source_node"] + sink_node = dag.graph["sink_node"] + + # In order to solve max flow on edges with both lower and upper bounds, + # we first need to convert it to another DAG that only has upper bounds. + unbound_dag = nx.DiGraph(dag) + + # For every edge, capacity = ub - lb. + for _, _, edge_attrs in unbound_dag.edges(data=True): + edge_attrs["capacity"] = edge_attrs["ub"] - edge_attrs["lb"] + + # Add a new node s', which will become the new source node. + # We constructed the AOA DAG, so we know that node IDs are integers. + node_ids: list[int] = list(unbound_dag.nodes) + s_prime_id = max(node_ids) + 1 + unbound_dag.add_node(s_prime_id) + + # For every node u in the original graph, add an edge (s', u) with capacity + # equal to the sum of all lower bounds of u's parents. + for u in dag.nodes: + capacity = 0.0 + for pred_id in dag.predecessors(u): + capacity += dag[pred_id][u]["lb"] + unbound_dag.add_edge(s_prime_id, u, capacity=capacity) + + # Add a new node t', which will become the new sink node. + t_prime_id = s_prime_id + 1 + unbound_dag.add_node(t_prime_id) + + # For every node u in the original graph, add an edge (u, t') with capacity + # equal to the sum of all lower bounds of u's children. + for u in dag.nodes: + capacity = 0.0 + for succ_id in dag.successors(u): + capacity += dag[u][succ_id]["lb"] + unbound_dag.add_edge(u, t_prime_id, capacity=capacity) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Unbound DAG") + logger.debug("Number of nodes: %d", unbound_dag.number_of_nodes()) + logger.debug("Number of edges: %d", unbound_dag.number_of_edges()) + logger.debug( + "Sum of capacities: %f", + sum(attr["capacity"] for _, _, attr in unbound_dag.edges(data=True)), + ) + + # Add an edge from t to s with infinite capacity. + unbound_dag.add_edge( + sink_node, + source_node, + capacity=float("inf"), + ) + + profiling_min_cut_setup = time.time() - profiling_min_cut_setup + logger.info( + "PROFILING PhillipsDessouky::find_min_cut setup time: %.10fs", + profiling_min_cut_setup, + ) + + # Helper function for Rust interop + # Rust's pathfinding::edmonds_karp does not return edges with 0 flow, + # but nx.max_flow does. So we fill in the 0s and empty nodes. + def reformat_rust_flow_to_dict( + flow_vec: list[tuple[tuple[int, int], float]], dag: nx.DiGraph + ) -> dict[int, dict[int, float]]: + flow_dict = dict() + for u in dag.nodes: + flow_dict[u] = dict() + for v in dag.successors(u): + flow_dict[u][v] = 0.0 + + for (u, v), cap in flow_vec: + flow_dict[u][v] = cap + + return flow_dict + + # We're done with constructing the DAG with only flow upper bounds. + # Find the maximum flow on this DAG. + profiling_max_flow = time.time() + nodes, edges = format_rust_inputs(unbound_dag) + + profiling_data_transfer = time.time() + rust_dag = _lowtime_rs.PhillipsDessouky(nodes, s_prime_id, t_prime_id, edges) + profiling_data_transfer = time.time() - profiling_data_transfer + logger.info( + "PROFILING PhillipsDessouky::find_min_cut data transfer time: %.10fs", + profiling_data_transfer, + ) + + ##### TESTING print all edges and capacities + # logger.info(f"s_prime_id: {s_prime_id}") + # logger.info(f"t_prime_id: {t_prime_id}") + # logger.info("Python Printing graph:") + # logger.info(f"Num edges: {len(unbound_dag.edges())}") + # for from_, to_, edge_attrs in unbound_dag.edges(data=True): + # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") + + # ohjun: FIRST MAX FLOW + rust_flow_vec = rust_dag.max_flow_depr() + flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) + + ############# NEW ############# + # TEMP(ohjun): in current wip version, do everything until after 1st max flow in Rust + ohjun_rust_flow_vec = ohjun_rust_runner.find_min_cut_wip() + ohjun_flow_dict = reformat_rust_flow_to_dict(ohjun_rust_flow_vec, unbound_dag) + + depr_node_ids = rust_dag.get_dag_node_ids() + depr_edges = rust_dag.get_dag_ek_processed_edges() + new_node_ids = ohjun_rust_runner.get_unbound_dag_node_ids() + new_edges = ohjun_rust_runner.get_unbound_dag_ek_processed_edges() + print(f"depr_node_ids: {depr_node_ids}") + print(f"new_node_ids: {new_node_ids}") + assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" + assert depr_node_ids == new_node_ids, "DIFF in node_ids" + assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" + if sorted(depr_edges) != sorted(new_edges): + for depr_edge, new_edge in zip(sorted(depr_edges), sorted(new_edges)): + if depr_edge == new_edge: + logger.info("edges EQUAL") + logger.info(f"depr_edge: {depr_edge}") + logger.info(f"new_edge : {new_edge}") + else: + logger.info("edges DIFFERENT") + logger.info(f"depr_edge: {depr_edge}") + logger.info(f"new_edge : {new_edge}") + assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" + + def print_dict(d): + for from_, inner in d.items(): + for to_, flow in inner.items(): + logger.info(f'{from_} -> {to_}: {flow}') + logger.info('flow_dict:') + # print_dict(flow_dict) + logger.info('ohjun_flow_dict:') + # print_dict(ohjun_flow_dict) + assert flow_dict == ohjun_flow_dict, "flow dicts were different :(" + ############# NEW ############# + + profiling_max_flow = time.time() - profiling_max_flow + logger.info( + "PROFILING PhillipsDessouky::find_min_cut maximum_flow_1 time: %.10fs", + profiling_max_flow, + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("After first max flow") + total_flow = 0.0 + for d in flow_dict.values(): + for flow in d.values(): + total_flow += flow + logger.debug("Sum of all flow values: %f", total_flow) + + profiling_min_cut_between_max_flows = time.time() + + # Check if residual graph is saturated. If so, we have a feasible flow. + for u in unbound_dag.successors(s_prime_id): + if ( + abs(flow_dict[s_prime_id][u] - unbound_dag[s_prime_id][u]["capacity"]) + > FP_ERROR + ): + logger.error( + "s' -> %s unsaturated (flow: %s, capacity: %s)", + u, + flow_dict[s_prime_id][u], + unbound_dag[s_prime_id][u]["capacity"], + ) + raise LowtimeFlowError( + "ERROR: Max flow on unbounded DAG didn't saturate." + ) + for u in unbound_dag.predecessors(t_prime_id): + if ( + abs(flow_dict[u][t_prime_id] - unbound_dag[u][t_prime_id]["capacity"]) + > FP_ERROR + ): + logger.error( + "%s -> t' unsaturated (flow: %s, capacity: %s)", + u, + flow_dict[u][t_prime_id], + unbound_dag[u][t_prime_id]["capacity"], + ) + raise LowtimeFlowError( + "ERROR: Max flow on unbounded DAG didn't saturate." + ) + + # We have a feasible flow. Construct a new residual graph with the same + # shape as the capacity DAG so that we can find the min cut. + # First, retrieve the flow amounts to the original capacity graph, where for + # each edge u -> v, the flow amount is `flow + lb`. + for u, v in dag.edges: + dag[u][v]["flow"] = flow_dict[u][v] + dag[u][v]["lb"] + + # Construct a new residual graph (same shape as capacity DAG) with + # u -> v capacity `ub - flow` and v -> u capacity `flow - lb`. + residual_graph = nx.DiGraph(dag) + for u, v in dag.edges: + # Rounding small negative values to 0.0 avoids Rust-side + # pathfinding::edmonds_karp from entering unreachable code. + # This edge case did not exist in Python-side nx.max_flow call. + uv_capacity = residual_graph[u][v]["ub"] - residual_graph[u][v]["flow"] + uv_capacity = 0.0 if abs(uv_capacity) < FP_ERROR else uv_capacity + residual_graph[u][v]["capacity"] = uv_capacity + + vu_capacity = residual_graph[u][v]["flow"] - residual_graph[u][v]["lb"] + vu_capacity = 0.0 if abs(vu_capacity) < FP_ERROR else vu_capacity + if dag.has_edge(v, u): + residual_graph[v][u]["capacity"] = vu_capacity + else: + residual_graph.add_edge(v, u, capacity=vu_capacity) + + profiling_min_cut_between_max_flows = ( + time.time() - profiling_min_cut_between_max_flows + ) + logger.info( + "PROFILING PhillipsDessouky::find_min_cut between max flows time: %.10fs", + profiling_min_cut_between_max_flows, + ) + + # Run max flow on the new residual graph. + profiling_max_flow = time.time() + nodes, edges = format_rust_inputs(residual_graph) + + profiling_data_transfer = time.time() + rust_dag = _lowtime_rs.PhillipsDessouky(nodes, source_node, sink_node, edges) + profiling_data_transfer = time.time() - profiling_data_transfer + logger.info( + "PROFILING PhillipsDessouky::find_min_cut data transfer 2 time: %.10fs", + profiling_data_transfer, + ) + + # ohjun: SECOND MAX FLOW + rust_flow_vec = rust_dag.max_flow_depr() + flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) + + profiling_max_flow = time.time() - profiling_max_flow + logger.info( + "PROFILING PhillipsDessouky::find_min_cut maximum_flow_2 time: %.10fs", + profiling_max_flow, + ) + + profiling_min_cut_after_max_flows = time.time() + + # Add additional flow we get to the original graph + for u, v in dag.edges: + dag[u][v]["flow"] += flow_dict[u][v] + dag[u][v]["flow"] -= flow_dict[v][u] + + # Construct the new residual graph. + new_residual = nx.DiGraph(dag) + for u, v in dag.edges: + new_residual[u][v]["flow"] = dag[u][v]["ub"] - dag[u][v]["flow"] + new_residual.add_edge(v, u, flow=dag[u][v]["flow"] - dag[u][v]["lb"]) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("New residual graph") + logger.debug("Number of nodes: %d", new_residual.number_of_nodes()) + logger.debug("Number of edges: %d", new_residual.number_of_edges()) + logger.debug( + "Sum of flow: %f", + sum(attr["flow"] for _, _, attr in new_residual.edges(data=True)), + ) + + # Find the s-t cut induced by the second maximum flow above. + # Only following `flow > 0` edges, find the set of nodes reachable from + # source node. That's the s-set, and the rest is the t-set. + s_set = set[int]() + q: deque[int] = deque() + q.append(source_node) + while q: + cur_id = q.pop() + s_set.add(cur_id) + if cur_id == sink_node: + break + for child_id in list(new_residual.successors(cur_id)): + if ( + child_id not in s_set + and abs(new_residual[cur_id][child_id]["flow"]) > FP_ERROR + ): + q.append(child_id) + t_set = set(new_residual.nodes) - s_set + + profiling_min_cut_after_max_flows = ( + time.time() - profiling_min_cut_after_max_flows + ) + logger.info( + "PROFILING PhillipsDessouky::find_min_cut after max flows time: %.10fs", + profiling_min_cut_after_max_flows, + ) + + return s_set, t_set + + def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: + """In-place annotate the critical DAG with flow capacities.""" + # XXX(JW): Is this always large enough? + # It is necessary to monitor the `cost_change` value in `IterationResult` + # and make sure they are way smaller than this value. Even if the cost + # change is close or larger than this, users can scale down their cost + # value in `ExecutionOption`s. + inf = 10000.0 + for _, _, edge_attr in critical_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + duration = op.duration + # Dummy operations don't constrain the flow. + if op.is_dummy: # noqa: SIM114 + lb, ub = 0.0, inf + # Cannot be sped up or down. + elif duration == op.min_duration == op.max_duration: + lb, ub = 0.0, inf + # Cannot be sped up. + elif duration - 1 < op.min_duration: + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = inf + # Cannot be slowed down. + elif duration + 1 > op.max_duration: + lb = 0.0 + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + else: + # In case the cost model is almost linear, give this edge some room. + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR + + # XXX(JW): This roundiing may not be necessary. + edge_attr["lb"] = lb // FP_ERROR * FP_ERROR + edge_attr["ub"] = ub // FP_ERROR * FP_ERROR diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index f355789..39c2d78 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -1,25 +1,28 @@ -use std::collections::HashMap; +use std::cmp::Ordering; +use std::collections::{HashMap, HashSet}; use ordered_float::OrderedFloat; use pathfinding::directed::edmonds_karp::{ SparseCapacity, Edge, + EKFlows, edmonds_karp }; use std::time::Instant; -use log::info; +use log::{info, debug, Level}; use crate::operation::Operation; use crate::utils; +#[derive(Clone)] pub struct LowtimeGraph { node_ids: Vec, source_node_id: Option, sink_node_id: Option, edges: HashMap>, - preds: HashMap, + preds: HashMap>, num_edges: usize, } @@ -36,13 +39,14 @@ impl LowtimeGraph { } pub fn of_python( - node_ids: Vec, + mut node_ids: Vec, source_node_id: u32, sink_node_id: u32, edges_raw: Vec<((u32, u32), (f64, f64, f64, f64), Option<(bool, u64, u64, u64, u64, u64, u64, u64)>)>, ) -> Self { let mut graph = LowtimeGraph::new(); - graph.node_ids = node_ids; + node_ids.sort(); + graph.node_ids = node_ids.clone(); graph.source_node_id = Some(source_node_id); graph.sink_node_id = Some(sink_node_id); @@ -75,11 +79,17 @@ impl LowtimeGraph { graph } - pub fn max_flow_bak(&self) -> Vec<((u32, u32), f64)> { + pub fn max_flow(&self) -> EKFlows> { let edges_edmonds_karp = self.get_ek_preprocessed_edges(); + // TESTING(ohjun) + // info!("self.node_ids.len(): {}", self.node_ids.len()); + // info!("edges_edmonds_karp.len(): {}", edges_edmonds_karp.len()); + // info!("self.source_node_id.unwrap(): {}", self.source_node_id.unwrap()); + // info!("self.sink_node_id.unwrap(): {}", self.sink_node_id.unwrap()); + let profiling_start = Instant::now(); - let (flows, _max_flow, _min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( + let (flows, max_flow, min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( &self.node_ids, &self.source_node_id.unwrap(), &self.sink_node_id.unwrap(), @@ -88,32 +98,70 @@ impl LowtimeGraph { let profiling_end = Instant::now(); info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { - ((*from, *to), flow.into_inner()) - }).collect(); + (flows, max_flow, min_cut) + } - flows_f64 + pub fn get_source_node_id(&self) -> u32 { + self.source_node_id.unwrap() } - fn get_node_ids(&self) -> Vec { - let mut node_ids: Vec = self.edges.keys().cloned().collect(); - node_ids.sort(); - node_ids + pub fn set_source_node_id(&mut self, new_source_node_id: u32) -> () { + self.source_node_id = Some(new_source_node_id); } - fn add_edge(&mut self, from: u32, to: u32, edge: LowtimeEdge) -> () { - self.edges - .entry(from) - .or_insert_with(HashMap::new) - .insert(to, edge); - self.preds.insert(to, from); - self.num_edges += 1; + pub fn get_sink_node_id(&self) -> u32 { + self.sink_node_id.unwrap() } - fn get_edge(&self, from: u32, to: u32) -> Option<&LowtimeEdge> { - self.edges - .get(&from) - .and_then(|to_edges| to_edges.get(&to)) + pub fn set_sink_node_id(&mut self, new_sink_node_id: u32) -> () { + self.sink_node_id = Some(new_sink_node_id); + } + + pub fn num_nodes(&self) -> usize { + self.node_ids.len() + } + + pub fn num_edges(&self) -> usize { + self.num_edges + } + + pub fn successors(&self, node_id: u32) -> Option> { + self.edges.get(&node_id).map(|succs| succs.keys()) + } + + pub fn predecessors(&self, node_id: u32) -> Option> { + self.preds.get(&node_id).map(|preds| preds.iter()) + } + + pub fn edges(&mut self) -> impl Iterator { + self.edges.iter().flat_map(|(from, inner)| { + inner.iter().map(move |(to, edge)| (from, to, edge)) + }) + } + + pub fn edges_mut(&mut self) -> impl Iterator { + self.edges.iter_mut().flat_map(|(from, inner)| { + inner.iter_mut().map(move |(to, edge)| (from, to, edge)) + }) + } + + pub fn get_node_ids(&self) -> &Vec { + &self.node_ids + } + + pub fn add_node_id(&mut self, node_id: u32) -> () { + assert!(self.node_ids.last().unwrap() < &node_id, "New node ids must be larger than all existing node ids"); + self.node_ids.push(node_id) + } + + pub fn get_edge(&self, from: u32, to: u32) -> &LowtimeEdge { + self.edges.get(&from).unwrap().get(&to).unwrap() + } + + pub fn add_edge(&mut self, from: u32, to: u32, edge: LowtimeEdge) -> () { + self.edges.entry(from).or_insert_with(HashMap::new).insert(to, edge); + self.preds.entry(to).or_insert_with(HashSet::new).insert(from); + self.num_edges += 1; } fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut LowtimeEdge> { @@ -122,7 +170,8 @@ impl LowtimeGraph { .and_then(|to_edges| to_edges.get_mut(&to)) } - fn get_ek_preprocessed_edges(&self, ) -> Vec>> { + // TESTING(ohjun): should make private when testing functions are deleted + pub fn get_ek_preprocessed_edges(&self, ) -> Vec>> { let mut processed_edges = Vec::with_capacity(self.num_edges); processed_edges.extend( self.edges.iter().flat_map(|(from, inner)| @@ -131,8 +180,30 @@ impl LowtimeGraph { ))); processed_edges } + + // TESTING(ohjun) + pub fn print_all_capacities(&self) -> () { + let mut processed_edges = self.get_ek_preprocessed_edges(); + processed_edges.sort_by(|((a_from, a_to), a_cap): &((u32, u32), OrderedFloat), + ((b_from, b_to), b_cap): &((u32, u32), OrderedFloat)| { + // a_from < b_from || (a_from == b_from && a_to < b_to) + let from_cmp = a_from.cmp(&b_from); + if from_cmp == Ordering::Equal { + a_to.cmp(&b_to) + } + else { + from_cmp + } + }); + info!("Rust Printing graph:"); + info!("Num edges: {}", processed_edges.len()); + processed_edges.iter().for_each(|((from, to), cap)| { + info!("{} -> {}: {}", from, to, cap); + }); + } } +#[derive(Clone)] pub struct LowtimeEdge { op: Option, capacity: OrderedFloat, @@ -142,7 +213,7 @@ pub struct LowtimeEdge { } impl LowtimeEdge { - fn new( + pub fn new( op: Option, capacity: f64, flow: f64, @@ -158,23 +229,37 @@ impl LowtimeEdge { } } - fn get_op(&self) -> Option<&Operation> { + pub fn new_only_capacity(capacity: OrderedFloat) -> Self { + LowtimeEdge { + op: None, + capacity, + flow: OrderedFloat(0.0), + ub: OrderedFloat(0.0), + lb: OrderedFloat(0.0), + } + } + + pub fn get_op(&self) -> Option<&Operation> { self.op.as_ref() } - fn get_capacity(&self) -> OrderedFloat { + pub fn get_capacity(&self) -> OrderedFloat { self.capacity } - fn get_flow(&self) -> OrderedFloat { + pub fn set_capacity(&mut self, new_capacity: OrderedFloat) -> () { + self.capacity = new_capacity + } + + pub fn get_flow(&self) -> OrderedFloat { self.flow } - fn get_ub(&self) -> OrderedFloat { + pub fn get_ub(&self) -> OrderedFloat { self.ub } - fn get_lb(&self) -> OrderedFloat { + pub fn get_lb(&self) -> OrderedFloat { self.lb } diff --git a/src/operation.rs b/src/operation.rs index 9d14f36..393007b 100644 --- a/src/operation.rs +++ b/src/operation.rs @@ -1,6 +1,7 @@ // use crate::cost_model::CostModel; +#[derive(Clone)] pub struct Operation { is_dummy: bool, diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index c814fae..d85e16f 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -10,15 +10,16 @@ use pathfinding::directed::edmonds_karp::{ }; use std::time::Instant; -use log::info; +use log::{info, debug, Level}; -use crate::lowtime_graph::LowtimeGraph; +use crate::lowtime_graph::{LowtimeGraph, LowtimeEdge}; use crate::utils; #[pyclass] pub struct PhillipsDessouky { - graph: LowtimeGraph, + dag: LowtimeGraph, + unbound_dag_temp: LowtimeGraph, // TESTING(ohjun) } #[pymethods] @@ -31,18 +32,52 @@ impl PhillipsDessouky { edges_raw: Vec<((u32, u32), (f64, f64, f64, f64), Option<(bool, u64, u64, u64, u64, u64, u64, u64)>)>, ) -> PyResult { Ok(PhillipsDessouky { - graph: LowtimeGraph::of_python( + dag: LowtimeGraph::of_python( node_ids, source_node_id, sink_node_id, edges_raw, - ) + ), + unbound_dag_temp: LowtimeGraph::new(), // TESTING(ohjun) }) } - // TODO(ohjun): this is backup, remove when obsolete - fn max_flow_bak(&self) -> Vec<((u32, u32), f64)> { - self.graph.max_flow_bak() + // TESTING ohjun + fn get_dag_node_ids(&self) -> Vec { + self.dag.get_node_ids().clone() + } + + // TESTING ohjun + fn get_dag_ek_processed_edges(&self) -> Vec<((u32, u32), f64)> { + let rs_edges = self.dag.get_ek_preprocessed_edges(); + let py_edges: Vec<((u32, u32), f64)> = rs_edges.iter().map(|((from, to), cap)| { + ((*from, *to), cap.into_inner()) + }).collect(); + py_edges + } + + // TESTING ohjun + fn get_unbound_dag_node_ids(&self) -> Vec { + self.unbound_dag_temp.get_node_ids().clone() + } + + // TESTING ohjun + fn get_unbound_dag_ek_processed_edges(&self) -> Vec<((u32, u32), f64)> { + let rs_edges = self.unbound_dag_temp.get_ek_preprocessed_edges(); + let py_edges: Vec<((u32, u32), f64)> = rs_edges.iter().map(|((from, to), cap)| { + ((*from, *to), cap.into_inner()) + }).collect(); + py_edges + } + + // TESTING(ohjun) + fn max_flow_depr(&self) -> Vec<((u32, u32), f64)> { + info!("CALLING MAX FLOW FROM max_flow_depr"); + let (flows, _, _) = self.dag.max_flow(); + let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { + ((*from, *to), flow.into_inner()) + }).collect(); + flows_f64 } // TODO(ohjun): iteratively implement/verify in _wip until done, @@ -50,9 +85,91 @@ impl PhillipsDessouky { // fn find_min_cut(&self) -> (HashSet, HashSet) { // } - // fn find_min_cut_wip(&self) -> Vec<((u32, u32), f64)> { + fn find_min_cut_wip(&mut self) -> Vec<((u32, u32), f64)> { + // In order to solve max flow on edges with both lower and upper bounds, + // we first need to convert it to another DAG that only has upper bounds. + let mut unbound_dag: LowtimeGraph = self.dag.clone(); - // } + // For every edge, capacity = ub - lb. + unbound_dag.edges_mut().for_each(|(_from, _to, edge)| + edge.set_capacity(edge.get_ub() - edge.get_lb()) + ); + + // Add a new node s', which will become the new source node. + // We constructed the AOA DAG, so we know that node IDs are integers. + let s_prime_id = unbound_dag.get_node_ids().last().unwrap() + 1; + unbound_dag.add_node_id(s_prime_id); + + // For every node u in the original graph, add an edge (s', u) with capacity + // equal to the sum of all lower bounds of u's parents. + let orig_node_ids = self.dag.get_node_ids(); + for u in orig_node_ids.iter() { + let mut capacity = OrderedFloat(0.0); + if let Some(preds) = unbound_dag.predecessors(*u) { + capacity = preds.fold(OrderedFloat(0.0), |acc, pred_id| { + acc + unbound_dag.get_edge(*pred_id, *u).get_lb() + }); + } + unbound_dag.add_edge(s_prime_id, *u, LowtimeEdge::new_only_capacity(capacity)); + } + + // Add a new node t', which will become the new sink node. + let t_prime_id = s_prime_id + 1; + unbound_dag.add_node_id(t_prime_id); + + // For every node u in the original graph, add an edge (u, t') with capacity + // equal to the sum of all lower bounds of u's children. + for u in orig_node_ids.iter() { + let mut capacity = OrderedFloat(0.0); + if let Some(succs) = unbound_dag.successors(*u) { + capacity = succs.fold(OrderedFloat(0.0), |acc, succ_id| { + acc + unbound_dag.get_edge(*u, *succ_id).get_lb() + }); + } + unbound_dag.add_edge(*u, t_prime_id, LowtimeEdge::new_only_capacity(capacity)); + } + + if log::log_enabled!(Level::Debug) { + debug!("Unbound DAG"); + debug!("Number of nodes: {}", unbound_dag.num_nodes()); + debug!("Number of edges: {}", unbound_dag.num_edges()); + let total_capacity = unbound_dag.edges() + .fold(OrderedFloat(0.0), |acc, (_from, _to, edge)| acc + edge.get_capacity()); + debug!("Sum of capacities: {}", total_capacity); + } + + // Add an edge from t to s with infinite capacity. + unbound_dag.add_edge( + unbound_dag.get_sink_node_id(), + unbound_dag.get_source_node_id(), + LowtimeEdge::new_only_capacity(OrderedFloat(f64::INFINITY)), + ); + + // Update source and sink on unbound_dag + unbound_dag.set_source_node_id(s_prime_id); + unbound_dag.set_sink_node_id(t_prime_id); + + // First Max Flow + info!("CALLING MAX FLOW FROM find_min_cut_wip"); // TESTING(ohjun) + let (flows, _, _) = unbound_dag.max_flow(); + + if log::log_enabled!(Level::Debug) { + debug!("After first max flow"); + let total_flow = flows.iter() + .fold(OrderedFloat(0.0), |acc, ((_from, _to), flow)| acc + flow); + debug!("Sum of all flow values: {}", total_flow); + } + + + let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { + ((*from, *to), flow.into_inner()) + }).collect(); + + //// TESTING(ohjun) + self.unbound_dag_temp = unbound_dag; + + flows_f64 + } } // not exposed to Python From 3e9333740d7d6c1dfd4c38b989617247be3bc1ae Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 20 Nov 2024 14:27:23 -0500 Subject: [PATCH 17/55] working: rename solvers --- lowtime/{solver.testing.py => solver.py} | 0 lowtime/solver.up_to_first.py | 757 +++++++++++++++++++++++ 2 files changed, 757 insertions(+) rename lowtime/{solver.testing.py => solver.py} (100%) create mode 100644 lowtime/solver.up_to_first.py diff --git a/lowtime/solver.testing.py b/lowtime/solver.py similarity index 100% rename from lowtime/solver.testing.py rename to lowtime/solver.py diff --git a/lowtime/solver.up_to_first.py b/lowtime/solver.up_to_first.py new file mode 100644 index 0000000..c061c37 --- /dev/null +++ b/lowtime/solver.up_to_first.py @@ -0,0 +1,757 @@ +# Copyright (C) 2023 Jae-Won Chung +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" + + +from __future__ import annotations + +import time +import sys +import logging +from collections import deque +from collections.abc import Generator + +import networkx as nx +from attrs import define, field + +from lowtime.operation import Operation +from lowtime.graph_utils import ( + aon_dag_to_aoa_dag, + aoa_to_critical_dag, + get_critical_aoa_dag_total_time, + get_total_cost, +) +from lowtime.exceptions import LowtimeFlowError +from lowtime import _lowtime_rs + +FP_ERROR = 1e-6 + +logger = logging.getLogger(__name__) + + +@define +class IterationResult: + """Holds results after one PD iteration. + + Attributes: + iteration: The number of optimization iterations experienced by the DAG. + cost_change: The increase in cost from reducing the DAG's + quantized execution time by 1. + cost: The current total cost of the DAG. + quant_time: The current quantized total execution time of the DAG. + unit_time: The unit time used for time quantization. + real_time: The current real (de-quantized) total execution time of the DAG. + """ + + iteration: int + cost_change: float + cost: float + quant_time: int + unit_time: float + real_time: float = field(init=False) + + def __attrs_post_init__(self) -> None: + """Set `real_time` after initialization.""" + self.real_time = self.quant_time * self.unit_time + + +class PhillipsDessouky: + """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" + + def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: + """Initialize the Phillips-Dessouky solver. + + Assumptions: + - The graph is a Directed Acyclic Graph (DAG). + - Operations are annotated on nodes with name `attr_name`. + - There is only one source (entry) node. The source node is annotated as + `dag.graph["source_node"]`. + - There is only one sink (exit) node. The sink node is annotated as + `dag.graph["sink_node"]`. + - The `unit_time` attribute of every operation is the same. + + Args: + dag: A networkx DiGraph object that represents the computation DAG. + The aforementioned assumptions should hold for the DAG. + attr_name: The name of the attribute on nodes that holds the operation + object. Defaults to "op". + """ + self.attr_name = attr_name + + # Run checks on the DAG and cache some properties. + # Check: It's a DAG. + if not nx.is_directed_acyclic_graph(dag): + raise ValueError("The graph should be a Directed Acyclic Graph.") + + # Check: Only one source node that matches annotation. + if (source_node := dag.graph.get("source_node")) is None: + raise ValueError("The graph should have a `source_node` attribute.") + source_node_candidates = [] + for node_id, in_degree in dag.in_degree(): + if in_degree == 0: + source_node_candidates.append(node_id) + if len(source_node_candidates) == 0: + raise ValueError( + "Found zero nodes with in-degree 0. Cannot determine source node." + ) + if len(source_node_candidates) > 1: + raise ValueError( + f"Expecting only one source node, found {source_node_candidates}." + ) + if (detected_source_node := source_node_candidates[0]) != source_node: + raise ValueError( + f"Detected source node ({detected_source_node}) does not match " + f"the annotated source node ({source_node})." + ) + + # Check: Only one sink node that matches annotation. + if (sink_node := dag.graph.get("sink_node")) is None: + raise ValueError("The graph should have a `sink_node` attribute.") + sink_node_candidates = [] + for node_id, out_degree in dag.out_degree(): + if out_degree == 0: + sink_node_candidates.append(node_id) + if len(sink_node_candidates) == 0: + raise ValueError( + "Found zero nodes with out-degree 0. Cannot determine sink node." + ) + if len(sink_node_candidates) > 1: + raise ValueError( + f"Expecting only one sink node, found {sink_node_candidates}." + ) + if (detected_sink_node := sink_node_candidates[0]) != sink_node: + raise ValueError( + f"Detected sink node ({detected_sink_node}) does not match " + f"the annotated sink node ({sink_node})." + ) + + # Check: The `unit_time` attributes of every operation should be the same. + unit_time_candidates = set[float]() + for _, node_attr in dag.nodes(data=True): + if self.attr_name in node_attr: + op: Operation = node_attr[self.attr_name] + if op.is_dummy: + continue + unit_time_candidates.update( + option.unit_time for option in op.spec.options.options + ) + if len(unit_time_candidates) == 0: + raise ValueError( + "Found zero operations in the graph. Make sure you " + f"added operations as node attributes with key `{self.attr_name}`.", + ) + if len(unit_time_candidates) > 1: + raise ValueError( + f"Expecting the same `unit_time` across all operations, " + f"found {unit_time_candidates}." + ) + + self.aon_dag = dag + self.unit_time = unit_time_candidates.pop() + + def run(self) -> Generator[IterationResult, None, None]: + """Run the algorithm and yield a DAG after each iteration. + + The solver will not deepcopy operations on the DAG but rather in-place modify + them for speed. The caller should deepcopy the operations or the DAG if needed + before running the next iteration. + + Upon yield, it is guaranteed that the earliest/latest start/finish time values + of all operations are up to date w.r.t. the `duration` of each operation. + """ + logger.info("Starting Phillips-Dessouky solver.") + profiling_setup = time.time() + + # Convert the original activity-on-node DAG to activity-on-arc DAG form. + # AOA DAGs are purely internal. All public input and output of this class + # should be in AON form. + aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) + + # Estimate the minimum execution time of the DAG by setting every operation + # to run at its minimum duration. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.min_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + min_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected minimum quantized execution time: %d", min_time) + + # Estimated the maximum execution time of the DAG by setting every operation + # to run at its maximum duration. This is also our initial start point. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.max_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + max_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected maximum quantized execution time: %d", max_time) + + num_iters = max_time - min_time + 1 + logger.info("Expected number of PD iterations: %d", num_iters) + + profiling_setup = time.time() - profiling_setup + logger.info( + "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup + ) + + # Iteratively reduce the execution time of the DAG. + for iteration in range(sys.maxsize): + logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) + profiling_iter = time.time() + + # At this point, `critical_dag` always exists and is what we want. + # For the first iteration, the critical DAG is computed before the for + # loop in order to estimate the number of iterations. For subsequent + # iterations, the critcal DAG is computed after each iteration in + # in order to construct `IterationResult`. + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Critical DAG:") + logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) + logger.debug("Number of edges: %d", critical_dag.number_of_edges()) + non_dummy_ops = [ + attr[self.attr_name] + for _, _, attr in critical_dag.edges(data=True) + if not attr[self.attr_name].is_dummy + ] + logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) + logger.debug( + "Sum of non-dummy durations: %d", + sum(op.duration for op in non_dummy_ops), + ) + + profiling_annotate = time.time() + self.annotate_capacities(critical_dag) + profiling_annotate = time.time() - profiling_annotate + logger.info( + "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", + profiling_annotate, + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Capacity DAG:") + logger.debug( + "Total lb value: %f", + sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), + ) + logger.debug( + "Total ub value: %f", + sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), + ) + + try: + profiling_min_cut = time.time() + s_set, t_set = self.find_min_cut(critical_dag) + profiling_min_cut = time.time() - profiling_min_cut + logger.info( + "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", + profiling_min_cut, + ) + except LowtimeFlowError as e: + logger.info("Could not find minimum cut: %s", e.message) + logger.info("Terminating PD iteration.") + break + + profiling_reduce = time.time() + cost_change = self.reduce_durations(critical_dag, s_set, t_set) + profiling_reduce = time.time() - profiling_reduce + logger.info( + "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", + profiling_reduce, + ) + + if cost_change == float("inf") or abs(cost_change) < FP_ERROR: + logger.info("No further time reduction possible.") + logger.info("Terminating PD iteration.") + break + + # Earliest/latest start/finish times on operations also annotated here. + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + + # We directly modify operation attributes in the DAG, so after we + # ran one iteration, the AON DAG holds updated attributes. + result = IterationResult( + iteration=iteration + 1, + cost_change=cost_change, + cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), + quant_time=get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ), + unit_time=self.unit_time, + ) + logger.info("%s", result) + profiling_iter = time.time() - profiling_iter + logger.info( + "PROFILING PhillipsDessouky::run single iteration time: %.10fs", + profiling_iter, + ) + yield result + + def reduce_durations( + self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] + ) -> float: + """Modify operation durations to reduce the DAG's execution time by 1.""" + speed_up_edges: list[Operation] = [] + for node_id in s_set: + for child_id in list(dag.successors(node_id)): + if child_id in t_set: + op: Operation = dag[node_id][child_id][self.attr_name] + speed_up_edges.append(op) + + slow_down_edges: list[Operation] = [] + for node_id in t_set: + for child_id in list(dag.successors(node_id)): + if child_id in s_set: + op: Operation = dag[node_id][child_id][self.attr_name] + slow_down_edges.append(op) + + if not speed_up_edges: + logger.info("No speed up candidate operations.") + return 0.0 + + cost_change = 0.0 + + # Reduce the duration of edges (speed up) by quant_time 1. + for op in speed_up_edges: + if op.is_dummy: + logger.info("Cannot speed up dummy operation.") + return float("inf") + if op.duration - 1 < op.min_duration: + logger.info("Operation %s has reached the limit of speed up", op) + return float("inf") + cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) + op_before_str = str(op) + op.duration -= 1 + logger.info("Sped up %s to %s", op_before_str, op) + + # Increase the duration of edges (slow down) by quant_time 1. + for op in slow_down_edges: + # Dummy edges can always be slowed down. + if op.is_dummy: + logger.info("Slowed down DummyOperation (didn't really slowdown).") + continue + elif op.duration + 1 > op.max_duration: + logger.info("Operation %s has reached the limit of slow down", op) + return float("inf") + cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) + before_op_str = str(op) + op.duration += 1 + logger.info("Slowed down %s to %s", before_op_str, op) + + return cost_change + + def find_min_cut(self, dag: nx.DiGraph) -> tuple[set[int], set[int]]: + """Find the min cut of the DAG annotated with lower/upper bound flow capacities. + + Assumptions: + - The capacity DAG is in AOA form. + - The capacity DAG has been annotated with `lb` and `ub` attributes on edges, + representing the lower and upper bounds of the flow on the edge. + + Returns: + A tuple of (s_set, t_set) where s_set is the set of nodes on the source side + of the min cut and t_set is the set of nodes on the sink side of the min cut. + Returns None if no feasible flow exists. + + Raises: + LowtimeFlowError: When no feasible flow exists. + """ + # Helper function for Rust interop + def format_rust_inputs( + dag: nx.DiGraph, + ) -> tuple[ + nx.classes.reportviews.NodeView, + list[ + tuple[ + tuple[int, int], + tuple[float, float, float, float], + tuple[bool, int, int, int, int, int, int, int] | None, + ] + ], + ]: + nodes = dag.nodes + edges = [] + for from_, to_, edge_attrs in dag.edges(data=True): + op_details = ( + None + if self.attr_name not in edge_attrs + else ( + edge_attrs[self.attr_name].is_dummy, + edge_attrs[self.attr_name].duration, + edge_attrs[self.attr_name].max_duration, + edge_attrs[self.attr_name].min_duration, + edge_attrs[self.attr_name].earliest_start, + edge_attrs[self.attr_name].latest_start, + edge_attrs[self.attr_name].earliest_finish, + edge_attrs[self.attr_name].latest_finish, + ) + ) + edges.append( + ( + (from_, to_), + ( + edge_attrs.get("capacity", 0), + edge_attrs.get("flow", 0), + edge_attrs.get("ub", 0), + edge_attrs.get("lb", 0), + ), + op_details, + ) + ) + return nodes, edges + + ########### NEW ########### + # Construct Rust runner with input (critical) dag + ohjun_nodes, ohjun_edges = format_rust_inputs(dag) + # print(ohjun_nodes) + # print(ohjun_edges) + ohjun_rust_runner = _lowtime_rs.PhillipsDessouky( + ohjun_nodes, + dag.graph["source_node"], + dag.graph["sink_node"], + ohjun_edges + ) + ########### NEW ########### + + profiling_min_cut_setup = time.time() + source_node = dag.graph["source_node"] + sink_node = dag.graph["sink_node"] + + # In order to solve max flow on edges with both lower and upper bounds, + # we first need to convert it to another DAG that only has upper bounds. + unbound_dag = nx.DiGraph(dag) + + # For every edge, capacity = ub - lb. + for _, _, edge_attrs in unbound_dag.edges(data=True): + edge_attrs["capacity"] = edge_attrs["ub"] - edge_attrs["lb"] + + # Add a new node s', which will become the new source node. + # We constructed the AOA DAG, so we know that node IDs are integers. + node_ids: list[int] = list(unbound_dag.nodes) + s_prime_id = max(node_ids) + 1 + unbound_dag.add_node(s_prime_id) + + # For every node u in the original graph, add an edge (s', u) with capacity + # equal to the sum of all lower bounds of u's parents. + for u in dag.nodes: + capacity = 0.0 + for pred_id in dag.predecessors(u): + capacity += dag[pred_id][u]["lb"] + unbound_dag.add_edge(s_prime_id, u, capacity=capacity) + + # Add a new node t', which will become the new sink node. + t_prime_id = s_prime_id + 1 + unbound_dag.add_node(t_prime_id) + + # For every node u in the original graph, add an edge (u, t') with capacity + # equal to the sum of all lower bounds of u's children. + for u in dag.nodes: + capacity = 0.0 + for succ_id in dag.successors(u): + capacity += dag[u][succ_id]["lb"] + unbound_dag.add_edge(u, t_prime_id, capacity=capacity) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Unbound DAG") + logger.debug("Number of nodes: %d", unbound_dag.number_of_nodes()) + logger.debug("Number of edges: %d", unbound_dag.number_of_edges()) + logger.debug( + "Sum of capacities: %f", + sum(attr["capacity"] for _, _, attr in unbound_dag.edges(data=True)), + ) + + # Add an edge from t to s with infinite capacity. + unbound_dag.add_edge( + sink_node, + source_node, + capacity=float("inf"), + ) + + profiling_min_cut_setup = time.time() - profiling_min_cut_setup + logger.info( + "PROFILING PhillipsDessouky::find_min_cut setup time: %.10fs", + profiling_min_cut_setup, + ) + + # Helper function for Rust interop + # Rust's pathfinding::edmonds_karp does not return edges with 0 flow, + # but nx.max_flow does. So we fill in the 0s and empty nodes. + def reformat_rust_flow_to_dict( + flow_vec: list[tuple[tuple[int, int], float]], dag: nx.DiGraph + ) -> dict[int, dict[int, float]]: + flow_dict = dict() + for u in dag.nodes: + flow_dict[u] = dict() + for v in dag.successors(u): + flow_dict[u][v] = 0.0 + + for (u, v), cap in flow_vec: + flow_dict[u][v] = cap + + return flow_dict + + # We're done with constructing the DAG with only flow upper bounds. + # Find the maximum flow on this DAG. + profiling_max_flow = time.time() + nodes, edges = format_rust_inputs(unbound_dag) + + profiling_data_transfer = time.time() + rust_dag = _lowtime_rs.PhillipsDessouky(nodes, s_prime_id, t_prime_id, edges) + profiling_data_transfer = time.time() - profiling_data_transfer + logger.info( + "PROFILING PhillipsDessouky::find_min_cut data transfer time: %.10fs", + profiling_data_transfer, + ) + + ##### TESTING print all edges and capacities + # logger.info(f"s_prime_id: {s_prime_id}") + # logger.info(f"t_prime_id: {t_prime_id}") + # logger.info("Python Printing graph:") + # logger.info(f"Num edges: {len(unbound_dag.edges())}") + # for from_, to_, edge_attrs in unbound_dag.edges(data=True): + # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") + + # ohjun: FIRST MAX FLOW + rust_flow_vec = rust_dag.max_flow_depr() + flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) + + ############# NEW ############# + # TEMP(ohjun): in current wip version, do everything until after 1st max flow in Rust + ohjun_rust_flow_vec = ohjun_rust_runner.find_min_cut_wip() + ohjun_flow_dict = reformat_rust_flow_to_dict(ohjun_rust_flow_vec, unbound_dag) + + depr_node_ids = rust_dag.get_dag_node_ids() + depr_edges = rust_dag.get_dag_ek_processed_edges() + new_node_ids = ohjun_rust_runner.get_unbound_dag_node_ids() + new_edges = ohjun_rust_runner.get_unbound_dag_ek_processed_edges() + print(f"depr_node_ids: {depr_node_ids}") + print(f"new_node_ids: {new_node_ids}") + assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" + assert depr_node_ids == new_node_ids, "DIFF in node_ids" + assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" + if sorted(depr_edges) != sorted(new_edges): + for depr_edge, new_edge in zip(sorted(depr_edges), sorted(new_edges)): + if depr_edge == new_edge: + logger.info("edges EQUAL") + logger.info(f"depr_edge: {depr_edge}") + logger.info(f"new_edge : {new_edge}") + else: + logger.info("edges DIFFERENT") + logger.info(f"depr_edge: {depr_edge}") + logger.info(f"new_edge : {new_edge}") + assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" + + def print_dict(d): + for from_, inner in d.items(): + for to_, flow in inner.items(): + logger.info(f'{from_} -> {to_}: {flow}') + logger.info('flow_dict:') + # print_dict(flow_dict) + logger.info('ohjun_flow_dict:') + # print_dict(ohjun_flow_dict) + assert flow_dict == ohjun_flow_dict, "flow dicts were different :(" + ############# NEW ############# + + profiling_max_flow = time.time() - profiling_max_flow + logger.info( + "PROFILING PhillipsDessouky::find_min_cut maximum_flow_1 time: %.10fs", + profiling_max_flow, + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("After first max flow") + total_flow = 0.0 + for d in flow_dict.values(): + for flow in d.values(): + total_flow += flow + logger.debug("Sum of all flow values: %f", total_flow) + + profiling_min_cut_between_max_flows = time.time() + + # Check if residual graph is saturated. If so, we have a feasible flow. + for u in unbound_dag.successors(s_prime_id): + if ( + abs(flow_dict[s_prime_id][u] - unbound_dag[s_prime_id][u]["capacity"]) + > FP_ERROR + ): + logger.error( + "s' -> %s unsaturated (flow: %s, capacity: %s)", + u, + flow_dict[s_prime_id][u], + unbound_dag[s_prime_id][u]["capacity"], + ) + raise LowtimeFlowError( + "ERROR: Max flow on unbounded DAG didn't saturate." + ) + for u in unbound_dag.predecessors(t_prime_id): + if ( + abs(flow_dict[u][t_prime_id] - unbound_dag[u][t_prime_id]["capacity"]) + > FP_ERROR + ): + logger.error( + "%s -> t' unsaturated (flow: %s, capacity: %s)", + u, + flow_dict[u][t_prime_id], + unbound_dag[u][t_prime_id]["capacity"], + ) + raise LowtimeFlowError( + "ERROR: Max flow on unbounded DAG didn't saturate." + ) + + # We have a feasible flow. Construct a new residual graph with the same + # shape as the capacity DAG so that we can find the min cut. + # First, retrieve the flow amounts to the original capacity graph, where for + # each edge u -> v, the flow amount is `flow + lb`. + for u, v in dag.edges: + dag[u][v]["flow"] = flow_dict[u][v] + dag[u][v]["lb"] + + # Construct a new residual graph (same shape as capacity DAG) with + # u -> v capacity `ub - flow` and v -> u capacity `flow - lb`. + residual_graph = nx.DiGraph(dag) + for u, v in dag.edges: + # Rounding small negative values to 0.0 avoids Rust-side + # pathfinding::edmonds_karp from entering unreachable code. + # This edge case did not exist in Python-side nx.max_flow call. + uv_capacity = residual_graph[u][v]["ub"] - residual_graph[u][v]["flow"] + uv_capacity = 0.0 if abs(uv_capacity) < FP_ERROR else uv_capacity + residual_graph[u][v]["capacity"] = uv_capacity + + vu_capacity = residual_graph[u][v]["flow"] - residual_graph[u][v]["lb"] + vu_capacity = 0.0 if abs(vu_capacity) < FP_ERROR else vu_capacity + if dag.has_edge(v, u): + residual_graph[v][u]["capacity"] = vu_capacity + else: + residual_graph.add_edge(v, u, capacity=vu_capacity) + + profiling_min_cut_between_max_flows = ( + time.time() - profiling_min_cut_between_max_flows + ) + logger.info( + "PROFILING PhillipsDessouky::find_min_cut between max flows time: %.10fs", + profiling_min_cut_between_max_flows, + ) + + # Run max flow on the new residual graph. + profiling_max_flow = time.time() + nodes, edges = format_rust_inputs(residual_graph) + + profiling_data_transfer = time.time() + rust_dag = _lowtime_rs.PhillipsDessouky(nodes, source_node, sink_node, edges) + profiling_data_transfer = time.time() - profiling_data_transfer + logger.info( + "PROFILING PhillipsDessouky::find_min_cut data transfer 2 time: %.10fs", + profiling_data_transfer, + ) + + # ohjun: SECOND MAX FLOW + rust_flow_vec = rust_dag.max_flow_depr() + flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) + + profiling_max_flow = time.time() - profiling_max_flow + logger.info( + "PROFILING PhillipsDessouky::find_min_cut maximum_flow_2 time: %.10fs", + profiling_max_flow, + ) + + profiling_min_cut_after_max_flows = time.time() + + # Add additional flow we get to the original graph + for u, v in dag.edges: + dag[u][v]["flow"] += flow_dict[u][v] + dag[u][v]["flow"] -= flow_dict[v][u] + + # Construct the new residual graph. + new_residual = nx.DiGraph(dag) + for u, v in dag.edges: + new_residual[u][v]["flow"] = dag[u][v]["ub"] - dag[u][v]["flow"] + new_residual.add_edge(v, u, flow=dag[u][v]["flow"] - dag[u][v]["lb"]) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("New residual graph") + logger.debug("Number of nodes: %d", new_residual.number_of_nodes()) + logger.debug("Number of edges: %d", new_residual.number_of_edges()) + logger.debug( + "Sum of flow: %f", + sum(attr["flow"] for _, _, attr in new_residual.edges(data=True)), + ) + + # Find the s-t cut induced by the second maximum flow above. + # Only following `flow > 0` edges, find the set of nodes reachable from + # source node. That's the s-set, and the rest is the t-set. + s_set = set[int]() + q: deque[int] = deque() + q.append(source_node) + while q: + cur_id = q.pop() + s_set.add(cur_id) + if cur_id == sink_node: + break + for child_id in list(new_residual.successors(cur_id)): + if ( + child_id not in s_set + and abs(new_residual[cur_id][child_id]["flow"]) > FP_ERROR + ): + q.append(child_id) + t_set = set(new_residual.nodes) - s_set + + profiling_min_cut_after_max_flows = ( + time.time() - profiling_min_cut_after_max_flows + ) + logger.info( + "PROFILING PhillipsDessouky::find_min_cut after max flows time: %.10fs", + profiling_min_cut_after_max_flows, + ) + + return s_set, t_set + + def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: + """In-place annotate the critical DAG with flow capacities.""" + # XXX(JW): Is this always large enough? + # It is necessary to monitor the `cost_change` value in `IterationResult` + # and make sure they are way smaller than this value. Even if the cost + # change is close or larger than this, users can scale down their cost + # value in `ExecutionOption`s. + inf = 10000.0 + for _, _, edge_attr in critical_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + duration = op.duration + # Dummy operations don't constrain the flow. + if op.is_dummy: # noqa: SIM114 + lb, ub = 0.0, inf + # Cannot be sped up or down. + elif duration == op.min_duration == op.max_duration: + lb, ub = 0.0, inf + # Cannot be sped up. + elif duration - 1 < op.min_duration: + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = inf + # Cannot be slowed down. + elif duration + 1 > op.max_duration: + lb = 0.0 + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + else: + # In case the cost model is almost linear, give this edge some room. + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR + + # XXX(JW): This roundiing may not be necessary. + edge_attr["lb"] = lb // FP_ERROR * FP_ERROR + edge_attr["ub"] = ub // FP_ERROR * FP_ERROR From 86fe7e33b2e3b79498bfe38dcbe94ef5593df9b0 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 20 Nov 2024 14:30:01 -0500 Subject: [PATCH 18/55] set up next goal state: up to 2nd max flow --- lowtime/solver.py | 75 ++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/lowtime/solver.py b/lowtime/solver.py index c061c37..cad111f 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -533,43 +533,6 @@ def reformat_rust_flow_to_dict( rust_flow_vec = rust_dag.max_flow_depr() flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) - ############# NEW ############# - # TEMP(ohjun): in current wip version, do everything until after 1st max flow in Rust - ohjun_rust_flow_vec = ohjun_rust_runner.find_min_cut_wip() - ohjun_flow_dict = reformat_rust_flow_to_dict(ohjun_rust_flow_vec, unbound_dag) - - depr_node_ids = rust_dag.get_dag_node_ids() - depr_edges = rust_dag.get_dag_ek_processed_edges() - new_node_ids = ohjun_rust_runner.get_unbound_dag_node_ids() - new_edges = ohjun_rust_runner.get_unbound_dag_ek_processed_edges() - print(f"depr_node_ids: {depr_node_ids}") - print(f"new_node_ids: {new_node_ids}") - assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" - assert depr_node_ids == new_node_ids, "DIFF in node_ids" - assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" - if sorted(depr_edges) != sorted(new_edges): - for depr_edge, new_edge in zip(sorted(depr_edges), sorted(new_edges)): - if depr_edge == new_edge: - logger.info("edges EQUAL") - logger.info(f"depr_edge: {depr_edge}") - logger.info(f"new_edge : {new_edge}") - else: - logger.info("edges DIFFERENT") - logger.info(f"depr_edge: {depr_edge}") - logger.info(f"new_edge : {new_edge}") - assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" - - def print_dict(d): - for from_, inner in d.items(): - for to_, flow in inner.items(): - logger.info(f'{from_} -> {to_}: {flow}') - logger.info('flow_dict:') - # print_dict(flow_dict) - logger.info('ohjun_flow_dict:') - # print_dict(ohjun_flow_dict) - assert flow_dict == ohjun_flow_dict, "flow dicts were different :(" - ############# NEW ############# - profiling_max_flow = time.time() - profiling_max_flow logger.info( "PROFILING PhillipsDessouky::find_min_cut maximum_flow_1 time: %.10fs", @@ -665,6 +628,44 @@ def print_dict(d): rust_flow_vec = rust_dag.max_flow_depr() flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) + ############# NEW ############# + # TEMP(ohjun): in current wip version, do everything until after 1st max flow in Rust + ohjun_rust_flow_vec = ohjun_rust_runner.find_min_cut_wip() + ohjun_flow_dict = reformat_rust_flow_to_dict(ohjun_rust_flow_vec, residual_graph) + + depr_node_ids = rust_dag.get_dag_node_ids() + depr_edges = rust_dag.get_dag_ek_processed_edges() + new_node_ids = ohjun_rust_runner.get_unbound_dag_node_ids() + new_edges = ohjun_rust_runner.get_unbound_dag_ek_processed_edges() + + # print(f"depr_node_ids: {depr_node_ids}") + # print(f"new_node_ids: {new_node_ids}") + assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" + assert depr_node_ids == new_node_ids, "DIFF in node_ids" + assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" + # if sorted(depr_edges) != sorted(new_edges): + # for depr_edge, new_edge in zip(sorted(depr_edges), sorted(new_edges)): + # if depr_edge == new_edge: + # logger.info("edges EQUAL") + # logger.info(f"depr_edge: {depr_edge}") + # logger.info(f"new_edge : {new_edge}") + # else: + # logger.info("edges DIFFERENT") + # logger.info(f"depr_edge: {depr_edge}") + # logger.info(f"new_edge : {new_edge}") + assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" + + # def print_dict(d): + # for from_, inner in d.items(): + # for to_, flow in inner.items(): + # logger.info(f'{from_} -> {to_}: {flow}') + # logger.info('flow_dict:') + # print_dict(flow_dict) + # logger.info('ohjun_flow_dict:') + # print_dict(ohjun_flow_dict) + assert flow_dict == ohjun_flow_dict, "flow dicts were different :(" + ############# NEW ############# + profiling_max_flow = time.time() - profiling_max_flow logger.info( "PROFILING PhillipsDessouky::find_min_cut maximum_flow_2 time: %.10fs", From 065e948c9984bca572a88ce9de7f77907b1e6885 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 20 Nov 2024 14:31:10 -0500 Subject: [PATCH 19/55] minor solver touches --- lowtime/solver.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lowtime/solver.py b/lowtime/solver.py index cad111f..d0cb8f3 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -521,14 +521,6 @@ def reformat_rust_flow_to_dict( profiling_data_transfer, ) - ##### TESTING print all edges and capacities - # logger.info(f"s_prime_id: {s_prime_id}") - # logger.info(f"t_prime_id: {t_prime_id}") - # logger.info("Python Printing graph:") - # logger.info(f"Num edges: {len(unbound_dag.edges())}") - # for from_, to_, edge_attrs in unbound_dag.edges(data=True): - # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") - # ohjun: FIRST MAX FLOW rust_flow_vec = rust_dag.max_flow_depr() flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) @@ -629,7 +621,15 @@ def reformat_rust_flow_to_dict( flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) ############# NEW ############# - # TEMP(ohjun): in current wip version, do everything until after 1st max flow in Rust + ##### TESTING print all edges and capacities + # logger.info(f"s_prime_id: {s_prime_id}") + # logger.info(f"t_prime_id: {t_prime_id}") + # logger.info("Python Printing graph:") + # logger.info(f"Num edges: {len(unbound_dag.edges())}") + # for from_, to_, edge_attrs in unbound_dag.edges(data=True): + # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") + + # TEMP(ohjun): in current wip version, do everything until after 2nd max flow in Rust ohjun_rust_flow_vec = ohjun_rust_runner.find_min_cut_wip() ohjun_flow_dict = reformat_rust_flow_to_dict(ohjun_rust_flow_vec, residual_graph) From 3d55a54f505c4a67641a23c167ce66d83805e210 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 20 Nov 2024 14:35:40 -0500 Subject: [PATCH 20/55] note very hard to find bug caused by difference in py/rs impl --- src/phillips_dessouky.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index d85e16f..5035ce4 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -146,6 +146,12 @@ impl PhillipsDessouky { ); // Update source and sink on unbound_dag + // Note: This part is not in original Python solver, because the original solver + // never explicitly updates the source and sink; it simply passes in the + // new node_ids directly to max_flow. However, in this codebase, it makes + // sense to have LowtimeGraph be responsible for tracking its source/sink. + // I am noting this because it resulted in an extremely hard-to-find bug, + // and in the course of rewriting further a similar bug may appear again. unbound_dag.set_source_node_id(s_prime_id); unbound_dag.set_sink_node_id(t_prime_id); @@ -160,7 +166,8 @@ impl PhillipsDessouky { debug!("Sum of all flow values: {}", total_flow); } - + // TODO(ohjun): rewrite up to 2nd max flow + let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { ((*from, *to), flow.into_inner()) }).collect(); From 0f823b66c61c8ba47898b9c4dfadbdab266e85c3 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Thu, 21 Nov 2024 18:57:53 -0500 Subject: [PATCH 21/55] working: replace everything up to 2nd max flow --- lowtime/solver.py | 9 +-- src/lowtime_graph.rs | 37 ++++++++--- src/phillips_dessouky.rs | 128 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 155 insertions(+), 19 deletions(-) diff --git a/lowtime/solver.py b/lowtime/solver.py index d0cb8f3..0cece09 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -424,6 +424,7 @@ def format_rust_inputs( # print(ohjun_nodes) # print(ohjun_edges) ohjun_rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, ohjun_nodes, dag.graph["source_node"], dag.graph["sink_node"], @@ -514,7 +515,7 @@ def reformat_rust_flow_to_dict( nodes, edges = format_rust_inputs(unbound_dag) profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(nodes, s_prime_id, t_prime_id, edges) + rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, s_prime_id, t_prime_id, edges) profiling_data_transfer = time.time() - profiling_data_transfer logger.info( "PROFILING PhillipsDessouky::find_min_cut data transfer time: %.10fs", @@ -609,7 +610,7 @@ def reformat_rust_flow_to_dict( nodes, edges = format_rust_inputs(residual_graph) profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(nodes, source_node, sink_node, edges) + rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, source_node, sink_node, edges) profiling_data_transfer = time.time() - profiling_data_transfer logger.info( "PROFILING PhillipsDessouky::find_min_cut data transfer 2 time: %.10fs", @@ -635,8 +636,8 @@ def reformat_rust_flow_to_dict( depr_node_ids = rust_dag.get_dag_node_ids() depr_edges = rust_dag.get_dag_ek_processed_edges() - new_node_ids = ohjun_rust_runner.get_unbound_dag_node_ids() - new_edges = ohjun_rust_runner.get_unbound_dag_ek_processed_edges() + new_node_ids = ohjun_rust_runner.get_residual_graph_node_ids() + new_edges = ohjun_rust_runner.get_residual_graph_ek_processed_edges() # print(f"depr_node_ids: {depr_node_ids}") # print(f"new_node_ids: {new_node_ids}") diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index 39c2d78..dfde958 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -133,7 +133,7 @@ impl LowtimeGraph { self.preds.get(&node_id).map(|preds| preds.iter()) } - pub fn edges(&mut self) -> impl Iterator { + pub fn edges(&self) -> impl Iterator { self.edges.iter().flat_map(|(from, inner)| { inner.iter().map(move |(to, edge)| (from, to, edge)) }) @@ -154,8 +154,23 @@ impl LowtimeGraph { self.node_ids.push(node_id) } + pub fn has_edge(&self, from: u32, to: u32) -> bool { + match self.edges.get(&from).and_then(|inner| inner.get(&to)) { + Some(_) => true, + None => false, + } + } + pub fn get_edge(&self, from: u32, to: u32) -> &LowtimeEdge { - self.edges.get(&from).unwrap().get(&to).unwrap() + self.edges.get(&from) + .and_then(|inner| inner.get(&to)) + .expect(&format!("Edge {} to {} not found", from, to)) + } + + pub fn get_edge_mut(&mut self, from: u32, to: u32) -> &mut LowtimeEdge { + self.edges.get_mut(&from) + .and_then(|inner| inner.get_mut(&to)) + .expect(&format!("Edge {} to {} not found", from, to)) } pub fn add_edge(&mut self, from: u32, to: u32, edge: LowtimeEdge) -> () { @@ -164,11 +179,11 @@ impl LowtimeGraph { self.num_edges += 1; } - fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut LowtimeEdge> { - self.edges - .get_mut(&from) - .and_then(|to_edges| to_edges.get_mut(&to)) - } + // fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut LowtimeEdge> { + // self.edges + // .get_mut(&from) + // .and_then(|to_edges| to_edges.get_mut(&to)) + // } // TESTING(ohjun): should make private when testing functions are deleted pub fn get_ek_preprocessed_edges(&self, ) -> Vec>> { @@ -184,8 +199,8 @@ impl LowtimeGraph { // TESTING(ohjun) pub fn print_all_capacities(&self) -> () { let mut processed_edges = self.get_ek_preprocessed_edges(); - processed_edges.sort_by(|((a_from, a_to), a_cap): &((u32, u32), OrderedFloat), - ((b_from, b_to), b_cap): &((u32, u32), OrderedFloat)| { + processed_edges.sort_by(|((a_from, a_to), _a_cap): &((u32, u32), OrderedFloat), + ((b_from, b_to), _b_cap): &((u32, u32), OrderedFloat)| { // a_from < b_from || (a_from == b_from && a_to < b_to) let from_cmp = a_from.cmp(&b_from); if from_cmp == Ordering::Equal { @@ -255,6 +270,10 @@ impl LowtimeEdge { self.flow } + pub fn set_flow(&mut self, new_flow: OrderedFloat) -> () { + self.flow = new_flow; + } + pub fn get_ub(&self) -> OrderedFloat { self.ub } diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index 5035ce4..c36f8d9 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -1,6 +1,6 @@ use pyo3::prelude::*; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use ordered_float::OrderedFloat; use pathfinding::directed::edmonds_karp::{ @@ -10,7 +10,7 @@ use pathfinding::directed::edmonds_karp::{ }; use std::time::Instant; -use log::{info, debug, Level}; +use log::{info, debug, error, Level}; use crate::lowtime_graph::{LowtimeGraph, LowtimeEdge}; use crate::utils; @@ -19,13 +19,16 @@ use crate::utils; #[pyclass] pub struct PhillipsDessouky { dag: LowtimeGraph, + fp_error: f64, unbound_dag_temp: LowtimeGraph, // TESTING(ohjun) + residual_graph_temp: LowtimeGraph, // TESTING(ohjun) } #[pymethods] impl PhillipsDessouky { #[new] fn new( + fp_error: f64, node_ids: Vec, source_node_id: u32, sink_node_id: u32, @@ -38,7 +41,9 @@ impl PhillipsDessouky { sink_node_id, edges_raw, ), + fp_error, unbound_dag_temp: LowtimeGraph::new(), // TESTING(ohjun) + residual_graph_temp: LowtimeGraph::new(), // TESTING(ohjun) }) } @@ -70,6 +75,20 @@ impl PhillipsDessouky { py_edges } + // TESTING ohjun + fn get_residual_graph_node_ids(&self) -> Vec { + self.residual_graph_temp.get_node_ids().clone() + } + + // TESTING ohjun + fn get_residual_graph_ek_processed_edges(&self) -> Vec<((u32, u32), f64)> { + let rs_edges = self.residual_graph_temp.get_ek_preprocessed_edges(); + let py_edges: Vec<((u32, u32), f64)> = rs_edges.iter().map(|((from, to), cap)| { + ((*from, *to), cap.into_inner()) + }).collect(); + py_edges + } + // TESTING(ohjun) fn max_flow_depr(&self) -> Vec<((u32, u32), f64)> { info!("CALLING MAX FLOW FROM max_flow_depr"); @@ -155,9 +174,14 @@ impl PhillipsDessouky { unbound_dag.set_source_node_id(s_prime_id); unbound_dag.set_sink_node_id(t_prime_id); - // First Max Flow - info!("CALLING MAX FLOW FROM find_min_cut_wip"); // TESTING(ohjun) - let (flows, _, _) = unbound_dag.max_flow(); + + // We're done with constructing the DAG with only flow upper bounds. + // Find the maximum flow on this DAG. + let (flows, _max_flow, _min_cut): ( + Vec<((u32, u32), OrderedFloat)>, + OrderedFloat, + Vec<((u32, u32), OrderedFloat)>, + ) = unbound_dag.max_flow(); if log::log_enabled!(Level::Debug) { debug!("After first max flow"); @@ -166,7 +190,98 @@ impl PhillipsDessouky { debug!("Sum of all flow values: {}", total_flow); } - // TODO(ohjun): rewrite up to 2nd max flow + // Convert flows to dict for faster lookup + let flow_dict = flows.iter().fold( + HashMap::new(), + |mut acc: HashMap>>, ((from, to), flow)| { + acc.entry(*from) + .or_insert_with(HashMap::new) + .insert(*to, *flow); + acc + } + ); + + // Check if residual graph is saturated. If so, we have a feasible flow. + if let Some(succs) = unbound_dag.successors(s_prime_id) { + for u in succs { + let flow = flow_dict.get(&s_prime_id) + .and_then(|inner| inner.get(u)) + .unwrap_or(&OrderedFloat(0.0)); + let cap = unbound_dag.get_edge(s_prime_id, *u).get_capacity(); + let diff = (flow - cap).into_inner().abs(); + if diff > self.fp_error { + error!( + "s' -> {} unsaturated (flow: {}, capacity: {})", + u, + flow_dict[&s_prime_id][u], + unbound_dag.get_edge(s_prime_id, *u).get_capacity(), + ); + // TODO(ohjun): integrate with pyo3 exceptions + panic!("ERROR: Max flow on unbounded DAG didn't saturate."); + } + } + } + if let Some(preds) = unbound_dag.predecessors(t_prime_id) { + for u in preds { + let flow = flow_dict.get(u) + .and_then(|inner| inner.get(&t_prime_id)) + .unwrap_or(&OrderedFloat(0.0)); + let cap = unbound_dag.get_edge(*u, t_prime_id).get_capacity(); + let diff = (flow - cap).into_inner().abs(); + if diff > self.fp_error { + error!( + "{} -> t' unsaturated (flow: {}, capacity: {})", + u, + flow_dict[u][&t_prime_id], + unbound_dag.get_edge(*u, t_prime_id).get_capacity(), + ); + // TODO(ohjun): integrate with pyo3 exceptions + panic!("ERROR: Max flow on unbounded DAG didn't saturate."); + } + } + } + + // We have a feasible flow. Construct a new residual graph with the same + // shape as the capacity DAG so that we can find the min cut. + // First, retrieve the flow amounts to the original capacity graph, where for + // each edge u -> v, the flow amount is `flow + lb`. + for (u, v, edge) in self.dag.edges_mut() { + let flow = flow_dict.get(u) + .and_then(|inner| inner.get(v)) + .unwrap_or(&OrderedFloat(0.0)); + edge.set_flow(flow + edge.get_lb()); + } + + // Construct a new residual graph (same shape as capacity DAG) with + // u -> v capacity `ub - flow` and v -> u capacity `flow - lb`. + let mut residual_graph = self.dag.clone(); + for (u, v, _dag_edge) in self.dag.edges() { + // Rounding small negative values to 0.0 avoids pathfinding::edmonds_karp + // from entering unreachable code. Has no impact on correctness in test runs. + let residual_uv_edge = residual_graph.get_edge_mut(*u, *v); + let mut uv_capacity = residual_uv_edge.get_ub() - residual_uv_edge.get_flow(); + if uv_capacity.into_inner().abs() < self.fp_error { + uv_capacity = OrderedFloat(0.0); + } + residual_uv_edge.set_capacity(uv_capacity); + + let mut vu_capacity = residual_uv_edge.get_flow() - residual_uv_edge.get_lb(); + if vu_capacity.into_inner().abs() < self.fp_error { + vu_capacity = OrderedFloat(0.0); + } + + match self.dag.has_edge(*v, *u) { + true => residual_graph.get_edge_mut(*v, *u).set_capacity(vu_capacity), + false => residual_graph.add_edge(*v, *u, LowtimeEdge::new_only_capacity(vu_capacity)), + } + } + + // Run max flow on the new residual graph. + let (flows, _max_flow, _min_cut): ( + Vec<((u32, u32), OrderedFloat)>, + OrderedFloat, + Vec<((u32, u32), OrderedFloat)>, + ) = residual_graph.max_flow(); let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { ((*from, *to), flow.into_inner()) @@ -174,6 +289,7 @@ impl PhillipsDessouky { //// TESTING(ohjun) self.unbound_dag_temp = unbound_dag; + self.residual_graph_temp = residual_graph; flows_f64 } From b89e1c0ea6a4536d93555d59b8abaddbb6041765 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Thu, 21 Nov 2024 19:10:06 -0500 Subject: [PATCH 22/55] set up next goal state solver --- lowtime/solver.py | 74 ++-- lowtime/solver.up_to_second.py | 759 +++++++++++++++++++++++++++++++++ 2 files changed, 787 insertions(+), 46 deletions(-) create mode 100644 lowtime/solver.up_to_second.py diff --git a/lowtime/solver.py b/lowtime/solver.py index 0cece09..c2d3f6d 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -621,52 +621,6 @@ def reformat_rust_flow_to_dict( rust_flow_vec = rust_dag.max_flow_depr() flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) - ############# NEW ############# - ##### TESTING print all edges and capacities - # logger.info(f"s_prime_id: {s_prime_id}") - # logger.info(f"t_prime_id: {t_prime_id}") - # logger.info("Python Printing graph:") - # logger.info(f"Num edges: {len(unbound_dag.edges())}") - # for from_, to_, edge_attrs in unbound_dag.edges(data=True): - # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") - - # TEMP(ohjun): in current wip version, do everything until after 2nd max flow in Rust - ohjun_rust_flow_vec = ohjun_rust_runner.find_min_cut_wip() - ohjun_flow_dict = reformat_rust_flow_to_dict(ohjun_rust_flow_vec, residual_graph) - - depr_node_ids = rust_dag.get_dag_node_ids() - depr_edges = rust_dag.get_dag_ek_processed_edges() - new_node_ids = ohjun_rust_runner.get_residual_graph_node_ids() - new_edges = ohjun_rust_runner.get_residual_graph_ek_processed_edges() - - # print(f"depr_node_ids: {depr_node_ids}") - # print(f"new_node_ids: {new_node_ids}") - assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" - assert depr_node_ids == new_node_ids, "DIFF in node_ids" - assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" - # if sorted(depr_edges) != sorted(new_edges): - # for depr_edge, new_edge in zip(sorted(depr_edges), sorted(new_edges)): - # if depr_edge == new_edge: - # logger.info("edges EQUAL") - # logger.info(f"depr_edge: {depr_edge}") - # logger.info(f"new_edge : {new_edge}") - # else: - # logger.info("edges DIFFERENT") - # logger.info(f"depr_edge: {depr_edge}") - # logger.info(f"new_edge : {new_edge}") - assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" - - # def print_dict(d): - # for from_, inner in d.items(): - # for to_, flow in inner.items(): - # logger.info(f'{from_} -> {to_}: {flow}') - # logger.info('flow_dict:') - # print_dict(flow_dict) - # logger.info('ohjun_flow_dict:') - # print_dict(ohjun_flow_dict) - assert flow_dict == ohjun_flow_dict, "flow dicts were different :(" - ############# NEW ############# - profiling_max_flow = time.time() - profiling_max_flow logger.info( "PROFILING PhillipsDessouky::find_min_cut maximum_flow_2 time: %.10fs", @@ -722,6 +676,34 @@ def reformat_rust_flow_to_dict( profiling_min_cut_after_max_flows, ) + ############# NEW ############# + ##### TESTING print all edges and capacities + # logger.info(f"s_prime_id: {s_prime_id}") + # logger.info(f"t_prime_id: {t_prime_id}") + # logger.info("Python Printing graph:") + # logger.info(f"Num edges: {len(unbound_dag.edges())}") + # for from_, to_, edge_attrs in unbound_dag.edges(data=True): + # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") + + # TEMP(ohjun): in current wip version, do everything until after 2nd max flow in Rust + ohjun_s_set, ohjun_t_set = ohjun_rust_runner.find_min_cut_wip() + + depr_node_ids = list(new_residual.nodes) + depr_edges = list(new_residual.edges(data=False)) + new_node_ids = ohjun_rust_runner.get_new_residual_graph_node_ids() + new_edges = ohjun_rust_runner.get_new_residual_graph_ek_processed_edges() + + # print(f"depr_node_ids: {depr_node_ids}") + # print(f"new_node_ids: {new_node_ids}") + assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" + assert depr_node_ids == new_node_ids, "DIFF in node_ids" + assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" + assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" + + assert s_set == ohjun_s_set, "DIFF in s_set" + assert t_set == ohjun_t_set, "DIFF in t_set" + + ############# NEW ############# return s_set, t_set def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: diff --git a/lowtime/solver.up_to_second.py b/lowtime/solver.up_to_second.py new file mode 100644 index 0000000..0cece09 --- /dev/null +++ b/lowtime/solver.up_to_second.py @@ -0,0 +1,759 @@ +# Copyright (C) 2023 Jae-Won Chung +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" + + +from __future__ import annotations + +import time +import sys +import logging +from collections import deque +from collections.abc import Generator + +import networkx as nx +from attrs import define, field + +from lowtime.operation import Operation +from lowtime.graph_utils import ( + aon_dag_to_aoa_dag, + aoa_to_critical_dag, + get_critical_aoa_dag_total_time, + get_total_cost, +) +from lowtime.exceptions import LowtimeFlowError +from lowtime import _lowtime_rs + +FP_ERROR = 1e-6 + +logger = logging.getLogger(__name__) + + +@define +class IterationResult: + """Holds results after one PD iteration. + + Attributes: + iteration: The number of optimization iterations experienced by the DAG. + cost_change: The increase in cost from reducing the DAG's + quantized execution time by 1. + cost: The current total cost of the DAG. + quant_time: The current quantized total execution time of the DAG. + unit_time: The unit time used for time quantization. + real_time: The current real (de-quantized) total execution time of the DAG. + """ + + iteration: int + cost_change: float + cost: float + quant_time: int + unit_time: float + real_time: float = field(init=False) + + def __attrs_post_init__(self) -> None: + """Set `real_time` after initialization.""" + self.real_time = self.quant_time * self.unit_time + + +class PhillipsDessouky: + """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" + + def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: + """Initialize the Phillips-Dessouky solver. + + Assumptions: + - The graph is a Directed Acyclic Graph (DAG). + - Operations are annotated on nodes with name `attr_name`. + - There is only one source (entry) node. The source node is annotated as + `dag.graph["source_node"]`. + - There is only one sink (exit) node. The sink node is annotated as + `dag.graph["sink_node"]`. + - The `unit_time` attribute of every operation is the same. + + Args: + dag: A networkx DiGraph object that represents the computation DAG. + The aforementioned assumptions should hold for the DAG. + attr_name: The name of the attribute on nodes that holds the operation + object. Defaults to "op". + """ + self.attr_name = attr_name + + # Run checks on the DAG and cache some properties. + # Check: It's a DAG. + if not nx.is_directed_acyclic_graph(dag): + raise ValueError("The graph should be a Directed Acyclic Graph.") + + # Check: Only one source node that matches annotation. + if (source_node := dag.graph.get("source_node")) is None: + raise ValueError("The graph should have a `source_node` attribute.") + source_node_candidates = [] + for node_id, in_degree in dag.in_degree(): + if in_degree == 0: + source_node_candidates.append(node_id) + if len(source_node_candidates) == 0: + raise ValueError( + "Found zero nodes with in-degree 0. Cannot determine source node." + ) + if len(source_node_candidates) > 1: + raise ValueError( + f"Expecting only one source node, found {source_node_candidates}." + ) + if (detected_source_node := source_node_candidates[0]) != source_node: + raise ValueError( + f"Detected source node ({detected_source_node}) does not match " + f"the annotated source node ({source_node})." + ) + + # Check: Only one sink node that matches annotation. + if (sink_node := dag.graph.get("sink_node")) is None: + raise ValueError("The graph should have a `sink_node` attribute.") + sink_node_candidates = [] + for node_id, out_degree in dag.out_degree(): + if out_degree == 0: + sink_node_candidates.append(node_id) + if len(sink_node_candidates) == 0: + raise ValueError( + "Found zero nodes with out-degree 0. Cannot determine sink node." + ) + if len(sink_node_candidates) > 1: + raise ValueError( + f"Expecting only one sink node, found {sink_node_candidates}." + ) + if (detected_sink_node := sink_node_candidates[0]) != sink_node: + raise ValueError( + f"Detected sink node ({detected_sink_node}) does not match " + f"the annotated sink node ({sink_node})." + ) + + # Check: The `unit_time` attributes of every operation should be the same. + unit_time_candidates = set[float]() + for _, node_attr in dag.nodes(data=True): + if self.attr_name in node_attr: + op: Operation = node_attr[self.attr_name] + if op.is_dummy: + continue + unit_time_candidates.update( + option.unit_time for option in op.spec.options.options + ) + if len(unit_time_candidates) == 0: + raise ValueError( + "Found zero operations in the graph. Make sure you " + f"added operations as node attributes with key `{self.attr_name}`.", + ) + if len(unit_time_candidates) > 1: + raise ValueError( + f"Expecting the same `unit_time` across all operations, " + f"found {unit_time_candidates}." + ) + + self.aon_dag = dag + self.unit_time = unit_time_candidates.pop() + + def run(self) -> Generator[IterationResult, None, None]: + """Run the algorithm and yield a DAG after each iteration. + + The solver will not deepcopy operations on the DAG but rather in-place modify + them for speed. The caller should deepcopy the operations or the DAG if needed + before running the next iteration. + + Upon yield, it is guaranteed that the earliest/latest start/finish time values + of all operations are up to date w.r.t. the `duration` of each operation. + """ + logger.info("Starting Phillips-Dessouky solver.") + profiling_setup = time.time() + + # Convert the original activity-on-node DAG to activity-on-arc DAG form. + # AOA DAGs are purely internal. All public input and output of this class + # should be in AON form. + aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) + + # Estimate the minimum execution time of the DAG by setting every operation + # to run at its minimum duration. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.min_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + min_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected minimum quantized execution time: %d", min_time) + + # Estimated the maximum execution time of the DAG by setting every operation + # to run at its maximum duration. This is also our initial start point. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.max_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + max_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected maximum quantized execution time: %d", max_time) + + num_iters = max_time - min_time + 1 + logger.info("Expected number of PD iterations: %d", num_iters) + + profiling_setup = time.time() - profiling_setup + logger.info( + "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup + ) + + # Iteratively reduce the execution time of the DAG. + for iteration in range(sys.maxsize): + logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) + profiling_iter = time.time() + + # At this point, `critical_dag` always exists and is what we want. + # For the first iteration, the critical DAG is computed before the for + # loop in order to estimate the number of iterations. For subsequent + # iterations, the critcal DAG is computed after each iteration in + # in order to construct `IterationResult`. + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Critical DAG:") + logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) + logger.debug("Number of edges: %d", critical_dag.number_of_edges()) + non_dummy_ops = [ + attr[self.attr_name] + for _, _, attr in critical_dag.edges(data=True) + if not attr[self.attr_name].is_dummy + ] + logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) + logger.debug( + "Sum of non-dummy durations: %d", + sum(op.duration for op in non_dummy_ops), + ) + + profiling_annotate = time.time() + self.annotate_capacities(critical_dag) + profiling_annotate = time.time() - profiling_annotate + logger.info( + "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", + profiling_annotate, + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Capacity DAG:") + logger.debug( + "Total lb value: %f", + sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), + ) + logger.debug( + "Total ub value: %f", + sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), + ) + + try: + profiling_min_cut = time.time() + s_set, t_set = self.find_min_cut(critical_dag) + profiling_min_cut = time.time() - profiling_min_cut + logger.info( + "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", + profiling_min_cut, + ) + except LowtimeFlowError as e: + logger.info("Could not find minimum cut: %s", e.message) + logger.info("Terminating PD iteration.") + break + + profiling_reduce = time.time() + cost_change = self.reduce_durations(critical_dag, s_set, t_set) + profiling_reduce = time.time() - profiling_reduce + logger.info( + "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", + profiling_reduce, + ) + + if cost_change == float("inf") or abs(cost_change) < FP_ERROR: + logger.info("No further time reduction possible.") + logger.info("Terminating PD iteration.") + break + + # Earliest/latest start/finish times on operations also annotated here. + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + + # We directly modify operation attributes in the DAG, so after we + # ran one iteration, the AON DAG holds updated attributes. + result = IterationResult( + iteration=iteration + 1, + cost_change=cost_change, + cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), + quant_time=get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ), + unit_time=self.unit_time, + ) + logger.info("%s", result) + profiling_iter = time.time() - profiling_iter + logger.info( + "PROFILING PhillipsDessouky::run single iteration time: %.10fs", + profiling_iter, + ) + yield result + + def reduce_durations( + self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] + ) -> float: + """Modify operation durations to reduce the DAG's execution time by 1.""" + speed_up_edges: list[Operation] = [] + for node_id in s_set: + for child_id in list(dag.successors(node_id)): + if child_id in t_set: + op: Operation = dag[node_id][child_id][self.attr_name] + speed_up_edges.append(op) + + slow_down_edges: list[Operation] = [] + for node_id in t_set: + for child_id in list(dag.successors(node_id)): + if child_id in s_set: + op: Operation = dag[node_id][child_id][self.attr_name] + slow_down_edges.append(op) + + if not speed_up_edges: + logger.info("No speed up candidate operations.") + return 0.0 + + cost_change = 0.0 + + # Reduce the duration of edges (speed up) by quant_time 1. + for op in speed_up_edges: + if op.is_dummy: + logger.info("Cannot speed up dummy operation.") + return float("inf") + if op.duration - 1 < op.min_duration: + logger.info("Operation %s has reached the limit of speed up", op) + return float("inf") + cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) + op_before_str = str(op) + op.duration -= 1 + logger.info("Sped up %s to %s", op_before_str, op) + + # Increase the duration of edges (slow down) by quant_time 1. + for op in slow_down_edges: + # Dummy edges can always be slowed down. + if op.is_dummy: + logger.info("Slowed down DummyOperation (didn't really slowdown).") + continue + elif op.duration + 1 > op.max_duration: + logger.info("Operation %s has reached the limit of slow down", op) + return float("inf") + cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) + before_op_str = str(op) + op.duration += 1 + logger.info("Slowed down %s to %s", before_op_str, op) + + return cost_change + + def find_min_cut(self, dag: nx.DiGraph) -> tuple[set[int], set[int]]: + """Find the min cut of the DAG annotated with lower/upper bound flow capacities. + + Assumptions: + - The capacity DAG is in AOA form. + - The capacity DAG has been annotated with `lb` and `ub` attributes on edges, + representing the lower and upper bounds of the flow on the edge. + + Returns: + A tuple of (s_set, t_set) where s_set is the set of nodes on the source side + of the min cut and t_set is the set of nodes on the sink side of the min cut. + Returns None if no feasible flow exists. + + Raises: + LowtimeFlowError: When no feasible flow exists. + """ + # Helper function for Rust interop + def format_rust_inputs( + dag: nx.DiGraph, + ) -> tuple[ + nx.classes.reportviews.NodeView, + list[ + tuple[ + tuple[int, int], + tuple[float, float, float, float], + tuple[bool, int, int, int, int, int, int, int] | None, + ] + ], + ]: + nodes = dag.nodes + edges = [] + for from_, to_, edge_attrs in dag.edges(data=True): + op_details = ( + None + if self.attr_name not in edge_attrs + else ( + edge_attrs[self.attr_name].is_dummy, + edge_attrs[self.attr_name].duration, + edge_attrs[self.attr_name].max_duration, + edge_attrs[self.attr_name].min_duration, + edge_attrs[self.attr_name].earliest_start, + edge_attrs[self.attr_name].latest_start, + edge_attrs[self.attr_name].earliest_finish, + edge_attrs[self.attr_name].latest_finish, + ) + ) + edges.append( + ( + (from_, to_), + ( + edge_attrs.get("capacity", 0), + edge_attrs.get("flow", 0), + edge_attrs.get("ub", 0), + edge_attrs.get("lb", 0), + ), + op_details, + ) + ) + return nodes, edges + + ########### NEW ########### + # Construct Rust runner with input (critical) dag + ohjun_nodes, ohjun_edges = format_rust_inputs(dag) + # print(ohjun_nodes) + # print(ohjun_edges) + ohjun_rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, + ohjun_nodes, + dag.graph["source_node"], + dag.graph["sink_node"], + ohjun_edges + ) + ########### NEW ########### + + profiling_min_cut_setup = time.time() + source_node = dag.graph["source_node"] + sink_node = dag.graph["sink_node"] + + # In order to solve max flow on edges with both lower and upper bounds, + # we first need to convert it to another DAG that only has upper bounds. + unbound_dag = nx.DiGraph(dag) + + # For every edge, capacity = ub - lb. + for _, _, edge_attrs in unbound_dag.edges(data=True): + edge_attrs["capacity"] = edge_attrs["ub"] - edge_attrs["lb"] + + # Add a new node s', which will become the new source node. + # We constructed the AOA DAG, so we know that node IDs are integers. + node_ids: list[int] = list(unbound_dag.nodes) + s_prime_id = max(node_ids) + 1 + unbound_dag.add_node(s_prime_id) + + # For every node u in the original graph, add an edge (s', u) with capacity + # equal to the sum of all lower bounds of u's parents. + for u in dag.nodes: + capacity = 0.0 + for pred_id in dag.predecessors(u): + capacity += dag[pred_id][u]["lb"] + unbound_dag.add_edge(s_prime_id, u, capacity=capacity) + + # Add a new node t', which will become the new sink node. + t_prime_id = s_prime_id + 1 + unbound_dag.add_node(t_prime_id) + + # For every node u in the original graph, add an edge (u, t') with capacity + # equal to the sum of all lower bounds of u's children. + for u in dag.nodes: + capacity = 0.0 + for succ_id in dag.successors(u): + capacity += dag[u][succ_id]["lb"] + unbound_dag.add_edge(u, t_prime_id, capacity=capacity) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Unbound DAG") + logger.debug("Number of nodes: %d", unbound_dag.number_of_nodes()) + logger.debug("Number of edges: %d", unbound_dag.number_of_edges()) + logger.debug( + "Sum of capacities: %f", + sum(attr["capacity"] for _, _, attr in unbound_dag.edges(data=True)), + ) + + # Add an edge from t to s with infinite capacity. + unbound_dag.add_edge( + sink_node, + source_node, + capacity=float("inf"), + ) + + profiling_min_cut_setup = time.time() - profiling_min_cut_setup + logger.info( + "PROFILING PhillipsDessouky::find_min_cut setup time: %.10fs", + profiling_min_cut_setup, + ) + + # Helper function for Rust interop + # Rust's pathfinding::edmonds_karp does not return edges with 0 flow, + # but nx.max_flow does. So we fill in the 0s and empty nodes. + def reformat_rust_flow_to_dict( + flow_vec: list[tuple[tuple[int, int], float]], dag: nx.DiGraph + ) -> dict[int, dict[int, float]]: + flow_dict = dict() + for u in dag.nodes: + flow_dict[u] = dict() + for v in dag.successors(u): + flow_dict[u][v] = 0.0 + + for (u, v), cap in flow_vec: + flow_dict[u][v] = cap + + return flow_dict + + # We're done with constructing the DAG with only flow upper bounds. + # Find the maximum flow on this DAG. + profiling_max_flow = time.time() + nodes, edges = format_rust_inputs(unbound_dag) + + profiling_data_transfer = time.time() + rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, s_prime_id, t_prime_id, edges) + profiling_data_transfer = time.time() - profiling_data_transfer + logger.info( + "PROFILING PhillipsDessouky::find_min_cut data transfer time: %.10fs", + profiling_data_transfer, + ) + + # ohjun: FIRST MAX FLOW + rust_flow_vec = rust_dag.max_flow_depr() + flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) + + profiling_max_flow = time.time() - profiling_max_flow + logger.info( + "PROFILING PhillipsDessouky::find_min_cut maximum_flow_1 time: %.10fs", + profiling_max_flow, + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("After first max flow") + total_flow = 0.0 + for d in flow_dict.values(): + for flow in d.values(): + total_flow += flow + logger.debug("Sum of all flow values: %f", total_flow) + + profiling_min_cut_between_max_flows = time.time() + + # Check if residual graph is saturated. If so, we have a feasible flow. + for u in unbound_dag.successors(s_prime_id): + if ( + abs(flow_dict[s_prime_id][u] - unbound_dag[s_prime_id][u]["capacity"]) + > FP_ERROR + ): + logger.error( + "s' -> %s unsaturated (flow: %s, capacity: %s)", + u, + flow_dict[s_prime_id][u], + unbound_dag[s_prime_id][u]["capacity"], + ) + raise LowtimeFlowError( + "ERROR: Max flow on unbounded DAG didn't saturate." + ) + for u in unbound_dag.predecessors(t_prime_id): + if ( + abs(flow_dict[u][t_prime_id] - unbound_dag[u][t_prime_id]["capacity"]) + > FP_ERROR + ): + logger.error( + "%s -> t' unsaturated (flow: %s, capacity: %s)", + u, + flow_dict[u][t_prime_id], + unbound_dag[u][t_prime_id]["capacity"], + ) + raise LowtimeFlowError( + "ERROR: Max flow on unbounded DAG didn't saturate." + ) + + # We have a feasible flow. Construct a new residual graph with the same + # shape as the capacity DAG so that we can find the min cut. + # First, retrieve the flow amounts to the original capacity graph, where for + # each edge u -> v, the flow amount is `flow + lb`. + for u, v in dag.edges: + dag[u][v]["flow"] = flow_dict[u][v] + dag[u][v]["lb"] + + # Construct a new residual graph (same shape as capacity DAG) with + # u -> v capacity `ub - flow` and v -> u capacity `flow - lb`. + residual_graph = nx.DiGraph(dag) + for u, v in dag.edges: + # Rounding small negative values to 0.0 avoids Rust-side + # pathfinding::edmonds_karp from entering unreachable code. + # This edge case did not exist in Python-side nx.max_flow call. + uv_capacity = residual_graph[u][v]["ub"] - residual_graph[u][v]["flow"] + uv_capacity = 0.0 if abs(uv_capacity) < FP_ERROR else uv_capacity + residual_graph[u][v]["capacity"] = uv_capacity + + vu_capacity = residual_graph[u][v]["flow"] - residual_graph[u][v]["lb"] + vu_capacity = 0.0 if abs(vu_capacity) < FP_ERROR else vu_capacity + if dag.has_edge(v, u): + residual_graph[v][u]["capacity"] = vu_capacity + else: + residual_graph.add_edge(v, u, capacity=vu_capacity) + + profiling_min_cut_between_max_flows = ( + time.time() - profiling_min_cut_between_max_flows + ) + logger.info( + "PROFILING PhillipsDessouky::find_min_cut between max flows time: %.10fs", + profiling_min_cut_between_max_flows, + ) + + # Run max flow on the new residual graph. + profiling_max_flow = time.time() + nodes, edges = format_rust_inputs(residual_graph) + + profiling_data_transfer = time.time() + rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, source_node, sink_node, edges) + profiling_data_transfer = time.time() - profiling_data_transfer + logger.info( + "PROFILING PhillipsDessouky::find_min_cut data transfer 2 time: %.10fs", + profiling_data_transfer, + ) + + # ohjun: SECOND MAX FLOW + rust_flow_vec = rust_dag.max_flow_depr() + flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) + + ############# NEW ############# + ##### TESTING print all edges and capacities + # logger.info(f"s_prime_id: {s_prime_id}") + # logger.info(f"t_prime_id: {t_prime_id}") + # logger.info("Python Printing graph:") + # logger.info(f"Num edges: {len(unbound_dag.edges())}") + # for from_, to_, edge_attrs in unbound_dag.edges(data=True): + # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") + + # TEMP(ohjun): in current wip version, do everything until after 2nd max flow in Rust + ohjun_rust_flow_vec = ohjun_rust_runner.find_min_cut_wip() + ohjun_flow_dict = reformat_rust_flow_to_dict(ohjun_rust_flow_vec, residual_graph) + + depr_node_ids = rust_dag.get_dag_node_ids() + depr_edges = rust_dag.get_dag_ek_processed_edges() + new_node_ids = ohjun_rust_runner.get_residual_graph_node_ids() + new_edges = ohjun_rust_runner.get_residual_graph_ek_processed_edges() + + # print(f"depr_node_ids: {depr_node_ids}") + # print(f"new_node_ids: {new_node_ids}") + assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" + assert depr_node_ids == new_node_ids, "DIFF in node_ids" + assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" + # if sorted(depr_edges) != sorted(new_edges): + # for depr_edge, new_edge in zip(sorted(depr_edges), sorted(new_edges)): + # if depr_edge == new_edge: + # logger.info("edges EQUAL") + # logger.info(f"depr_edge: {depr_edge}") + # logger.info(f"new_edge : {new_edge}") + # else: + # logger.info("edges DIFFERENT") + # logger.info(f"depr_edge: {depr_edge}") + # logger.info(f"new_edge : {new_edge}") + assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" + + # def print_dict(d): + # for from_, inner in d.items(): + # for to_, flow in inner.items(): + # logger.info(f'{from_} -> {to_}: {flow}') + # logger.info('flow_dict:') + # print_dict(flow_dict) + # logger.info('ohjun_flow_dict:') + # print_dict(ohjun_flow_dict) + assert flow_dict == ohjun_flow_dict, "flow dicts were different :(" + ############# NEW ############# + + profiling_max_flow = time.time() - profiling_max_flow + logger.info( + "PROFILING PhillipsDessouky::find_min_cut maximum_flow_2 time: %.10fs", + profiling_max_flow, + ) + + profiling_min_cut_after_max_flows = time.time() + + # Add additional flow we get to the original graph + for u, v in dag.edges: + dag[u][v]["flow"] += flow_dict[u][v] + dag[u][v]["flow"] -= flow_dict[v][u] + + # Construct the new residual graph. + new_residual = nx.DiGraph(dag) + for u, v in dag.edges: + new_residual[u][v]["flow"] = dag[u][v]["ub"] - dag[u][v]["flow"] + new_residual.add_edge(v, u, flow=dag[u][v]["flow"] - dag[u][v]["lb"]) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("New residual graph") + logger.debug("Number of nodes: %d", new_residual.number_of_nodes()) + logger.debug("Number of edges: %d", new_residual.number_of_edges()) + logger.debug( + "Sum of flow: %f", + sum(attr["flow"] for _, _, attr in new_residual.edges(data=True)), + ) + + # Find the s-t cut induced by the second maximum flow above. + # Only following `flow > 0` edges, find the set of nodes reachable from + # source node. That's the s-set, and the rest is the t-set. + s_set = set[int]() + q: deque[int] = deque() + q.append(source_node) + while q: + cur_id = q.pop() + s_set.add(cur_id) + if cur_id == sink_node: + break + for child_id in list(new_residual.successors(cur_id)): + if ( + child_id not in s_set + and abs(new_residual[cur_id][child_id]["flow"]) > FP_ERROR + ): + q.append(child_id) + t_set = set(new_residual.nodes) - s_set + + profiling_min_cut_after_max_flows = ( + time.time() - profiling_min_cut_after_max_flows + ) + logger.info( + "PROFILING PhillipsDessouky::find_min_cut after max flows time: %.10fs", + profiling_min_cut_after_max_flows, + ) + + return s_set, t_set + + def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: + """In-place annotate the critical DAG with flow capacities.""" + # XXX(JW): Is this always large enough? + # It is necessary to monitor the `cost_change` value in `IterationResult` + # and make sure they are way smaller than this value. Even if the cost + # change is close or larger than this, users can scale down their cost + # value in `ExecutionOption`s. + inf = 10000.0 + for _, _, edge_attr in critical_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + duration = op.duration + # Dummy operations don't constrain the flow. + if op.is_dummy: # noqa: SIM114 + lb, ub = 0.0, inf + # Cannot be sped up or down. + elif duration == op.min_duration == op.max_duration: + lb, ub = 0.0, inf + # Cannot be sped up. + elif duration - 1 < op.min_duration: + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = inf + # Cannot be slowed down. + elif duration + 1 > op.max_duration: + lb = 0.0 + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + else: + # In case the cost model is almost linear, give this edge some room. + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR + + # XXX(JW): This roundiing may not be necessary. + edge_attr["lb"] = lb // FP_ERROR * FP_ERROR + edge_attr["ub"] = ub // FP_ERROR * FP_ERROR From d2ee220d7843d841a58227f42df5c6d3fdcf3254 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Thu, 21 Nov 2024 21:48:53 -0500 Subject: [PATCH 23/55] working: full find_min_cut rewrite --- ...al-state.py => solver.all-find-min-cut.py} | 302 ++++++++++++- lowtime/solver.py | 403 +++--------------- src/lowtime_graph.rs | 20 +- src/phillips_dessouky.rs | 107 ++++- 4 files changed, 464 insertions(+), 368 deletions(-) rename lowtime/{solver.goal-state.py => solver.all-find-min-cut.py} (60%) diff --git a/lowtime/solver.goal-state.py b/lowtime/solver.all-find-min-cut.py similarity index 60% rename from lowtime/solver.goal-state.py rename to lowtime/solver.all-find-min-cut.py index 304b3a4..2c0cd8c 100644 --- a/lowtime/solver.goal-state.py +++ b/lowtime/solver.all-find-min-cut.py @@ -408,7 +408,7 @@ def format_rust_inputs( ( (from_, to_), ( - edge_attrs.get("capacity", 0), # TODO(ohjun): suss + edge_attrs.get("capacity", 0), edge_attrs.get("flow", 0), edge_attrs.get("ub", 0), edge_attrs.get("lb", 0), @@ -418,16 +418,306 @@ def format_rust_inputs( ) return nodes, edges + ########### NEW ########### # Construct Rust runner with input (critical) dag - nodes, edges = format_rust_inputs(dag) - rust_runner = _lowtime_rs.PhillipsDessouky( - nodes, + ohjun_nodes, ohjun_edges = format_rust_inputs(dag) + # print(ohjun_nodes) + # print(ohjun_edges) + ohjun_rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, + ohjun_nodes, dag.graph["source_node"], dag.graph["sink_node"], - edges + ohjun_edges + ) + ########### NEW ########### + + profiling_min_cut_setup = time.time() + source_node = dag.graph["source_node"] + sink_node = dag.graph["sink_node"] + + # In order to solve max flow on edges with both lower and upper bounds, + # we first need to convert it to another DAG that only has upper bounds. + unbound_dag = nx.DiGraph(dag) + + # For every edge, capacity = ub - lb. + for _, _, edge_attrs in unbound_dag.edges(data=True): + edge_attrs["capacity"] = edge_attrs["ub"] - edge_attrs["lb"] + + # Add a new node s', which will become the new source node. + # We constructed the AOA DAG, so we know that node IDs are integers. + node_ids: list[int] = list(unbound_dag.nodes) + s_prime_id = max(node_ids) + 1 + unbound_dag.add_node(s_prime_id) + + # For every node u in the original graph, add an edge (s', u) with capacity + # equal to the sum of all lower bounds of u's parents. + for u in dag.nodes: + capacity = 0.0 + for pred_id in dag.predecessors(u): + capacity += dag[pred_id][u]["lb"] + unbound_dag.add_edge(s_prime_id, u, capacity=capacity) + + # Add a new node t', which will become the new sink node. + t_prime_id = s_prime_id + 1 + unbound_dag.add_node(t_prime_id) + + # For every node u in the original graph, add an edge (u, t') with capacity + # equal to the sum of all lower bounds of u's children. + for u in dag.nodes: + capacity = 0.0 + for succ_id in dag.successors(u): + capacity += dag[u][succ_id]["lb"] + unbound_dag.add_edge(u, t_prime_id, capacity=capacity) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Unbound DAG") + logger.debug("Number of nodes: %d", unbound_dag.number_of_nodes()) + logger.debug("Number of edges: %d", unbound_dag.number_of_edges()) + logger.debug( + "Sum of capacities: %f", + sum(attr["capacity"] for _, _, attr in unbound_dag.edges(data=True)), + ) + + # Add an edge from t to s with infinite capacity. + unbound_dag.add_edge( + sink_node, + source_node, + capacity=float("inf"), + ) + + profiling_min_cut_setup = time.time() - profiling_min_cut_setup + logger.info( + "PROFILING PhillipsDessouky::find_min_cut setup time: %.10fs", + profiling_min_cut_setup, + ) + + # Helper function for Rust interop + # Rust's pathfinding::edmonds_karp does not return edges with 0 flow, + # but nx.max_flow does. So we fill in the 0s and empty nodes. + def reformat_rust_flow_to_dict( + flow_vec: list[tuple[tuple[int, int], float]], dag: nx.DiGraph + ) -> dict[int, dict[int, float]]: + flow_dict = dict() + for u in dag.nodes: + flow_dict[u] = dict() + for v in dag.successors(u): + flow_dict[u][v] = 0.0 + + for (u, v), cap in flow_vec: + flow_dict[u][v] = cap + + return flow_dict + + # We're done with constructing the DAG with only flow upper bounds. + # Find the maximum flow on this DAG. + profiling_max_flow = time.time() + nodes, edges = format_rust_inputs(unbound_dag) + + profiling_data_transfer = time.time() + rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, s_prime_id, t_prime_id, edges) + profiling_data_transfer = time.time() - profiling_data_transfer + logger.info( + "PROFILING PhillipsDessouky::find_min_cut data transfer time: %.10fs", + profiling_data_transfer, + ) + + # ohjun: FIRST MAX FLOW + rust_flow_vec = rust_dag.max_flow_depr() + flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) + + profiling_max_flow = time.time() - profiling_max_flow + logger.info( + "PROFILING PhillipsDessouky::find_min_cut maximum_flow_1 time: %.10fs", + profiling_max_flow, + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("After first max flow") + total_flow = 0.0 + for d in flow_dict.values(): + for flow in d.values(): + total_flow += flow + logger.debug("Sum of all flow values: %f", total_flow) + + profiling_min_cut_between_max_flows = time.time() + + # Check if residual graph is saturated. If so, we have a feasible flow. + for u in unbound_dag.successors(s_prime_id): + if ( + abs(flow_dict[s_prime_id][u] - unbound_dag[s_prime_id][u]["capacity"]) + > FP_ERROR + ): + logger.error( + "s' -> %s unsaturated (flow: %s, capacity: %s)", + u, + flow_dict[s_prime_id][u], + unbound_dag[s_prime_id][u]["capacity"], + ) + raise LowtimeFlowError( + "ERROR: Max flow on unbounded DAG didn't saturate." + ) + for u in unbound_dag.predecessors(t_prime_id): + if ( + abs(flow_dict[u][t_prime_id] - unbound_dag[u][t_prime_id]["capacity"]) + > FP_ERROR + ): + logger.error( + "%s -> t' unsaturated (flow: %s, capacity: %s)", + u, + flow_dict[u][t_prime_id], + unbound_dag[u][t_prime_id]["capacity"], + ) + raise LowtimeFlowError( + "ERROR: Max flow on unbounded DAG didn't saturate." + ) + + # We have a feasible flow. Construct a new residual graph with the same + # shape as the capacity DAG so that we can find the min cut. + # First, retrieve the flow amounts to the original capacity graph, where for + # each edge u -> v, the flow amount is `flow + lb`. + for u, v in dag.edges: + dag[u][v]["flow"] = flow_dict[u][v] + dag[u][v]["lb"] + + # Construct a new residual graph (same shape as capacity DAG) with + # u -> v capacity `ub - flow` and v -> u capacity `flow - lb`. + residual_graph = nx.DiGraph(dag) + for u, v in dag.edges: + # Rounding small negative values to 0.0 avoids Rust-side + # pathfinding::edmonds_karp from entering unreachable code. + # This edge case did not exist in Python-side nx.max_flow call. + uv_capacity = residual_graph[u][v]["ub"] - residual_graph[u][v]["flow"] + uv_capacity = 0.0 if abs(uv_capacity) < FP_ERROR else uv_capacity + residual_graph[u][v]["capacity"] = uv_capacity + + vu_capacity = residual_graph[u][v]["flow"] - residual_graph[u][v]["lb"] + vu_capacity = 0.0 if abs(vu_capacity) < FP_ERROR else vu_capacity + if dag.has_edge(v, u): + residual_graph[v][u]["capacity"] = vu_capacity + else: + residual_graph.add_edge(v, u, capacity=vu_capacity) + + profiling_min_cut_between_max_flows = ( + time.time() - profiling_min_cut_between_max_flows + ) + logger.info( + "PROFILING PhillipsDessouky::find_min_cut between max flows time: %.10fs", + profiling_min_cut_between_max_flows, + ) + + # Run max flow on the new residual graph. + profiling_max_flow = time.time() + nodes, edges = format_rust_inputs(residual_graph) + + profiling_data_transfer = time.time() + rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, source_node, sink_node, edges) + profiling_data_transfer = time.time() - profiling_data_transfer + logger.info( + "PROFILING PhillipsDessouky::find_min_cut data transfer 2 time: %.10fs", + profiling_data_transfer, + ) + + # ohjun: SECOND MAX FLOW + rust_flow_vec = rust_dag.max_flow_depr() + flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) + + profiling_max_flow = time.time() - profiling_max_flow + logger.info( + "PROFILING PhillipsDessouky::find_min_cut maximum_flow_2 time: %.10fs", + profiling_max_flow, + ) + + profiling_min_cut_after_max_flows = time.time() + + # Add additional flow we get to the original graph + for u, v in dag.edges: + dag[u][v]["flow"] += flow_dict[u][v] + dag[u][v]["flow"] -= flow_dict[v][u] + + # Construct the new residual graph. + new_residual = nx.DiGraph(dag) + for u, v in dag.edges: + new_residual[u][v]["flow"] = dag[u][v]["ub"] - dag[u][v]["flow"] + new_residual.add_edge(v, u, flow=dag[u][v]["flow"] - dag[u][v]["lb"]) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("New residual graph") + logger.debug("Number of nodes: %d", new_residual.number_of_nodes()) + logger.debug("Number of edges: %d", new_residual.number_of_edges()) + logger.debug( + "Sum of flow: %f", + sum(attr["flow"] for _, _, attr in new_residual.edges(data=True)), + ) + + # Find the s-t cut induced by the second maximum flow above. + # Only following `flow > 0` edges, find the set of nodes reachable from + # source node. That's the s-set, and the rest is the t-set. + s_set = set[int]() + q: deque[int] = deque() + q.append(source_node) + while q: + cur_id = q.pop() + s_set.add(cur_id) + if cur_id == sink_node: + break + for child_id in list(new_residual.successors(cur_id)): + if ( + child_id not in s_set + and abs(new_residual[cur_id][child_id]["flow"]) > FP_ERROR + ): + q.append(child_id) + t_set = set(new_residual.nodes) - s_set + + profiling_min_cut_after_max_flows = ( + time.time() - profiling_min_cut_after_max_flows + ) + logger.info( + "PROFILING PhillipsDessouky::find_min_cut after max flows time: %.10fs", + profiling_min_cut_after_max_flows, ) - s_set, t_set = rust_runner.find_min_cut() + ############# NEW ############# + ##### TESTING print all edges and capacities + # logger.info(f"s_prime_id: {s_prime_id}") + # logger.info(f"t_prime_id: {t_prime_id}") + # logger.info("Python Printing graph:") + # logger.info(f"Num edges: {len(unbound_dag.edges())}") + # for from_, to_, edge_attrs in unbound_dag.edges(data=True): + # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") + + # TEMP(ohjun): in current wip version, do everything until after 2nd max flow in Rust + ohjun_s_set, ohjun_t_set = ohjun_rust_runner.find_min_cut() + + depr_node_ids = list(new_residual.nodes) + depr_edges = list(new_residual.edges(data=False)) + new_node_ids = ohjun_rust_runner.get_new_residual_node_ids() + new_edges = ohjun_rust_runner.get_new_residual_ek_processed_edges() + new_edges = [from_to for (from_to, cap) in new_edges] + + # print(f"depr_node_ids: {depr_node_ids}") + # print(f"new_node_ids: {new_node_ids}") + assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" + assert depr_node_ids == new_node_ids, "DIFF in node_ids" + assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" + # for depr_edge, new_edge in zip(sorted(depr_edges), sorted(new_edges)): + # if depr_edge == new_edge: + # print("same:") + # else: + # print("DIFF:") + # print(f" depr_edge: {depr_edge}") + # print(f" new_edge: {new_edge}") + assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" + + # print(f"s_set: {s_set}") + # print(f"t_set: {t_set}") + # print(f"ohjun_s_set: {ohjun_s_set}") + # print(f"ohjun_t_set: {ohjun_t_set}") + assert len(s_set) == len(ohjun_s_set), "LENGTH MISMATCH in s_set" + assert len(t_set) == len(ohjun_t_set), "LENGTH MISMATCH in t_set" + assert s_set == ohjun_s_set, "DIFF in s_set" + assert t_set == ohjun_t_set, "DIFF in t_set" + + ############# NEW ############# return s_set, t_set def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: diff --git a/lowtime/solver.py b/lowtime/solver.py index c2d3f6d..b5216a2 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -161,6 +161,51 @@ def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: self.aon_dag = dag self.unit_time = unit_time_candidates.pop() + # Helper function for Rust interop + def format_rust_inputs( + self, + dag: nx.DiGraph, + ) -> tuple[ + nx.classes.reportviews.NodeView, + list[ + tuple[ + tuple[int, int], + tuple[float, float, float, float], + tuple[bool, int, int, int, int, int, int, int] | None, + ] + ], + ]: + nodes = dag.nodes + edges = [] + for from_, to_, edge_attrs in dag.edges(data=True): + op_details = ( + None + if self.attr_name not in edge_attrs + else ( + edge_attrs[self.attr_name].is_dummy, + edge_attrs[self.attr_name].duration, + edge_attrs[self.attr_name].max_duration, + edge_attrs[self.attr_name].min_duration, + edge_attrs[self.attr_name].earliest_start, + edge_attrs[self.attr_name].latest_start, + edge_attrs[self.attr_name].earliest_finish, + edge_attrs[self.attr_name].latest_finish, + ) + ) + edges.append( + ( + (from_, to_), + ( + edge_attrs.get("capacity", 0), + edge_attrs.get("flow", 0), + edge_attrs.get("ub", 0), + edge_attrs.get("lb", 0), + ), + op_details, + ) + ) + return nodes, edges + def run(self) -> Generator[IterationResult, None, None]: """Run the algorithm and yield a DAG after each iteration. @@ -259,7 +304,15 @@ def run(self) -> Generator[IterationResult, None, None]: try: profiling_min_cut = time.time() - s_set, t_set = self.find_min_cut(critical_dag) + nodes, edges = self.format_rust_inputs(critical_dag) + rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, + nodes, + critical_dag.graph["source_node"], + critical_dag.graph["sink_node"], + edges + ) + s_set, t_set = rust_runner.find_min_cut() profiling_min_cut = time.time() - profiling_min_cut logger.info( "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", @@ -358,354 +411,6 @@ def reduce_durations( return cost_change - def find_min_cut(self, dag: nx.DiGraph) -> tuple[set[int], set[int]]: - """Find the min cut of the DAG annotated with lower/upper bound flow capacities. - - Assumptions: - - The capacity DAG is in AOA form. - - The capacity DAG has been annotated with `lb` and `ub` attributes on edges, - representing the lower and upper bounds of the flow on the edge. - - Returns: - A tuple of (s_set, t_set) where s_set is the set of nodes on the source side - of the min cut and t_set is the set of nodes on the sink side of the min cut. - Returns None if no feasible flow exists. - - Raises: - LowtimeFlowError: When no feasible flow exists. - """ - # Helper function for Rust interop - def format_rust_inputs( - dag: nx.DiGraph, - ) -> tuple[ - nx.classes.reportviews.NodeView, - list[ - tuple[ - tuple[int, int], - tuple[float, float, float, float], - tuple[bool, int, int, int, int, int, int, int] | None, - ] - ], - ]: - nodes = dag.nodes - edges = [] - for from_, to_, edge_attrs in dag.edges(data=True): - op_details = ( - None - if self.attr_name not in edge_attrs - else ( - edge_attrs[self.attr_name].is_dummy, - edge_attrs[self.attr_name].duration, - edge_attrs[self.attr_name].max_duration, - edge_attrs[self.attr_name].min_duration, - edge_attrs[self.attr_name].earliest_start, - edge_attrs[self.attr_name].latest_start, - edge_attrs[self.attr_name].earliest_finish, - edge_attrs[self.attr_name].latest_finish, - ) - ) - edges.append( - ( - (from_, to_), - ( - edge_attrs.get("capacity", 0), - edge_attrs.get("flow", 0), - edge_attrs.get("ub", 0), - edge_attrs.get("lb", 0), - ), - op_details, - ) - ) - return nodes, edges - - ########### NEW ########### - # Construct Rust runner with input (critical) dag - ohjun_nodes, ohjun_edges = format_rust_inputs(dag) - # print(ohjun_nodes) - # print(ohjun_edges) - ohjun_rust_runner = _lowtime_rs.PhillipsDessouky( - FP_ERROR, - ohjun_nodes, - dag.graph["source_node"], - dag.graph["sink_node"], - ohjun_edges - ) - ########### NEW ########### - - profiling_min_cut_setup = time.time() - source_node = dag.graph["source_node"] - sink_node = dag.graph["sink_node"] - - # In order to solve max flow on edges with both lower and upper bounds, - # we first need to convert it to another DAG that only has upper bounds. - unbound_dag = nx.DiGraph(dag) - - # For every edge, capacity = ub - lb. - for _, _, edge_attrs in unbound_dag.edges(data=True): - edge_attrs["capacity"] = edge_attrs["ub"] - edge_attrs["lb"] - - # Add a new node s', which will become the new source node. - # We constructed the AOA DAG, so we know that node IDs are integers. - node_ids: list[int] = list(unbound_dag.nodes) - s_prime_id = max(node_ids) + 1 - unbound_dag.add_node(s_prime_id) - - # For every node u in the original graph, add an edge (s', u) with capacity - # equal to the sum of all lower bounds of u's parents. - for u in dag.nodes: - capacity = 0.0 - for pred_id in dag.predecessors(u): - capacity += dag[pred_id][u]["lb"] - unbound_dag.add_edge(s_prime_id, u, capacity=capacity) - - # Add a new node t', which will become the new sink node. - t_prime_id = s_prime_id + 1 - unbound_dag.add_node(t_prime_id) - - # For every node u in the original graph, add an edge (u, t') with capacity - # equal to the sum of all lower bounds of u's children. - for u in dag.nodes: - capacity = 0.0 - for succ_id in dag.successors(u): - capacity += dag[u][succ_id]["lb"] - unbound_dag.add_edge(u, t_prime_id, capacity=capacity) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Unbound DAG") - logger.debug("Number of nodes: %d", unbound_dag.number_of_nodes()) - logger.debug("Number of edges: %d", unbound_dag.number_of_edges()) - logger.debug( - "Sum of capacities: %f", - sum(attr["capacity"] for _, _, attr in unbound_dag.edges(data=True)), - ) - - # Add an edge from t to s with infinite capacity. - unbound_dag.add_edge( - sink_node, - source_node, - capacity=float("inf"), - ) - - profiling_min_cut_setup = time.time() - profiling_min_cut_setup - logger.info( - "PROFILING PhillipsDessouky::find_min_cut setup time: %.10fs", - profiling_min_cut_setup, - ) - - # Helper function for Rust interop - # Rust's pathfinding::edmonds_karp does not return edges with 0 flow, - # but nx.max_flow does. So we fill in the 0s and empty nodes. - def reformat_rust_flow_to_dict( - flow_vec: list[tuple[tuple[int, int], float]], dag: nx.DiGraph - ) -> dict[int, dict[int, float]]: - flow_dict = dict() - for u in dag.nodes: - flow_dict[u] = dict() - for v in dag.successors(u): - flow_dict[u][v] = 0.0 - - for (u, v), cap in flow_vec: - flow_dict[u][v] = cap - - return flow_dict - - # We're done with constructing the DAG with only flow upper bounds. - # Find the maximum flow on this DAG. - profiling_max_flow = time.time() - nodes, edges = format_rust_inputs(unbound_dag) - - profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, s_prime_id, t_prime_id, edges) - profiling_data_transfer = time.time() - profiling_data_transfer - logger.info( - "PROFILING PhillipsDessouky::find_min_cut data transfer time: %.10fs", - profiling_data_transfer, - ) - - # ohjun: FIRST MAX FLOW - rust_flow_vec = rust_dag.max_flow_depr() - flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) - - profiling_max_flow = time.time() - profiling_max_flow - logger.info( - "PROFILING PhillipsDessouky::find_min_cut maximum_flow_1 time: %.10fs", - profiling_max_flow, - ) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("After first max flow") - total_flow = 0.0 - for d in flow_dict.values(): - for flow in d.values(): - total_flow += flow - logger.debug("Sum of all flow values: %f", total_flow) - - profiling_min_cut_between_max_flows = time.time() - - # Check if residual graph is saturated. If so, we have a feasible flow. - for u in unbound_dag.successors(s_prime_id): - if ( - abs(flow_dict[s_prime_id][u] - unbound_dag[s_prime_id][u]["capacity"]) - > FP_ERROR - ): - logger.error( - "s' -> %s unsaturated (flow: %s, capacity: %s)", - u, - flow_dict[s_prime_id][u], - unbound_dag[s_prime_id][u]["capacity"], - ) - raise LowtimeFlowError( - "ERROR: Max flow on unbounded DAG didn't saturate." - ) - for u in unbound_dag.predecessors(t_prime_id): - if ( - abs(flow_dict[u][t_prime_id] - unbound_dag[u][t_prime_id]["capacity"]) - > FP_ERROR - ): - logger.error( - "%s -> t' unsaturated (flow: %s, capacity: %s)", - u, - flow_dict[u][t_prime_id], - unbound_dag[u][t_prime_id]["capacity"], - ) - raise LowtimeFlowError( - "ERROR: Max flow on unbounded DAG didn't saturate." - ) - - # We have a feasible flow. Construct a new residual graph with the same - # shape as the capacity DAG so that we can find the min cut. - # First, retrieve the flow amounts to the original capacity graph, where for - # each edge u -> v, the flow amount is `flow + lb`. - for u, v in dag.edges: - dag[u][v]["flow"] = flow_dict[u][v] + dag[u][v]["lb"] - - # Construct a new residual graph (same shape as capacity DAG) with - # u -> v capacity `ub - flow` and v -> u capacity `flow - lb`. - residual_graph = nx.DiGraph(dag) - for u, v in dag.edges: - # Rounding small negative values to 0.0 avoids Rust-side - # pathfinding::edmonds_karp from entering unreachable code. - # This edge case did not exist in Python-side nx.max_flow call. - uv_capacity = residual_graph[u][v]["ub"] - residual_graph[u][v]["flow"] - uv_capacity = 0.0 if abs(uv_capacity) < FP_ERROR else uv_capacity - residual_graph[u][v]["capacity"] = uv_capacity - - vu_capacity = residual_graph[u][v]["flow"] - residual_graph[u][v]["lb"] - vu_capacity = 0.0 if abs(vu_capacity) < FP_ERROR else vu_capacity - if dag.has_edge(v, u): - residual_graph[v][u]["capacity"] = vu_capacity - else: - residual_graph.add_edge(v, u, capacity=vu_capacity) - - profiling_min_cut_between_max_flows = ( - time.time() - profiling_min_cut_between_max_flows - ) - logger.info( - "PROFILING PhillipsDessouky::find_min_cut between max flows time: %.10fs", - profiling_min_cut_between_max_flows, - ) - - # Run max flow on the new residual graph. - profiling_max_flow = time.time() - nodes, edges = format_rust_inputs(residual_graph) - - profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, source_node, sink_node, edges) - profiling_data_transfer = time.time() - profiling_data_transfer - logger.info( - "PROFILING PhillipsDessouky::find_min_cut data transfer 2 time: %.10fs", - profiling_data_transfer, - ) - - # ohjun: SECOND MAX FLOW - rust_flow_vec = rust_dag.max_flow_depr() - flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) - - profiling_max_flow = time.time() - profiling_max_flow - logger.info( - "PROFILING PhillipsDessouky::find_min_cut maximum_flow_2 time: %.10fs", - profiling_max_flow, - ) - - profiling_min_cut_after_max_flows = time.time() - - # Add additional flow we get to the original graph - for u, v in dag.edges: - dag[u][v]["flow"] += flow_dict[u][v] - dag[u][v]["flow"] -= flow_dict[v][u] - - # Construct the new residual graph. - new_residual = nx.DiGraph(dag) - for u, v in dag.edges: - new_residual[u][v]["flow"] = dag[u][v]["ub"] - dag[u][v]["flow"] - new_residual.add_edge(v, u, flow=dag[u][v]["flow"] - dag[u][v]["lb"]) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("New residual graph") - logger.debug("Number of nodes: %d", new_residual.number_of_nodes()) - logger.debug("Number of edges: %d", new_residual.number_of_edges()) - logger.debug( - "Sum of flow: %f", - sum(attr["flow"] for _, _, attr in new_residual.edges(data=True)), - ) - - # Find the s-t cut induced by the second maximum flow above. - # Only following `flow > 0` edges, find the set of nodes reachable from - # source node. That's the s-set, and the rest is the t-set. - s_set = set[int]() - q: deque[int] = deque() - q.append(source_node) - while q: - cur_id = q.pop() - s_set.add(cur_id) - if cur_id == sink_node: - break - for child_id in list(new_residual.successors(cur_id)): - if ( - child_id not in s_set - and abs(new_residual[cur_id][child_id]["flow"]) > FP_ERROR - ): - q.append(child_id) - t_set = set(new_residual.nodes) - s_set - - profiling_min_cut_after_max_flows = ( - time.time() - profiling_min_cut_after_max_flows - ) - logger.info( - "PROFILING PhillipsDessouky::find_min_cut after max flows time: %.10fs", - profiling_min_cut_after_max_flows, - ) - - ############# NEW ############# - ##### TESTING print all edges and capacities - # logger.info(f"s_prime_id: {s_prime_id}") - # logger.info(f"t_prime_id: {t_prime_id}") - # logger.info("Python Printing graph:") - # logger.info(f"Num edges: {len(unbound_dag.edges())}") - # for from_, to_, edge_attrs in unbound_dag.edges(data=True): - # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") - - # TEMP(ohjun): in current wip version, do everything until after 2nd max flow in Rust - ohjun_s_set, ohjun_t_set = ohjun_rust_runner.find_min_cut_wip() - - depr_node_ids = list(new_residual.nodes) - depr_edges = list(new_residual.edges(data=False)) - new_node_ids = ohjun_rust_runner.get_new_residual_graph_node_ids() - new_edges = ohjun_rust_runner.get_new_residual_graph_ek_processed_edges() - - # print(f"depr_node_ids: {depr_node_ids}") - # print(f"new_node_ids: {new_node_ids}") - assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" - assert depr_node_ids == new_node_ids, "DIFF in node_ids" - assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" - assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" - - assert s_set == ohjun_s_set, "DIFF in s_set" - assert t_set == ohjun_t_set, "DIFF in t_set" - - ############# NEW ############# - return s_set, t_set - def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: """In-place annotate the critical DAG with flow capacities.""" # XXX(JW): Is this always large enough? diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index dfde958..faa2450 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -244,6 +244,7 @@ impl LowtimeEdge { } } + // TODO(ohjun): there's probably a better way to do this with default args pub fn new_only_capacity(capacity: OrderedFloat) -> Self { LowtimeEdge { op: None, @@ -254,8 +255,15 @@ impl LowtimeEdge { } } - pub fn get_op(&self) -> Option<&Operation> { - self.op.as_ref() + // TODO(ohjun): there's probably a better way to do this with default args + pub fn new_only_flow(flow: OrderedFloat) -> Self { + LowtimeEdge { + op: None, + capacity: OrderedFloat(0.0), + flow, + ub: OrderedFloat(0.0), + lb: OrderedFloat(0.0), + } } pub fn get_capacity(&self) -> OrderedFloat { @@ -274,6 +282,14 @@ impl LowtimeEdge { self.flow = new_flow; } + pub fn incr_flow(&mut self, flow: OrderedFloat) -> () { + self.flow += flow; + } + + pub fn decr_flow(&mut self, flow: OrderedFloat) -> () { + self.flow -= flow; + } + pub fn get_ub(&self) -> OrderedFloat { self.ub } diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index c36f8d9..232f289 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -1,6 +1,6 @@ use pyo3::prelude::*; -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use ordered_float::OrderedFloat; use pathfinding::directed::edmonds_karp::{ @@ -22,6 +22,7 @@ pub struct PhillipsDessouky { fp_error: f64, unbound_dag_temp: LowtimeGraph, // TESTING(ohjun) residual_graph_temp: LowtimeGraph, // TESTING(ohjun) + new_residual_temp: LowtimeGraph, // TESTING(ohjun) } #[pymethods] @@ -44,6 +45,7 @@ impl PhillipsDessouky { fp_error, unbound_dag_temp: LowtimeGraph::new(), // TESTING(ohjun) residual_graph_temp: LowtimeGraph::new(), // TESTING(ohjun) + new_residual_temp: LowtimeGraph::new(), // TESTING(ohjun) }) } @@ -89,6 +91,20 @@ impl PhillipsDessouky { py_edges } + // TESTING ohjun + fn get_new_residual_node_ids(&self) -> Vec { + self.new_residual_temp.get_node_ids().clone() + } + + // TESTING ohjun + fn get_new_residual_ek_processed_edges(&self) -> Vec<((u32, u32), f64)> { + let rs_edges = self.new_residual_temp.get_ek_preprocessed_edges(); + let py_edges: Vec<((u32, u32), f64)> = rs_edges.iter().map(|((from, to), cap)| { + ((*from, *to), cap.into_inner()) + }).collect(); + py_edges + } + // TESTING(ohjun) fn max_flow_depr(&self) -> Vec<((u32, u32), f64)> { info!("CALLING MAX FLOW FROM max_flow_depr"); @@ -99,12 +115,22 @@ impl PhillipsDessouky { flows_f64 } - // TODO(ohjun): iteratively implement/verify in _wip until done, - // then replace with this function - // fn find_min_cut(&self) -> (HashSet, HashSet) { - // } - - fn find_min_cut_wip(&mut self) -> Vec<((u32, u32), f64)> { + /// Find the min cut of the DAG annotated with lower/upper bound flow capacities. + /// + /// Assumptions: + /// - The capacity DAG is in AOA form. + /// - The capacity DAG has been annotated with `lb` and `ub` attributes on edges, + /// representing the lower and upper bounds of the flow on the edge. + /// + /// Returns: + /// A tuple of (s_set, t_set) where s_set is the set of nodes on the source side + /// of the min cut and t_set is the set of nodes on the sink side of the min cut. + /// Returns None if no feasible flow exists. + /// + /// Raises: + /// LowtimeFlowError: When no feasible flow exists. + // TODO(ohjun): fix documentation comment above to match rust version + fn find_min_cut(&mut self) -> (HashSet, HashSet) { // In order to solve max flow on edges with both lower and upper bounds, // we first need to convert it to another DAG that only has upper bounds. let mut unbound_dag: LowtimeGraph = self.dag.clone(); @@ -283,15 +309,74 @@ impl PhillipsDessouky { Vec<((u32, u32), OrderedFloat)>, ) = residual_graph.max_flow(); - let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { - ((*from, *to), flow.into_inner()) - }).collect(); + // Convert flows to dict for faster lookup + let flow_dict = flows.iter().fold( + HashMap::new(), + |mut acc: HashMap>>, ((from, to), flow)| { + acc.entry(*from) + .or_insert_with(HashMap::new) + .insert(*to, *flow); + acc + } + ); + + // Add additional flow we get to the original graph + for (u, v, edge) in self.dag.edges_mut() { + edge.incr_flow(*flow_dict.get(u) + .and_then(|inner| inner.get(v)) + .unwrap_or(&OrderedFloat(0.0))); + edge.decr_flow(*flow_dict.get(v) + .and_then(|inner| inner.get(u)) + .unwrap_or(&OrderedFloat(0.0))); + } + + // Construct the new residual graph. + let mut new_residual = self.dag.clone(); + for (u, v, edge) in self.dag.edges() { + new_residual.get_edge_mut(*u, *v).set_flow(edge.get_ub() - edge.get_flow()); + new_residual.add_edge(*v, *u, + LowtimeEdge::new_only_flow(edge.get_flow() - edge.get_lb())); + } + + if log::log_enabled!(Level::Debug) { + debug!("New residual graph"); + debug!("Number of nodes: {}", new_residual.num_nodes()); + debug!("Number of edges: {}", new_residual.num_edges()); + let total_flow = unbound_dag.edges() + .fold(OrderedFloat(0.0), |acc, (_from, _to, edge)| acc + edge.get_flow()); + debug!("Sum of capacities: {}", total_flow); + } + + // Find the s-t cut induced by the second maximum flow above. + // Only following `flow > 0` edges, find the set of nodes reachable from + // source node. That's the s-set, and the rest is the t-set. + let mut s_set: HashSet = HashSet::new(); + let mut q: VecDeque = VecDeque::new(); + q.push_back(new_residual.get_source_node_id()); + while !q.is_empty() { + let cur_id = q.pop_back().unwrap(); + s_set.insert(cur_id); + if cur_id == new_residual.get_sink_node_id() { + break; + } + if let Some(succs) = new_residual.successors(cur_id) { + for child_id in succs { + let flow = new_residual.get_edge(cur_id, *child_id).get_flow().into_inner(); + if !s_set.contains(child_id) && flow.abs() > self.fp_error { + q.push_back(*child_id); + } + } + } + } + let all_nodes: HashSet = new_residual.get_node_ids().into_iter().copied().collect(); + let t_set: HashSet = all_nodes.difference(&s_set).copied().collect(); //// TESTING(ohjun) self.unbound_dag_temp = unbound_dag; self.residual_graph_temp = residual_graph; + self.new_residual_temp = new_residual; - flows_f64 + (s_set, t_set) } } From e9053a6479db7924fa059e496249ca93d5271712 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Thu, 21 Nov 2024 21:51:58 -0500 Subject: [PATCH 24/55] working: full f_m_c replace + remove testing functions --- src/phillips_dessouky.rs | 79 ---------------------------------------- 1 file changed, 79 deletions(-) diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index 232f289..e132bc9 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -20,9 +20,6 @@ use crate::utils; pub struct PhillipsDessouky { dag: LowtimeGraph, fp_error: f64, - unbound_dag_temp: LowtimeGraph, // TESTING(ohjun) - residual_graph_temp: LowtimeGraph, // TESTING(ohjun) - new_residual_temp: LowtimeGraph, // TESTING(ohjun) } #[pymethods] @@ -43,78 +40,9 @@ impl PhillipsDessouky { edges_raw, ), fp_error, - unbound_dag_temp: LowtimeGraph::new(), // TESTING(ohjun) - residual_graph_temp: LowtimeGraph::new(), // TESTING(ohjun) - new_residual_temp: LowtimeGraph::new(), // TESTING(ohjun) }) } - // TESTING ohjun - fn get_dag_node_ids(&self) -> Vec { - self.dag.get_node_ids().clone() - } - - // TESTING ohjun - fn get_dag_ek_processed_edges(&self) -> Vec<((u32, u32), f64)> { - let rs_edges = self.dag.get_ek_preprocessed_edges(); - let py_edges: Vec<((u32, u32), f64)> = rs_edges.iter().map(|((from, to), cap)| { - ((*from, *to), cap.into_inner()) - }).collect(); - py_edges - } - - // TESTING ohjun - fn get_unbound_dag_node_ids(&self) -> Vec { - self.unbound_dag_temp.get_node_ids().clone() - } - - // TESTING ohjun - fn get_unbound_dag_ek_processed_edges(&self) -> Vec<((u32, u32), f64)> { - let rs_edges = self.unbound_dag_temp.get_ek_preprocessed_edges(); - let py_edges: Vec<((u32, u32), f64)> = rs_edges.iter().map(|((from, to), cap)| { - ((*from, *to), cap.into_inner()) - }).collect(); - py_edges - } - - // TESTING ohjun - fn get_residual_graph_node_ids(&self) -> Vec { - self.residual_graph_temp.get_node_ids().clone() - } - - // TESTING ohjun - fn get_residual_graph_ek_processed_edges(&self) -> Vec<((u32, u32), f64)> { - let rs_edges = self.residual_graph_temp.get_ek_preprocessed_edges(); - let py_edges: Vec<((u32, u32), f64)> = rs_edges.iter().map(|((from, to), cap)| { - ((*from, *to), cap.into_inner()) - }).collect(); - py_edges - } - - // TESTING ohjun - fn get_new_residual_node_ids(&self) -> Vec { - self.new_residual_temp.get_node_ids().clone() - } - - // TESTING ohjun - fn get_new_residual_ek_processed_edges(&self) -> Vec<((u32, u32), f64)> { - let rs_edges = self.new_residual_temp.get_ek_preprocessed_edges(); - let py_edges: Vec<((u32, u32), f64)> = rs_edges.iter().map(|((from, to), cap)| { - ((*from, *to), cap.into_inner()) - }).collect(); - py_edges - } - - // TESTING(ohjun) - fn max_flow_depr(&self) -> Vec<((u32, u32), f64)> { - info!("CALLING MAX FLOW FROM max_flow_depr"); - let (flows, _, _) = self.dag.max_flow(); - let flows_f64: Vec<((u32, u32), f64)> = flows.iter().map(|((from, to), flow)| { - ((*from, *to), flow.into_inner()) - }).collect(); - flows_f64 - } - /// Find the min cut of the DAG annotated with lower/upper bound flow capacities. /// /// Assumptions: @@ -370,19 +298,12 @@ impl PhillipsDessouky { } let all_nodes: HashSet = new_residual.get_node_ids().into_iter().copied().collect(); let t_set: HashSet = all_nodes.difference(&s_set).copied().collect(); - - //// TESTING(ohjun) - self.unbound_dag_temp = unbound_dag; - self.residual_graph_temp = residual_graph; - self.new_residual_temp = new_residual; - (s_set, t_set) } } // not exposed to Python impl PhillipsDessouky { - // fn max_flow(&self) -> Vec<((u32, u32), f64)> { // self.graph.max_flow() // } From abef61a678af2f6ce1c47c37a198b5d936c95355 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 3 Dec 2024 17:20:58 -0500 Subject: [PATCH 25/55] working: full find_min_cut, rename solver checkpoints --- ...-cut.py => solver.find-min-cut.full-fn.py} | 0 lowtime/solver.find_min_cut.final.py | 446 ++++++++++++++++++ ....py => solver.find_min_cut.up_to_first.py} | 0 ...py => solver.find_min_cut.up_to_second.py} | 0 4 files changed, 446 insertions(+) rename lowtime/{solver.all-find-min-cut.py => solver.find-min-cut.full-fn.py} (100%) create mode 100644 lowtime/solver.find_min_cut.final.py rename lowtime/{solver.up_to_first.py => solver.find_min_cut.up_to_first.py} (100%) rename lowtime/{solver.up_to_second.py => solver.find_min_cut.up_to_second.py} (100%) diff --git a/lowtime/solver.all-find-min-cut.py b/lowtime/solver.find-min-cut.full-fn.py similarity index 100% rename from lowtime/solver.all-find-min-cut.py rename to lowtime/solver.find-min-cut.full-fn.py diff --git a/lowtime/solver.find_min_cut.final.py b/lowtime/solver.find_min_cut.final.py new file mode 100644 index 0000000..b5216a2 --- /dev/null +++ b/lowtime/solver.find_min_cut.final.py @@ -0,0 +1,446 @@ +# Copyright (C) 2023 Jae-Won Chung +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" + + +from __future__ import annotations + +import time +import sys +import logging +from collections import deque +from collections.abc import Generator + +import networkx as nx +from attrs import define, field + +from lowtime.operation import Operation +from lowtime.graph_utils import ( + aon_dag_to_aoa_dag, + aoa_to_critical_dag, + get_critical_aoa_dag_total_time, + get_total_cost, +) +from lowtime.exceptions import LowtimeFlowError +from lowtime import _lowtime_rs + +FP_ERROR = 1e-6 + +logger = logging.getLogger(__name__) + + +@define +class IterationResult: + """Holds results after one PD iteration. + + Attributes: + iteration: The number of optimization iterations experienced by the DAG. + cost_change: The increase in cost from reducing the DAG's + quantized execution time by 1. + cost: The current total cost of the DAG. + quant_time: The current quantized total execution time of the DAG. + unit_time: The unit time used for time quantization. + real_time: The current real (de-quantized) total execution time of the DAG. + """ + + iteration: int + cost_change: float + cost: float + quant_time: int + unit_time: float + real_time: float = field(init=False) + + def __attrs_post_init__(self) -> None: + """Set `real_time` after initialization.""" + self.real_time = self.quant_time * self.unit_time + + +class PhillipsDessouky: + """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" + + def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: + """Initialize the Phillips-Dessouky solver. + + Assumptions: + - The graph is a Directed Acyclic Graph (DAG). + - Operations are annotated on nodes with name `attr_name`. + - There is only one source (entry) node. The source node is annotated as + `dag.graph["source_node"]`. + - There is only one sink (exit) node. The sink node is annotated as + `dag.graph["sink_node"]`. + - The `unit_time` attribute of every operation is the same. + + Args: + dag: A networkx DiGraph object that represents the computation DAG. + The aforementioned assumptions should hold for the DAG. + attr_name: The name of the attribute on nodes that holds the operation + object. Defaults to "op". + """ + self.attr_name = attr_name + + # Run checks on the DAG and cache some properties. + # Check: It's a DAG. + if not nx.is_directed_acyclic_graph(dag): + raise ValueError("The graph should be a Directed Acyclic Graph.") + + # Check: Only one source node that matches annotation. + if (source_node := dag.graph.get("source_node")) is None: + raise ValueError("The graph should have a `source_node` attribute.") + source_node_candidates = [] + for node_id, in_degree in dag.in_degree(): + if in_degree == 0: + source_node_candidates.append(node_id) + if len(source_node_candidates) == 0: + raise ValueError( + "Found zero nodes with in-degree 0. Cannot determine source node." + ) + if len(source_node_candidates) > 1: + raise ValueError( + f"Expecting only one source node, found {source_node_candidates}." + ) + if (detected_source_node := source_node_candidates[0]) != source_node: + raise ValueError( + f"Detected source node ({detected_source_node}) does not match " + f"the annotated source node ({source_node})." + ) + + # Check: Only one sink node that matches annotation. + if (sink_node := dag.graph.get("sink_node")) is None: + raise ValueError("The graph should have a `sink_node` attribute.") + sink_node_candidates = [] + for node_id, out_degree in dag.out_degree(): + if out_degree == 0: + sink_node_candidates.append(node_id) + if len(sink_node_candidates) == 0: + raise ValueError( + "Found zero nodes with out-degree 0. Cannot determine sink node." + ) + if len(sink_node_candidates) > 1: + raise ValueError( + f"Expecting only one sink node, found {sink_node_candidates}." + ) + if (detected_sink_node := sink_node_candidates[0]) != sink_node: + raise ValueError( + f"Detected sink node ({detected_sink_node}) does not match " + f"the annotated sink node ({sink_node})." + ) + + # Check: The `unit_time` attributes of every operation should be the same. + unit_time_candidates = set[float]() + for _, node_attr in dag.nodes(data=True): + if self.attr_name in node_attr: + op: Operation = node_attr[self.attr_name] + if op.is_dummy: + continue + unit_time_candidates.update( + option.unit_time for option in op.spec.options.options + ) + if len(unit_time_candidates) == 0: + raise ValueError( + "Found zero operations in the graph. Make sure you " + f"added operations as node attributes with key `{self.attr_name}`.", + ) + if len(unit_time_candidates) > 1: + raise ValueError( + f"Expecting the same `unit_time` across all operations, " + f"found {unit_time_candidates}." + ) + + self.aon_dag = dag + self.unit_time = unit_time_candidates.pop() + + # Helper function for Rust interop + def format_rust_inputs( + self, + dag: nx.DiGraph, + ) -> tuple[ + nx.classes.reportviews.NodeView, + list[ + tuple[ + tuple[int, int], + tuple[float, float, float, float], + tuple[bool, int, int, int, int, int, int, int] | None, + ] + ], + ]: + nodes = dag.nodes + edges = [] + for from_, to_, edge_attrs in dag.edges(data=True): + op_details = ( + None + if self.attr_name not in edge_attrs + else ( + edge_attrs[self.attr_name].is_dummy, + edge_attrs[self.attr_name].duration, + edge_attrs[self.attr_name].max_duration, + edge_attrs[self.attr_name].min_duration, + edge_attrs[self.attr_name].earliest_start, + edge_attrs[self.attr_name].latest_start, + edge_attrs[self.attr_name].earliest_finish, + edge_attrs[self.attr_name].latest_finish, + ) + ) + edges.append( + ( + (from_, to_), + ( + edge_attrs.get("capacity", 0), + edge_attrs.get("flow", 0), + edge_attrs.get("ub", 0), + edge_attrs.get("lb", 0), + ), + op_details, + ) + ) + return nodes, edges + + def run(self) -> Generator[IterationResult, None, None]: + """Run the algorithm and yield a DAG after each iteration. + + The solver will not deepcopy operations on the DAG but rather in-place modify + them for speed. The caller should deepcopy the operations or the DAG if needed + before running the next iteration. + + Upon yield, it is guaranteed that the earliest/latest start/finish time values + of all operations are up to date w.r.t. the `duration` of each operation. + """ + logger.info("Starting Phillips-Dessouky solver.") + profiling_setup = time.time() + + # Convert the original activity-on-node DAG to activity-on-arc DAG form. + # AOA DAGs are purely internal. All public input and output of this class + # should be in AON form. + aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) + + # Estimate the minimum execution time of the DAG by setting every operation + # to run at its minimum duration. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.min_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + min_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected minimum quantized execution time: %d", min_time) + + # Estimated the maximum execution time of the DAG by setting every operation + # to run at its maximum duration. This is also our initial start point. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.max_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + max_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected maximum quantized execution time: %d", max_time) + + num_iters = max_time - min_time + 1 + logger.info("Expected number of PD iterations: %d", num_iters) + + profiling_setup = time.time() - profiling_setup + logger.info( + "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup + ) + + # Iteratively reduce the execution time of the DAG. + for iteration in range(sys.maxsize): + logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) + profiling_iter = time.time() + + # At this point, `critical_dag` always exists and is what we want. + # For the first iteration, the critical DAG is computed before the for + # loop in order to estimate the number of iterations. For subsequent + # iterations, the critcal DAG is computed after each iteration in + # in order to construct `IterationResult`. + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Critical DAG:") + logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) + logger.debug("Number of edges: %d", critical_dag.number_of_edges()) + non_dummy_ops = [ + attr[self.attr_name] + for _, _, attr in critical_dag.edges(data=True) + if not attr[self.attr_name].is_dummy + ] + logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) + logger.debug( + "Sum of non-dummy durations: %d", + sum(op.duration for op in non_dummy_ops), + ) + + profiling_annotate = time.time() + self.annotate_capacities(critical_dag) + profiling_annotate = time.time() - profiling_annotate + logger.info( + "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", + profiling_annotate, + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Capacity DAG:") + logger.debug( + "Total lb value: %f", + sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), + ) + logger.debug( + "Total ub value: %f", + sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), + ) + + try: + profiling_min_cut = time.time() + nodes, edges = self.format_rust_inputs(critical_dag) + rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, + nodes, + critical_dag.graph["source_node"], + critical_dag.graph["sink_node"], + edges + ) + s_set, t_set = rust_runner.find_min_cut() + profiling_min_cut = time.time() - profiling_min_cut + logger.info( + "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", + profiling_min_cut, + ) + except LowtimeFlowError as e: + logger.info("Could not find minimum cut: %s", e.message) + logger.info("Terminating PD iteration.") + break + + profiling_reduce = time.time() + cost_change = self.reduce_durations(critical_dag, s_set, t_set) + profiling_reduce = time.time() - profiling_reduce + logger.info( + "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", + profiling_reduce, + ) + + if cost_change == float("inf") or abs(cost_change) < FP_ERROR: + logger.info("No further time reduction possible.") + logger.info("Terminating PD iteration.") + break + + # Earliest/latest start/finish times on operations also annotated here. + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + + # We directly modify operation attributes in the DAG, so after we + # ran one iteration, the AON DAG holds updated attributes. + result = IterationResult( + iteration=iteration + 1, + cost_change=cost_change, + cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), + quant_time=get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ), + unit_time=self.unit_time, + ) + logger.info("%s", result) + profiling_iter = time.time() - profiling_iter + logger.info( + "PROFILING PhillipsDessouky::run single iteration time: %.10fs", + profiling_iter, + ) + yield result + + def reduce_durations( + self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] + ) -> float: + """Modify operation durations to reduce the DAG's execution time by 1.""" + speed_up_edges: list[Operation] = [] + for node_id in s_set: + for child_id in list(dag.successors(node_id)): + if child_id in t_set: + op: Operation = dag[node_id][child_id][self.attr_name] + speed_up_edges.append(op) + + slow_down_edges: list[Operation] = [] + for node_id in t_set: + for child_id in list(dag.successors(node_id)): + if child_id in s_set: + op: Operation = dag[node_id][child_id][self.attr_name] + slow_down_edges.append(op) + + if not speed_up_edges: + logger.info("No speed up candidate operations.") + return 0.0 + + cost_change = 0.0 + + # Reduce the duration of edges (speed up) by quant_time 1. + for op in speed_up_edges: + if op.is_dummy: + logger.info("Cannot speed up dummy operation.") + return float("inf") + if op.duration - 1 < op.min_duration: + logger.info("Operation %s has reached the limit of speed up", op) + return float("inf") + cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) + op_before_str = str(op) + op.duration -= 1 + logger.info("Sped up %s to %s", op_before_str, op) + + # Increase the duration of edges (slow down) by quant_time 1. + for op in slow_down_edges: + # Dummy edges can always be slowed down. + if op.is_dummy: + logger.info("Slowed down DummyOperation (didn't really slowdown).") + continue + elif op.duration + 1 > op.max_duration: + logger.info("Operation %s has reached the limit of slow down", op) + return float("inf") + cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) + before_op_str = str(op) + op.duration += 1 + logger.info("Slowed down %s to %s", before_op_str, op) + + return cost_change + + def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: + """In-place annotate the critical DAG with flow capacities.""" + # XXX(JW): Is this always large enough? + # It is necessary to monitor the `cost_change` value in `IterationResult` + # and make sure they are way smaller than this value. Even if the cost + # change is close or larger than this, users can scale down their cost + # value in `ExecutionOption`s. + inf = 10000.0 + for _, _, edge_attr in critical_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + duration = op.duration + # Dummy operations don't constrain the flow. + if op.is_dummy: # noqa: SIM114 + lb, ub = 0.0, inf + # Cannot be sped up or down. + elif duration == op.min_duration == op.max_duration: + lb, ub = 0.0, inf + # Cannot be sped up. + elif duration - 1 < op.min_duration: + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = inf + # Cannot be slowed down. + elif duration + 1 > op.max_duration: + lb = 0.0 + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + else: + # In case the cost model is almost linear, give this edge some room. + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR + + # XXX(JW): This roundiing may not be necessary. + edge_attr["lb"] = lb // FP_ERROR * FP_ERROR + edge_attr["ub"] = ub // FP_ERROR * FP_ERROR diff --git a/lowtime/solver.up_to_first.py b/lowtime/solver.find_min_cut.up_to_first.py similarity index 100% rename from lowtime/solver.up_to_first.py rename to lowtime/solver.find_min_cut.up_to_first.py diff --git a/lowtime/solver.up_to_second.py b/lowtime/solver.find_min_cut.up_to_second.py similarity index 100% rename from lowtime/solver.up_to_second.py rename to lowtime/solver.find_min_cut.up_to_second.py From da8a3e7e419ad80265fb503eb8e9231a463c727b Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 3 Dec 2024 17:33:10 -0500 Subject: [PATCH 26/55] set up next reduce_duration goal state --- ...fn.py => solver.1.find-min-cut.full-fn.py} | 0 ...inal.py => solver.1.find_min_cut.final.py} | 0 ...y => solver.1.find_min_cut.up_to_first.py} | 0 ... => solver.1.find_min_cut.up_to_second.py} | 0 lowtime/solver.2.reduce_duration.goal.py | 451 ++++++++++++++++++ lowtime/solver.py | 25 +- 6 files changed, 466 insertions(+), 10 deletions(-) rename lowtime/{solver.find-min-cut.full-fn.py => solver.1.find-min-cut.full-fn.py} (100%) rename lowtime/{solver.find_min_cut.final.py => solver.1.find_min_cut.final.py} (100%) rename lowtime/{solver.find_min_cut.up_to_first.py => solver.1.find_min_cut.up_to_first.py} (100%) rename lowtime/{solver.find_min_cut.up_to_second.py => solver.1.find_min_cut.up_to_second.py} (100%) create mode 100644 lowtime/solver.2.reduce_duration.goal.py diff --git a/lowtime/solver.find-min-cut.full-fn.py b/lowtime/solver.1.find-min-cut.full-fn.py similarity index 100% rename from lowtime/solver.find-min-cut.full-fn.py rename to lowtime/solver.1.find-min-cut.full-fn.py diff --git a/lowtime/solver.find_min_cut.final.py b/lowtime/solver.1.find_min_cut.final.py similarity index 100% rename from lowtime/solver.find_min_cut.final.py rename to lowtime/solver.1.find_min_cut.final.py diff --git a/lowtime/solver.find_min_cut.up_to_first.py b/lowtime/solver.1.find_min_cut.up_to_first.py similarity index 100% rename from lowtime/solver.find_min_cut.up_to_first.py rename to lowtime/solver.1.find_min_cut.up_to_first.py diff --git a/lowtime/solver.find_min_cut.up_to_second.py b/lowtime/solver.1.find_min_cut.up_to_second.py similarity index 100% rename from lowtime/solver.find_min_cut.up_to_second.py rename to lowtime/solver.1.find_min_cut.up_to_second.py diff --git a/lowtime/solver.2.reduce_duration.goal.py b/lowtime/solver.2.reduce_duration.goal.py new file mode 100644 index 0000000..6cf9dfc --- /dev/null +++ b/lowtime/solver.2.reduce_duration.goal.py @@ -0,0 +1,451 @@ +# Copyright (C) 2023 Jae-Won Chung +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" + + +from __future__ import annotations + +import time +import sys +import logging +from collections import deque +from collections.abc import Generator + +import networkx as nx +from attrs import define, field + +from lowtime.operation import Operation +from lowtime.graph_utils import ( + aon_dag_to_aoa_dag, + aoa_to_critical_dag, + get_critical_aoa_dag_total_time, + get_total_cost, +) +from lowtime.exceptions import LowtimeFlowError +from lowtime import _lowtime_rs + +FP_ERROR = 1e-6 + +logger = logging.getLogger(__name__) + + +@define +class IterationResult: + """Holds results after one PD iteration. + + Attributes: + iteration: The number of optimization iterations experienced by the DAG. + cost_change: The increase in cost from reducing the DAG's + quantized execution time by 1. + cost: The current total cost of the DAG. + quant_time: The current quantized total execution time of the DAG. + unit_time: The unit time used for time quantization. + real_time: The current real (de-quantized) total execution time of the DAG. + """ + + iteration: int + cost_change: float + cost: float + quant_time: int + unit_time: float + real_time: float = field(init=False) + + def __attrs_post_init__(self) -> None: + """Set `real_time` after initialization.""" + self.real_time = self.quant_time * self.unit_time + + +class PhillipsDessouky: + """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" + + def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: + """Initialize the Phillips-Dessouky solver. + + Assumptions: + - The graph is a Directed Acyclic Graph (DAG). + - Operations are annotated on nodes with name `attr_name`. + - There is only one source (entry) node. The source node is annotated as + `dag.graph["source_node"]`. + - There is only one sink (exit) node. The sink node is annotated as + `dag.graph["sink_node"]`. + - The `unit_time` attribute of every operation is the same. + + Args: + dag: A networkx DiGraph object that represents the computation DAG. + The aforementioned assumptions should hold for the DAG. + attr_name: The name of the attribute on nodes that holds the operation + object. Defaults to "op". + """ + self.attr_name = attr_name + # Set up Rust runner, will be initialized once critical DAG is formed + self.rust_runner = None + + # Run checks on the DAG and cache some properties. + # Check: It's a DAG. + if not nx.is_directed_acyclic_graph(dag): + raise ValueError("The graph should be a Directed Acyclic Graph.") + + # Check: Only one source node that matches annotation. + if (source_node := dag.graph.get("source_node")) is None: + raise ValueError("The graph should have a `source_node` attribute.") + source_node_candidates = [] + for node_id, in_degree in dag.in_degree(): + if in_degree == 0: + source_node_candidates.append(node_id) + if len(source_node_candidates) == 0: + raise ValueError( + "Found zero nodes with in-degree 0. Cannot determine source node." + ) + if len(source_node_candidates) > 1: + raise ValueError( + f"Expecting only one source node, found {source_node_candidates}." + ) + if (detected_source_node := source_node_candidates[0]) != source_node: + raise ValueError( + f"Detected source node ({detected_source_node}) does not match " + f"the annotated source node ({source_node})." + ) + + # Check: Only one sink node that matches annotation. + if (sink_node := dag.graph.get("sink_node")) is None: + raise ValueError("The graph should have a `sink_node` attribute.") + sink_node_candidates = [] + for node_id, out_degree in dag.out_degree(): + if out_degree == 0: + sink_node_candidates.append(node_id) + if len(sink_node_candidates) == 0: + raise ValueError( + "Found zero nodes with out-degree 0. Cannot determine sink node." + ) + if len(sink_node_candidates) > 1: + raise ValueError( + f"Expecting only one sink node, found {sink_node_candidates}." + ) + if (detected_sink_node := sink_node_candidates[0]) != sink_node: + raise ValueError( + f"Detected sink node ({detected_sink_node}) does not match " + f"the annotated sink node ({sink_node})." + ) + + # Check: The `unit_time` attributes of every operation should be the same. + unit_time_candidates = set[float]() + for _, node_attr in dag.nodes(data=True): + if self.attr_name in node_attr: + op: Operation = node_attr[self.attr_name] + if op.is_dummy: + continue + unit_time_candidates.update( + option.unit_time for option in op.spec.options.options + ) + if len(unit_time_candidates) == 0: + raise ValueError( + "Found zero operations in the graph. Make sure you " + f"added operations as node attributes with key `{self.attr_name}`.", + ) + if len(unit_time_candidates) > 1: + raise ValueError( + f"Expecting the same `unit_time` across all operations, " + f"found {unit_time_candidates}." + ) + + self.aon_dag = dag + self.unit_time = unit_time_candidates.pop() + + # Helper function for Rust interop + def format_rust_inputs( + self, + dag: nx.DiGraph, + ) -> tuple[ + nx.classes.reportviews.NodeView, + list[ + tuple[ + tuple[int, int], + tuple[float, float, float, float], + tuple[bool, int, int, int, int, int, int, int] | None, + ] + ], + ]: + nodes = dag.nodes + edges = [] + for from_, to_, edge_attrs in dag.edges(data=True): + op_details = ( + None + if self.attr_name not in edge_attrs + else ( + edge_attrs[self.attr_name].is_dummy, + edge_attrs[self.attr_name].duration, + edge_attrs[self.attr_name].max_duration, + edge_attrs[self.attr_name].min_duration, + edge_attrs[self.attr_name].earliest_start, + edge_attrs[self.attr_name].latest_start, + edge_attrs[self.attr_name].earliest_finish, + edge_attrs[self.attr_name].latest_finish, + ) + ) + edges.append( + ( + (from_, to_), + ( + edge_attrs.get("capacity", 0), + edge_attrs.get("flow", 0), + edge_attrs.get("ub", 0), + edge_attrs.get("lb", 0), + ), + op_details, + ) + ) + return nodes, edges + + def run(self) -> Generator[IterationResult, None, None]: + """Run the algorithm and yield a DAG after each iteration. + + The solver will not deepcopy operations on the DAG but rather in-place modify + them for speed. The caller should deepcopy the operations or the DAG if needed + before running the next iteration. + + Upon yield, it is guaranteed that the earliest/latest start/finish time values + of all operations are up to date w.r.t. the `duration` of each operation. + """ + logger.info("Starting Phillips-Dessouky solver.") + profiling_setup = time.time() + + # Convert the original activity-on-node DAG to activity-on-arc DAG form. + # AOA DAGs are purely internal. All public input and output of this class + # should be in AON form. + aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) + + # Estimate the minimum execution time of the DAG by setting every operation + # to run at its minimum duration. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.min_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + min_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected minimum quantized execution time: %d", min_time) + + # Estimated the maximum execution time of the DAG by setting every operation + # to run at its maximum duration. This is also our initial start point. + for _, _, edge_attr in aoa_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + if op.is_dummy: + continue + op.duration = op.max_duration + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + max_time = get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ) + logger.info("Expected maximum quantized execution time: %d", max_time) + + num_iters = max_time - min_time + 1 + logger.info("Expected number of PD iterations: %d", num_iters) + + profiling_setup = time.time() - profiling_setup + logger.info( + "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup + ) + + # Iteratively reduce the execution time of the DAG. + for iteration in range(sys.maxsize): + logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) + profiling_iter = time.time() + + # At this point, `critical_dag` always exists and is what we want. + # For the first iteration, the critical DAG is computed before the for + # loop in order to estimate the number of iterations. For subsequent + # iterations, the critcal DAG is computed after each iteration in + # in order to construct `IterationResult`. + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Critical DAG:") + logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) + logger.debug("Number of edges: %d", critical_dag.number_of_edges()) + non_dummy_ops = [ + attr[self.attr_name] + for _, _, attr in critical_dag.edges(data=True) + if not attr[self.attr_name].is_dummy + ] + logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) + logger.debug( + "Sum of non-dummy durations: %d", + sum(op.duration for op in non_dummy_ops), + ) + + profiling_annotate = time.time() + self.annotate_capacities(critical_dag) + profiling_annotate = time.time() - profiling_annotate + logger.info( + "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", + profiling_annotate, + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Capacity DAG:") + logger.debug( + "Total lb value: %f", + sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), + ) + logger.debug( + "Total ub value: %f", + sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), + ) + + # Preprocesses graph for Rust runner + nodes, edges = self.format_rust_inputs(critical_dag) + self.rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, + nodes, + critical_dag.graph["source_node"], + critical_dag.graph["sink_node"], + edges + ) + + try: + profiling_min_cut = time.time() + s_set, t_set = self.rust_runner.find_min_cut() + profiling_min_cut = time.time() - profiling_min_cut + logger.info( + "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", + profiling_min_cut, + ) + except LowtimeFlowError as e: + logger.info("Could not find minimum cut: %s", e.message) + logger.info("Terminating PD iteration.") + break + + profiling_reduce = time.time() + # cost_change = self.reduce_durations(critical_dag, s_set, t_set) + cost_change = self.rust_runner.reduce_durations() + profiling_reduce = time.time() - profiling_reduce + logger.info( + "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", + profiling_reduce, + ) + + if cost_change == float("inf") or abs(cost_change) < FP_ERROR: + logger.info("No further time reduction possible.") + logger.info("Terminating PD iteration.") + break + + # Earliest/latest start/finish times on operations also annotated here. + critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + + # We directly modify operation attributes in the DAG, so after we + # ran one iteration, the AON DAG holds updated attributes. + result = IterationResult( + iteration=iteration + 1, + cost_change=cost_change, + cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), + quant_time=get_critical_aoa_dag_total_time( + critical_dag, attr_name=self.attr_name + ), + unit_time=self.unit_time, + ) + logger.info("%s", result) + profiling_iter = time.time() - profiling_iter + logger.info( + "PROFILING PhillipsDessouky::run single iteration time: %.10fs", + profiling_iter, + ) + yield result + + def reduce_durations( + self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] + ) -> float: + """Modify operation durations to reduce the DAG's execution time by 1.""" + speed_up_edges: list[Operation] = [] + for node_id in s_set: + for child_id in list(dag.successors(node_id)): + if child_id in t_set: + op: Operation = dag[node_id][child_id][self.attr_name] + speed_up_edges.append(op) + + slow_down_edges: list[Operation] = [] + for node_id in t_set: + for child_id in list(dag.successors(node_id)): + if child_id in s_set: + op: Operation = dag[node_id][child_id][self.attr_name] + slow_down_edges.append(op) + + if not speed_up_edges: + logger.info("No speed up candidate operations.") + return 0.0 + + cost_change = 0.0 + + # Reduce the duration of edges (speed up) by quant_time 1. + for op in speed_up_edges: + if op.is_dummy: + logger.info("Cannot speed up dummy operation.") + return float("inf") + if op.duration - 1 < op.min_duration: + logger.info("Operation %s has reached the limit of speed up", op) + return float("inf") + cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) + op_before_str = str(op) + op.duration -= 1 + logger.info("Sped up %s to %s", op_before_str, op) + + # Increase the duration of edges (slow down) by quant_time 1. + for op in slow_down_edges: + # Dummy edges can always be slowed down. + if op.is_dummy: + logger.info("Slowed down DummyOperation (didn't really slowdown).") + continue + elif op.duration + 1 > op.max_duration: + logger.info("Operation %s has reached the limit of slow down", op) + return float("inf") + cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) + before_op_str = str(op) + op.duration += 1 + logger.info("Slowed down %s to %s", before_op_str, op) + + return cost_change + + def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: + """In-place annotate the critical DAG with flow capacities.""" + # XXX(JW): Is this always large enough? + # It is necessary to monitor the `cost_change` value in `IterationResult` + # and make sure they are way smaller than this value. Even if the cost + # change is close or larger than this, users can scale down their cost + # value in `ExecutionOption`s. + inf = 10000.0 + for _, _, edge_attr in critical_dag.edges(data=True): + op: Operation = edge_attr[self.attr_name] + duration = op.duration + # Dummy operations don't constrain the flow. + if op.is_dummy: # noqa: SIM114 + lb, ub = 0.0, inf + # Cannot be sped up or down. + elif duration == op.min_duration == op.max_duration: + lb, ub = 0.0, inf + # Cannot be sped up. + elif duration - 1 < op.min_duration: + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = inf + # Cannot be slowed down. + elif duration + 1 > op.max_duration: + lb = 0.0 + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + else: + # In case the cost model is almost linear, give this edge some room. + lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) + ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR + + # XXX(JW): This roundiing may not be necessary. + edge_attr["lb"] = lb // FP_ERROR * FP_ERROR + edge_attr["ub"] = ub // FP_ERROR * FP_ERROR diff --git a/lowtime/solver.py b/lowtime/solver.py index b5216a2..6cf9dfc 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -89,6 +89,8 @@ def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: object. Defaults to "op". """ self.attr_name = attr_name + # Set up Rust runner, will be initialized once critical DAG is formed + self.rust_runner = None # Run checks on the DAG and cache some properties. # Check: It's a DAG. @@ -302,17 +304,19 @@ def run(self) -> Generator[IterationResult, None, None]: sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), ) + # Preprocesses graph for Rust runner + nodes, edges = self.format_rust_inputs(critical_dag) + self.rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, + nodes, + critical_dag.graph["source_node"], + critical_dag.graph["sink_node"], + edges + ) + try: profiling_min_cut = time.time() - nodes, edges = self.format_rust_inputs(critical_dag) - rust_runner = _lowtime_rs.PhillipsDessouky( - FP_ERROR, - nodes, - critical_dag.graph["source_node"], - critical_dag.graph["sink_node"], - edges - ) - s_set, t_set = rust_runner.find_min_cut() + s_set, t_set = self.rust_runner.find_min_cut() profiling_min_cut = time.time() - profiling_min_cut logger.info( "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", @@ -324,7 +328,8 @@ def run(self) -> Generator[IterationResult, None, None]: break profiling_reduce = time.time() - cost_change = self.reduce_durations(critical_dag, s_set, t_set) + # cost_change = self.reduce_durations(critical_dag, s_set, t_set) + cost_change = self.rust_runner.reduce_durations() profiling_reduce = time.time() - profiling_reduce logger.info( "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", From 7ba89aec960205aca31e5baf4e4c3232393470cc Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 3 Dec 2024 18:06:31 -0500 Subject: [PATCH 27/55] redefine goal state with aoa_to_crit_dag --- ...n.goal.py => solver.2.aon_to_crit.goal.py} | 65 +++++++++++++++---- lowtime/solver.py | 65 +++++++++++++++---- 2 files changed, 106 insertions(+), 24 deletions(-) rename lowtime/{solver.2.reduce_duration.goal.py => solver.2.aon_to_crit.goal.py} (88%) diff --git a/lowtime/solver.2.reduce_duration.goal.py b/lowtime/solver.2.aon_to_crit.goal.py similarity index 88% rename from lowtime/solver.2.reduce_duration.goal.py rename to lowtime/solver.2.aon_to_crit.goal.py index 6cf9dfc..f0eed9f 100644 --- a/lowtime/solver.2.reduce_duration.goal.py +++ b/lowtime/solver.2.aon_to_crit.goal.py @@ -260,11 +260,32 @@ def run(self) -> Generator[IterationResult, None, None]: "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup ) + # TODO(ohjun): this should be here once all functions are implemented + # # Preprocesses graph for Rust runner + # nodes, edges = self.format_rust_inputs(critical_dag) + # self.rust_runner = _lowtime_rs.PhillipsDessouky( + # FP_ERROR, + # nodes, + # critical_dag.graph["source_node"], + # critical_dag.graph["sink_node"], + # edges + # ) + # Iteratively reduce the execution time of the DAG. for iteration in range(sys.maxsize): logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) profiling_iter = time.time() + # Preprocesses graph for Rust runner + nodes, edges = self.format_rust_inputs(critical_dag) + self.rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, + nodes, + critical_dag.graph["source_node"], + critical_dag.graph["sink_node"], + edges + ) + # At this point, `critical_dag` always exists and is what we want. # For the first iteration, the critical DAG is computed before the for # loop in order to estimate the number of iterations. For subsequent @@ -287,6 +308,7 @@ def run(self) -> Generator[IterationResult, None, None]: profiling_annotate = time.time() self.annotate_capacities(critical_dag) + # self.rust_runner.annotate_capacities() profiling_annotate = time.time() - profiling_annotate logger.info( "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", @@ -304,16 +326,6 @@ def run(self) -> Generator[IterationResult, None, None]: sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), ) - # Preprocesses graph for Rust runner - nodes, edges = self.format_rust_inputs(critical_dag) - self.rust_runner = _lowtime_rs.PhillipsDessouky( - FP_ERROR, - nodes, - critical_dag.graph["source_node"], - critical_dag.graph["sink_node"], - edges - ) - try: profiling_min_cut = time.time() s_set, t_set = self.rust_runner.find_min_cut() @@ -328,8 +340,8 @@ def run(self) -> Generator[IterationResult, None, None]: break profiling_reduce = time.time() - # cost_change = self.reduce_durations(critical_dag, s_set, t_set) - cost_change = self.rust_runner.reduce_durations() + cost_change = self.reduce_durations(critical_dag, s_set, t_set) + # cost_change = self.rust_runner.reduce_durations() profiling_reduce = time.time() - profiling_reduce logger.info( "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", @@ -343,6 +355,35 @@ def run(self) -> Generator[IterationResult, None, None]: # Earliest/latest start/finish times on operations also annotated here. critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + ####################################### + ########## NEW OHJUN START ########## + # ohjun: rust version of aoa_to_critical_dag + aoa_nodes, aoa_edges = self.format_rust_inputs(aoa_dag) + aoa_source_node = aoa_dag.graph["source_node"] + aoa_sink_node = aoa_dag.graph["sink_node"] + self.rust_runner.temp_aoa_to_critical_dag(aoa_nodes, aoa_source_node, aoa_sink_node, aoa_edges) + # ohjun: compare whether python vs rust versions are consistent + py_node_ids = critical_dag.get_dag_node_ids() + py_edges = critical_dag.get_dag_ek_processed_edges() + rs_node_ids = self.rust_runner.get_temp_crit_dag_node_ids() + rs_edges = self.rust_runner.get_temp_crit_dag_processed_edges() + + assert len(py_node_ids) == len(rs_node_ids), "LENGTH MISMATCH in node_ids" + assert py_node_ids == rs_node_ids, "DIFF in node_ids" + assert len(py_edges) == len(rs_edges), "LENGTH MISMATCH in edges" + # if sorted(py_edges) != sorted(rs_edges): + # for depr_edge, new_edge in zip(sorted(py_edges), sorted(rs_edges)): + # if depr_edge == new_edge: + # logger.info("edges EQUAL") + # logger.info(f"depr_edge: {depr_edge}") + # logger.info(f"new_edge : {new_edge}") + # else: + # logger.info("edges DIFFERENT") + # logger.info(f"depr_edge: {depr_edge}") + # logger.info(f"new_edge : {new_edge}") + assert sorted(py_edges) == sorted(rs_edges), "DIFF in edges" + ########## NEW OHJUN END ########## + ####################################### # We directly modify operation attributes in the DAG, so after we # ran one iteration, the AON DAG holds updated attributes. diff --git a/lowtime/solver.py b/lowtime/solver.py index 6cf9dfc..f0eed9f 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -260,11 +260,32 @@ def run(self) -> Generator[IterationResult, None, None]: "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup ) + # TODO(ohjun): this should be here once all functions are implemented + # # Preprocesses graph for Rust runner + # nodes, edges = self.format_rust_inputs(critical_dag) + # self.rust_runner = _lowtime_rs.PhillipsDessouky( + # FP_ERROR, + # nodes, + # critical_dag.graph["source_node"], + # critical_dag.graph["sink_node"], + # edges + # ) + # Iteratively reduce the execution time of the DAG. for iteration in range(sys.maxsize): logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) profiling_iter = time.time() + # Preprocesses graph for Rust runner + nodes, edges = self.format_rust_inputs(critical_dag) + self.rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, + nodes, + critical_dag.graph["source_node"], + critical_dag.graph["sink_node"], + edges + ) + # At this point, `critical_dag` always exists and is what we want. # For the first iteration, the critical DAG is computed before the for # loop in order to estimate the number of iterations. For subsequent @@ -287,6 +308,7 @@ def run(self) -> Generator[IterationResult, None, None]: profiling_annotate = time.time() self.annotate_capacities(critical_dag) + # self.rust_runner.annotate_capacities() profiling_annotate = time.time() - profiling_annotate logger.info( "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", @@ -304,16 +326,6 @@ def run(self) -> Generator[IterationResult, None, None]: sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), ) - # Preprocesses graph for Rust runner - nodes, edges = self.format_rust_inputs(critical_dag) - self.rust_runner = _lowtime_rs.PhillipsDessouky( - FP_ERROR, - nodes, - critical_dag.graph["source_node"], - critical_dag.graph["sink_node"], - edges - ) - try: profiling_min_cut = time.time() s_set, t_set = self.rust_runner.find_min_cut() @@ -328,8 +340,8 @@ def run(self) -> Generator[IterationResult, None, None]: break profiling_reduce = time.time() - # cost_change = self.reduce_durations(critical_dag, s_set, t_set) - cost_change = self.rust_runner.reduce_durations() + cost_change = self.reduce_durations(critical_dag, s_set, t_set) + # cost_change = self.rust_runner.reduce_durations() profiling_reduce = time.time() - profiling_reduce logger.info( "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", @@ -343,6 +355,35 @@ def run(self) -> Generator[IterationResult, None, None]: # Earliest/latest start/finish times on operations also annotated here. critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) + ####################################### + ########## NEW OHJUN START ########## + # ohjun: rust version of aoa_to_critical_dag + aoa_nodes, aoa_edges = self.format_rust_inputs(aoa_dag) + aoa_source_node = aoa_dag.graph["source_node"] + aoa_sink_node = aoa_dag.graph["sink_node"] + self.rust_runner.temp_aoa_to_critical_dag(aoa_nodes, aoa_source_node, aoa_sink_node, aoa_edges) + # ohjun: compare whether python vs rust versions are consistent + py_node_ids = critical_dag.get_dag_node_ids() + py_edges = critical_dag.get_dag_ek_processed_edges() + rs_node_ids = self.rust_runner.get_temp_crit_dag_node_ids() + rs_edges = self.rust_runner.get_temp_crit_dag_processed_edges() + + assert len(py_node_ids) == len(rs_node_ids), "LENGTH MISMATCH in node_ids" + assert py_node_ids == rs_node_ids, "DIFF in node_ids" + assert len(py_edges) == len(rs_edges), "LENGTH MISMATCH in edges" + # if sorted(py_edges) != sorted(rs_edges): + # for depr_edge, new_edge in zip(sorted(py_edges), sorted(rs_edges)): + # if depr_edge == new_edge: + # logger.info("edges EQUAL") + # logger.info(f"depr_edge: {depr_edge}") + # logger.info(f"new_edge : {new_edge}") + # else: + # logger.info("edges DIFFERENT") + # logger.info(f"depr_edge: {depr_edge}") + # logger.info(f"new_edge : {new_edge}") + assert sorted(py_edges) == sorted(rs_edges), "DIFF in edges" + ########## NEW OHJUN END ########## + ####################################### # We directly modify operation attributes in the DAG, so after we # ran one iteration, the AON DAG holds updated attributes. From b6ea2acfa64bf641e8a597d07e94aa2f4d2782a6 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 3 Dec 2024 18:22:52 -0500 Subject: [PATCH 28/55] set up rust side for aoa_to_crit --- lowtime/solver.py | 8 ++++---- src/graph_utils.rs | 7 +++++++ src/lib.rs | 1 + src/phillips_dessouky.rs | 40 ++++++++++++++++++++++++++++++++++------ 4 files changed, 46 insertions(+), 10 deletions(-) create mode 100644 src/graph_utils.rs diff --git a/lowtime/solver.py b/lowtime/solver.py index f0eed9f..f479e76 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -363,10 +363,10 @@ def run(self) -> Generator[IterationResult, None, None]: aoa_sink_node = aoa_dag.graph["sink_node"] self.rust_runner.temp_aoa_to_critical_dag(aoa_nodes, aoa_source_node, aoa_sink_node, aoa_edges) # ohjun: compare whether python vs rust versions are consistent - py_node_ids = critical_dag.get_dag_node_ids() - py_edges = critical_dag.get_dag_ek_processed_edges() - rs_node_ids = self.rust_runner.get_temp_crit_dag_node_ids() - rs_edges = self.rust_runner.get_temp_crit_dag_processed_edges() + py_node_ids = critical_dag.nodes + py_edges = critical_dag.edges(data="capacity") + rs_node_ids = self.rust_runner.get_dag_node_ids() + rs_edges = self.rust_runner.get_dag_ek_processed_edges() assert len(py_node_ids) == len(rs_node_ids), "LENGTH MISMATCH in node_ids" assert py_node_ids == rs_node_ids, "DIFF in node_ids" diff --git a/src/graph_utils.rs b/src/graph_utils.rs new file mode 100644 index 0000000..7fa3b6b --- /dev/null +++ b/src/graph_utils.rs @@ -0,0 +1,7 @@ +use crate::lowtime_graph::{LowtimeGraph, LowtimeEdge}; + + +pub fn aoa_to_critical_dag(aoa_dag: LowtimeGraph) -> LowtimeGraph { + // TODO + return aoa_dag; +} diff --git a/src/lib.rs b/src/lib.rs index a80a3c4..e80f2b0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ use pyo3::prelude::*; // mod cost_model; +mod graph_utils; mod lowtime_graph; mod operation; mod phillips_dessouky; diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index e132bc9..304cedf 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -12,6 +12,7 @@ use pathfinding::directed::edmonds_karp::{ use std::time::Instant; use log::{info, debug, error, Level}; +use crate::graph_utils; use crate::lowtime_graph::{LowtimeGraph, LowtimeEdge}; use crate::utils; @@ -43,6 +44,21 @@ impl PhillipsDessouky { }) } + // TESTING ohjun + fn get_dag_node_ids(&self) -> Vec { + self.dag.get_node_ids().clone() + } + + // TESTING ohjun + fn get_dag_ek_processed_edges(&self) -> Vec<((u32, u32), f64)> { + let rs_edges = self.dag.get_ek_preprocessed_edges(); + let py_edges: Vec<((u32, u32), f64)> = rs_edges.iter().map(|((from, to), cap)| { + ((*from, *to), cap.into_inner()) + }).collect(); + py_edges + } + + /// Find the min cut of the DAG annotated with lower/upper bound flow capacities. /// /// Assumptions: @@ -300,11 +316,23 @@ impl PhillipsDessouky { let t_set: HashSet = all_nodes.difference(&s_set).copied().collect(); (s_set, t_set) } -} -// not exposed to Python -impl PhillipsDessouky { - // fn max_flow(&self) -> Vec<((u32, u32), f64)> { - // self.graph.max_flow() - // } + fn temp_aoa_to_critical_dag(&mut self, + aoa_node_ids: Vec, + aoa_source_node_id: u32, + aoa_sink_node_id: u32, + aoa_edges_raw: Vec<((u32, u32), (f64, f64, f64, f64), Option<(bool, u64, u64, u64, u64, u64, u64, u64)>)>, + ) -> () { + let aoa_dag = LowtimeGraph::of_python( + aoa_node_ids, + aoa_source_node_id, + aoa_sink_node_id, + aoa_edges_raw, + ); + self.dag = graph_utils::aoa_to_critical_dag(aoa_dag); + } } + +// // not exposed to Python +// impl PhillipsDessouky { +// } From de77916731d23001eb8a6493a05feb92dee33c45 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 3 Dec 2024 18:37:54 -0500 Subject: [PATCH 29/55] final set up for aoa_to_crit --- lowtime/solver.py | 7 +++- src/graph_utils.rs | 84 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/lowtime/solver.py b/lowtime/solver.py index f479e76..343387f 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -361,7 +361,12 @@ def run(self) -> Generator[IterationResult, None, None]: aoa_nodes, aoa_edges = self.format_rust_inputs(aoa_dag) aoa_source_node = aoa_dag.graph["source_node"] aoa_sink_node = aoa_dag.graph["sink_node"] - self.rust_runner.temp_aoa_to_critical_dag(aoa_nodes, aoa_source_node, aoa_sink_node, aoa_edges) + self.rust_runner.temp_aoa_to_critical_dag( + aoa_nodes, + aoa_source_node, + aoa_sink_node, + aoa_edges, + ) # ohjun: compare whether python vs rust versions are consistent py_node_ids = critical_dag.nodes py_edges = critical_dag.edges(data="capacity") diff --git a/src/graph_utils.rs b/src/graph_utils.rs index 7fa3b6b..425b16f 100644 --- a/src/graph_utils.rs +++ b/src/graph_utils.rs @@ -1,7 +1,87 @@ use crate::lowtime_graph::{LowtimeGraph, LowtimeEdge}; +/// Convert the AOA DAG to a critical AOA DAG where only critical edges remain. +/// +/// This function modifies the earliest/latest start/end times of the `Operation`s +/// on the given graph. +/// +/// Assumptions: +/// - The graph is a DAG with `Operation`s annotated on edges. +/// - The graph has only one source node, annotated as "source_node" on the graph. +/// - The graph has only one sink node, annotated as "sink_node" on the graph. +// TODO(ohjun): adapt comment above for Rust version pub fn aoa_to_critical_dag(aoa_dag: LowtimeGraph) -> LowtimeGraph { - // TODO - return aoa_dag; + // if not nx.is_directed_acyclic_graph(aoa_dag): + // raise ValueError("The given graph is not a DAG.") + + // # Clear all earliest/latest start/end times. + // for _, _, edge_attr in aoa_dag.edges(data=True): + // operation: Operation = edge_attr[attr_name] + // operation.reset_times() + + // # Run the forward pass to set earliest start/end times. + // for node_id in nx.topological_sort(aoa_dag): + // for succ_id in aoa_dag.successors(node_id): + // cur_op: Operation = aoa_dag[node_id][succ_id][attr_name] + + // for succ_succ_id in aoa_dag.successors(succ_id): + // next_op: Operation = aoa_dag[succ_id][succ_succ_id][attr_name] + + // next_op.earliest_start = max( + // next_op.earliest_start, + // cur_op.earliest_finish, + // ) + // next_op.earliest_finish = next_op.earliest_start + next_op.duration + + // # Run the backward pass to set latest start/end times. + // # For the forward pass, `reset_times` was called on all `Operation`s, so we + // # didn't have to think about the initial values of earliest/latest_start. + // # For the backward pass, we need to find the largest `earliest_finish` value + // # among operations on incoming edges to the sink node, which is when the entire + // # DAG will finish executing. Then, we set that value as the latest_finish value + // # for operations on incoming edges to the sink node. + // sink_node = aoa_dag.graph["sink_node"] + // dag_earliest_finish = 0 + // for node_id in aoa_dag.predecessors(sink_node): + // op: Operation = aoa_dag[node_id][sink_node][attr_name] + // dag_earliest_finish = max(dag_earliest_finish, op.earliest_finish) + // for node_id in aoa_dag.predecessors(sink_node): + // op: Operation = aoa_dag[node_id][sink_node][attr_name] + // op.latest_finish = dag_earliest_finish + // op.latest_start = op.latest_finish - op.duration + + // for node_id in reversed(list(nx.topological_sort(aoa_dag))): + // for pred_id in aoa_dag.predecessors(node_id): + // cur_op: Operation = aoa_dag[pred_id][node_id][attr_name] + + // for pred_pred_id in aoa_dag.predecessors(pred_id): + // prev_op: Operation = aoa_dag[pred_pred_id][pred_id][attr_name] + + // prev_op.latest_start = min( + // prev_op.latest_start, + // cur_op.latest_start - prev_op.duration, + // ) + // prev_op.latest_finish = prev_op.latest_start + prev_op.duration + + // # Remove all edges that are not on the critical path. + // critical_dag = nx.DiGraph(aoa_dag) + // for u, v, edge_attr in aoa_dag.edges(data=True): + // op: Operation = edge_attr[attr_name] + // if op.earliest_finish != op.latest_finish: + // critical_dag.remove_edge(u, v) + + // # Copy over source and sink node IDs. + // source_id = critical_dag.graph["source_node"] = aoa_dag.graph["source_node"] + // sink_id = critical_dag.graph["sink_node"] = aoa_dag.graph["sink_node"] + // if source_id not in critical_dag and source_id in aoa_dag: + // raise RuntimeError( + // "Source node was removed from the DAG when getting critical DAG." + // ) + // if sink_id not in critical_dag and sink_id in aoa_dag: + // raise RuntimeError( + // "Sink node was removed from the DAG when getting critical DAG." + // ) + + // return critical_dag } From a5adb59a4b2c2963c92fc978db789e99990e9843 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 3 Dec 2024 19:55:24 -0500 Subject: [PATCH 30/55] issues with multiple immutable/mutable borrows --- lowtime/solver.2.aon_to_crit.goal.py | 34 ++++++++++++++----------- lowtime/solver.py | 19 +++++++------- src/graph_utils.rs | 38 ++++++++++++++++++++++------ src/lowtime_graph.rs | 24 +++++++++++++++++- src/operation.rs | 38 +++++++++++++++++++++------- 5 files changed, 110 insertions(+), 43 deletions(-) diff --git a/lowtime/solver.2.aon_to_crit.goal.py b/lowtime/solver.2.aon_to_crit.goal.py index f0eed9f..4d72110 100644 --- a/lowtime/solver.2.aon_to_crit.goal.py +++ b/lowtime/solver.2.aon_to_crit.goal.py @@ -276,16 +276,6 @@ def run(self) -> Generator[IterationResult, None, None]: logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) profiling_iter = time.time() - # Preprocesses graph for Rust runner - nodes, edges = self.format_rust_inputs(critical_dag) - self.rust_runner = _lowtime_rs.PhillipsDessouky( - FP_ERROR, - nodes, - critical_dag.graph["source_node"], - critical_dag.graph["sink_node"], - edges - ) - # At this point, `critical_dag` always exists and is what we want. # For the first iteration, the critical DAG is computed before the for # loop in order to estimate the number of iterations. For subsequent @@ -326,6 +316,15 @@ def run(self) -> Generator[IterationResult, None, None]: sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), ) + # Preprocesses graph for Rust runner + nodes, edges = self.format_rust_inputs(critical_dag) + self.rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, + nodes, + critical_dag.graph["source_node"], + critical_dag.graph["sink_node"], + edges + ) try: profiling_min_cut = time.time() s_set, t_set = self.rust_runner.find_min_cut() @@ -361,12 +360,17 @@ def run(self) -> Generator[IterationResult, None, None]: aoa_nodes, aoa_edges = self.format_rust_inputs(aoa_dag) aoa_source_node = aoa_dag.graph["source_node"] aoa_sink_node = aoa_dag.graph["sink_node"] - self.rust_runner.temp_aoa_to_critical_dag(aoa_nodes, aoa_source_node, aoa_sink_node, aoa_edges) + self.rust_runner.temp_aoa_to_critical_dag( + aoa_nodes, + aoa_source_node, + aoa_sink_node, + aoa_edges, + ) # ohjun: compare whether python vs rust versions are consistent - py_node_ids = critical_dag.get_dag_node_ids() - py_edges = critical_dag.get_dag_ek_processed_edges() - rs_node_ids = self.rust_runner.get_temp_crit_dag_node_ids() - rs_edges = self.rust_runner.get_temp_crit_dag_processed_edges() + py_node_ids = critical_dag.nodes + py_edges = critical_dag.edges(data="capacity") + rs_node_ids = self.rust_runner.get_dag_node_ids() + rs_edges = self.rust_runner.get_dag_ek_processed_edges() assert len(py_node_ids) == len(rs_node_ids), "LENGTH MISMATCH in node_ids" assert py_node_ids == rs_node_ids, "DIFF in node_ids" diff --git a/lowtime/solver.py b/lowtime/solver.py index 343387f..4d72110 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -276,16 +276,6 @@ def run(self) -> Generator[IterationResult, None, None]: logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) profiling_iter = time.time() - # Preprocesses graph for Rust runner - nodes, edges = self.format_rust_inputs(critical_dag) - self.rust_runner = _lowtime_rs.PhillipsDessouky( - FP_ERROR, - nodes, - critical_dag.graph["source_node"], - critical_dag.graph["sink_node"], - edges - ) - # At this point, `critical_dag` always exists and is what we want. # For the first iteration, the critical DAG is computed before the for # loop in order to estimate the number of iterations. For subsequent @@ -326,6 +316,15 @@ def run(self) -> Generator[IterationResult, None, None]: sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), ) + # Preprocesses graph for Rust runner + nodes, edges = self.format_rust_inputs(critical_dag) + self.rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, + nodes, + critical_dag.graph["source_node"], + critical_dag.graph["sink_node"], + edges + ) try: profiling_min_cut = time.time() s_set, t_set = self.rust_runner.find_min_cut() diff --git a/src/graph_utils.rs b/src/graph_utils.rs index 425b16f..7abd665 100644 --- a/src/graph_utils.rs +++ b/src/graph_utils.rs @@ -1,3 +1,7 @@ +use std::cmp; + +use log::{info, debug, Level}; + use crate::lowtime_graph::{LowtimeGraph, LowtimeEdge}; @@ -11,16 +15,32 @@ use crate::lowtime_graph::{LowtimeGraph, LowtimeEdge}; /// - The graph has only one source node, annotated as "source_node" on the graph. /// - The graph has only one sink node, annotated as "sink_node" on the graph. // TODO(ohjun): adapt comment above for Rust version -pub fn aoa_to_critical_dag(aoa_dag: LowtimeGraph) -> LowtimeGraph { - // if not nx.is_directed_acyclic_graph(aoa_dag): - // raise ValueError("The given graph is not a DAG.") +pub fn aoa_to_critical_dag(mut aoa_dag: LowtimeGraph) -> LowtimeGraph { + // Note: Python version checked whether aoa_dag is a dag; this Rust version does + // not. This extra check could be added in the future. - // # Clear all earliest/latest start/end times. - // for _, _, edge_attr in aoa_dag.edges(data=True): - // operation: Operation = edge_attr[attr_name] - // operation.reset_times() + // Clear all earliest/latest start/end times. + aoa_dag.edges_mut().for_each(|(_, _, edge)| edge.get_op_mut().reset_times()); - // # Run the forward pass to set earliest start/end times. + // Run the forward pass to set earliest start/end times. + for node_id in aoa_dag.get_topological_sorted_node_ids() { + if let Some(succs) = aoa_dag.successors(node_id) { + // let succs: Vec = succs.cloned().collect(); + for succ_id in succs { + let cur_op = aoa_dag.get_edge(node_id, *succ_id).get_op(); + if let Some(succ_succs) = aoa_dag.successors(*succ_id) { + // let succ_succs: Vec = succ_succs.cloned().collect(); + for succ_succ_id in succ_succs { + { + let next_op = aoa_dag.get_edge_mut(*succ_id, *succ_succ_id).get_op_mut(); + next_op.earliest_start = cmp::max(next_op.earliest_start, cur_op.earliest_finish); + next_op.earliest_finish = next_op.earliest_start + next_op.duration; + } + } + } + } + } + } // for node_id in nx.topological_sort(aoa_dag): // for succ_id in aoa_dag.successors(node_id): // cur_op: Operation = aoa_dag[node_id][succ_id][attr_name] @@ -84,4 +104,6 @@ pub fn aoa_to_critical_dag(aoa_dag: LowtimeGraph) -> LowtimeGraph { // ) // return critical_dag + + return aoa_dag; } diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index faa2450..11c17ee 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -6,8 +6,9 @@ use pathfinding::directed::edmonds_karp::{ SparseCapacity, Edge, EKFlows, - edmonds_karp + edmonds_karp, }; +use pathfinding::prelude::topological_sort; use std::time::Instant; use log::{info, debug, Level}; @@ -129,6 +130,10 @@ impl LowtimeGraph { self.edges.get(&node_id).map(|succs| succs.keys()) } + // pub fn successors_mut(&mut self, node_id: u32) -> Option> { + // self.edges.get(&node_id).map(|succs| succs.keys()) + // } + pub fn predecessors(&self, node_id: u32) -> Option> { self.preds.get(&node_id).map(|preds| preds.iter()) } @@ -149,6 +154,15 @@ impl LowtimeGraph { &self.node_ids } + pub fn get_topological_sorted_node_ids(&self) -> Vec { + topological_sort(&vec![self.get_source_node_id()], | node_id | { + match self.edges.get(&node_id) { + None => vec![], + Some(succs) => succs.keys().cloned().collect(), + } + }).unwrap() + } + pub fn add_node_id(&mut self, node_id: u32) -> () { assert!(self.node_ids.last().unwrap() < &node_id, "New node ids must be larger than all existing node ids"); self.node_ids.push(node_id) @@ -266,6 +280,14 @@ impl LowtimeEdge { } } + pub fn get_op(&self) -> &Operation { + self.op.as_ref().unwrap() + } + + pub fn get_op_mut(&mut self) -> &mut Operation { + self.op.as_mut().unwrap() + } + pub fn get_capacity(&self) -> OrderedFloat { self.capacity } diff --git a/src/operation.rs b/src/operation.rs index 393007b..3154fe2 100644 --- a/src/operation.rs +++ b/src/operation.rs @@ -5,14 +5,14 @@ pub struct Operation { is_dummy: bool, - duration: u64, - max_duration: u64, - min_duration: u64, + pub duration: u64, + pub max_duration: u64, + pub min_duration: u64, - earliest_start: u64, - latest_start: u64, - earliest_finish: u64, - latest_finish: u64, + pub earliest_start: u64, + pub latest_start: u64, + pub earliest_finish: u64, + pub latest_finish: u64, // cost_model: CostModel, } @@ -42,10 +42,30 @@ impl Operation { } } - pub fn get_duration(&self) -> u64 { - self.duration + pub fn reset_times(&mut self) -> () { + self.earliest_start = 0; + self.latest_start = u64::MAX; + self.earliest_finish = 0; + self.latest_finish = u64::MAX; } + // pub fn get_earliest_start(&self) -> u64 { + // self.earliest_start + // } + + // pub fn set_earliest_start(&mut self, new_earliest_start: u64) -> () { + // self.earliest_start = new_earliest_start + // } + + // pub fn get_latest_start(&self) -> u64 { + // self.latest_start + // } + + // pub fn set_latest_start(&mut self, new_latest_start: u64) -> () { + // self.latest_start = new_latest_start + // } + + // fn get_cost(&mut self, duration: u32) -> f64 { // self.cost_model.get_cost(duration) // } From edcc456a8eec2c7aad7f932a2ef69702a097875e Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 9 Dec 2024 13:17:00 -0500 Subject: [PATCH 31/55] undo aon_to_crit, polish full find_min_cut rewrite --- lowtime/solver.2.aon_to_crit.goal.py | 496 ------------------ lowtime/solver.py | 84 +-- src/graph_utils.rs | 109 ---- src/lib.rs | 3 - src/lowtime_graph.rs | 137 +---- src/{cost_model.rs => not_used.cost_model.rs} | 0 src/{operation.rs => not_used.operation.rs} | 25 - src/phillips_dessouky.rs | 102 ++-- src/utils.rs | 4 +- 9 files changed, 62 insertions(+), 898 deletions(-) delete mode 100644 lowtime/solver.2.aon_to_crit.goal.py delete mode 100644 src/graph_utils.rs rename src/{cost_model.rs => not_used.cost_model.rs} (100%) rename src/{operation.rs => not_used.operation.rs} (64%) diff --git a/lowtime/solver.2.aon_to_crit.goal.py b/lowtime/solver.2.aon_to_crit.goal.py deleted file mode 100644 index 4d72110..0000000 --- a/lowtime/solver.2.aon_to_crit.goal.py +++ /dev/null @@ -1,496 +0,0 @@ -# Copyright (C) 2023 Jae-Won Chung -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" - - -from __future__ import annotations - -import time -import sys -import logging -from collections import deque -from collections.abc import Generator - -import networkx as nx -from attrs import define, field - -from lowtime.operation import Operation -from lowtime.graph_utils import ( - aon_dag_to_aoa_dag, - aoa_to_critical_dag, - get_critical_aoa_dag_total_time, - get_total_cost, -) -from lowtime.exceptions import LowtimeFlowError -from lowtime import _lowtime_rs - -FP_ERROR = 1e-6 - -logger = logging.getLogger(__name__) - - -@define -class IterationResult: - """Holds results after one PD iteration. - - Attributes: - iteration: The number of optimization iterations experienced by the DAG. - cost_change: The increase in cost from reducing the DAG's - quantized execution time by 1. - cost: The current total cost of the DAG. - quant_time: The current quantized total execution time of the DAG. - unit_time: The unit time used for time quantization. - real_time: The current real (de-quantized) total execution time of the DAG. - """ - - iteration: int - cost_change: float - cost: float - quant_time: int - unit_time: float - real_time: float = field(init=False) - - def __attrs_post_init__(self) -> None: - """Set `real_time` after initialization.""" - self.real_time = self.quant_time * self.unit_time - - -class PhillipsDessouky: - """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" - - def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: - """Initialize the Phillips-Dessouky solver. - - Assumptions: - - The graph is a Directed Acyclic Graph (DAG). - - Operations are annotated on nodes with name `attr_name`. - - There is only one source (entry) node. The source node is annotated as - `dag.graph["source_node"]`. - - There is only one sink (exit) node. The sink node is annotated as - `dag.graph["sink_node"]`. - - The `unit_time` attribute of every operation is the same. - - Args: - dag: A networkx DiGraph object that represents the computation DAG. - The aforementioned assumptions should hold for the DAG. - attr_name: The name of the attribute on nodes that holds the operation - object. Defaults to "op". - """ - self.attr_name = attr_name - # Set up Rust runner, will be initialized once critical DAG is formed - self.rust_runner = None - - # Run checks on the DAG and cache some properties. - # Check: It's a DAG. - if not nx.is_directed_acyclic_graph(dag): - raise ValueError("The graph should be a Directed Acyclic Graph.") - - # Check: Only one source node that matches annotation. - if (source_node := dag.graph.get("source_node")) is None: - raise ValueError("The graph should have a `source_node` attribute.") - source_node_candidates = [] - for node_id, in_degree in dag.in_degree(): - if in_degree == 0: - source_node_candidates.append(node_id) - if len(source_node_candidates) == 0: - raise ValueError( - "Found zero nodes with in-degree 0. Cannot determine source node." - ) - if len(source_node_candidates) > 1: - raise ValueError( - f"Expecting only one source node, found {source_node_candidates}." - ) - if (detected_source_node := source_node_candidates[0]) != source_node: - raise ValueError( - f"Detected source node ({detected_source_node}) does not match " - f"the annotated source node ({source_node})." - ) - - # Check: Only one sink node that matches annotation. - if (sink_node := dag.graph.get("sink_node")) is None: - raise ValueError("The graph should have a `sink_node` attribute.") - sink_node_candidates = [] - for node_id, out_degree in dag.out_degree(): - if out_degree == 0: - sink_node_candidates.append(node_id) - if len(sink_node_candidates) == 0: - raise ValueError( - "Found zero nodes with out-degree 0. Cannot determine sink node." - ) - if len(sink_node_candidates) > 1: - raise ValueError( - f"Expecting only one sink node, found {sink_node_candidates}." - ) - if (detected_sink_node := sink_node_candidates[0]) != sink_node: - raise ValueError( - f"Detected sink node ({detected_sink_node}) does not match " - f"the annotated sink node ({sink_node})." - ) - - # Check: The `unit_time` attributes of every operation should be the same. - unit_time_candidates = set[float]() - for _, node_attr in dag.nodes(data=True): - if self.attr_name in node_attr: - op: Operation = node_attr[self.attr_name] - if op.is_dummy: - continue - unit_time_candidates.update( - option.unit_time for option in op.spec.options.options - ) - if len(unit_time_candidates) == 0: - raise ValueError( - "Found zero operations in the graph. Make sure you " - f"added operations as node attributes with key `{self.attr_name}`.", - ) - if len(unit_time_candidates) > 1: - raise ValueError( - f"Expecting the same `unit_time` across all operations, " - f"found {unit_time_candidates}." - ) - - self.aon_dag = dag - self.unit_time = unit_time_candidates.pop() - - # Helper function for Rust interop - def format_rust_inputs( - self, - dag: nx.DiGraph, - ) -> tuple[ - nx.classes.reportviews.NodeView, - list[ - tuple[ - tuple[int, int], - tuple[float, float, float, float], - tuple[bool, int, int, int, int, int, int, int] | None, - ] - ], - ]: - nodes = dag.nodes - edges = [] - for from_, to_, edge_attrs in dag.edges(data=True): - op_details = ( - None - if self.attr_name not in edge_attrs - else ( - edge_attrs[self.attr_name].is_dummy, - edge_attrs[self.attr_name].duration, - edge_attrs[self.attr_name].max_duration, - edge_attrs[self.attr_name].min_duration, - edge_attrs[self.attr_name].earliest_start, - edge_attrs[self.attr_name].latest_start, - edge_attrs[self.attr_name].earliest_finish, - edge_attrs[self.attr_name].latest_finish, - ) - ) - edges.append( - ( - (from_, to_), - ( - edge_attrs.get("capacity", 0), - edge_attrs.get("flow", 0), - edge_attrs.get("ub", 0), - edge_attrs.get("lb", 0), - ), - op_details, - ) - ) - return nodes, edges - - def run(self) -> Generator[IterationResult, None, None]: - """Run the algorithm and yield a DAG after each iteration. - - The solver will not deepcopy operations on the DAG but rather in-place modify - them for speed. The caller should deepcopy the operations or the DAG if needed - before running the next iteration. - - Upon yield, it is guaranteed that the earliest/latest start/finish time values - of all operations are up to date w.r.t. the `duration` of each operation. - """ - logger.info("Starting Phillips-Dessouky solver.") - profiling_setup = time.time() - - # Convert the original activity-on-node DAG to activity-on-arc DAG form. - # AOA DAGs are purely internal. All public input and output of this class - # should be in AON form. - aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) - - # Estimate the minimum execution time of the DAG by setting every operation - # to run at its minimum duration. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.min_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - min_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected minimum quantized execution time: %d", min_time) - - # Estimated the maximum execution time of the DAG by setting every operation - # to run at its maximum duration. This is also our initial start point. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.max_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - max_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected maximum quantized execution time: %d", max_time) - - num_iters = max_time - min_time + 1 - logger.info("Expected number of PD iterations: %d", num_iters) - - profiling_setup = time.time() - profiling_setup - logger.info( - "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup - ) - - # TODO(ohjun): this should be here once all functions are implemented - # # Preprocesses graph for Rust runner - # nodes, edges = self.format_rust_inputs(critical_dag) - # self.rust_runner = _lowtime_rs.PhillipsDessouky( - # FP_ERROR, - # nodes, - # critical_dag.graph["source_node"], - # critical_dag.graph["sink_node"], - # edges - # ) - - # Iteratively reduce the execution time of the DAG. - for iteration in range(sys.maxsize): - logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) - profiling_iter = time.time() - - # At this point, `critical_dag` always exists and is what we want. - # For the first iteration, the critical DAG is computed before the for - # loop in order to estimate the number of iterations. For subsequent - # iterations, the critcal DAG is computed after each iteration in - # in order to construct `IterationResult`. - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Critical DAG:") - logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) - logger.debug("Number of edges: %d", critical_dag.number_of_edges()) - non_dummy_ops = [ - attr[self.attr_name] - for _, _, attr in critical_dag.edges(data=True) - if not attr[self.attr_name].is_dummy - ] - logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) - logger.debug( - "Sum of non-dummy durations: %d", - sum(op.duration for op in non_dummy_ops), - ) - - profiling_annotate = time.time() - self.annotate_capacities(critical_dag) - # self.rust_runner.annotate_capacities() - profiling_annotate = time.time() - profiling_annotate - logger.info( - "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", - profiling_annotate, - ) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Capacity DAG:") - logger.debug( - "Total lb value: %f", - sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), - ) - logger.debug( - "Total ub value: %f", - sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), - ) - - # Preprocesses graph for Rust runner - nodes, edges = self.format_rust_inputs(critical_dag) - self.rust_runner = _lowtime_rs.PhillipsDessouky( - FP_ERROR, - nodes, - critical_dag.graph["source_node"], - critical_dag.graph["sink_node"], - edges - ) - try: - profiling_min_cut = time.time() - s_set, t_set = self.rust_runner.find_min_cut() - profiling_min_cut = time.time() - profiling_min_cut - logger.info( - "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", - profiling_min_cut, - ) - except LowtimeFlowError as e: - logger.info("Could not find minimum cut: %s", e.message) - logger.info("Terminating PD iteration.") - break - - profiling_reduce = time.time() - cost_change = self.reduce_durations(critical_dag, s_set, t_set) - # cost_change = self.rust_runner.reduce_durations() - profiling_reduce = time.time() - profiling_reduce - logger.info( - "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", - profiling_reduce, - ) - - if cost_change == float("inf") or abs(cost_change) < FP_ERROR: - logger.info("No further time reduction possible.") - logger.info("Terminating PD iteration.") - break - - # Earliest/latest start/finish times on operations also annotated here. - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - ####################################### - ########## NEW OHJUN START ########## - # ohjun: rust version of aoa_to_critical_dag - aoa_nodes, aoa_edges = self.format_rust_inputs(aoa_dag) - aoa_source_node = aoa_dag.graph["source_node"] - aoa_sink_node = aoa_dag.graph["sink_node"] - self.rust_runner.temp_aoa_to_critical_dag( - aoa_nodes, - aoa_source_node, - aoa_sink_node, - aoa_edges, - ) - # ohjun: compare whether python vs rust versions are consistent - py_node_ids = critical_dag.nodes - py_edges = critical_dag.edges(data="capacity") - rs_node_ids = self.rust_runner.get_dag_node_ids() - rs_edges = self.rust_runner.get_dag_ek_processed_edges() - - assert len(py_node_ids) == len(rs_node_ids), "LENGTH MISMATCH in node_ids" - assert py_node_ids == rs_node_ids, "DIFF in node_ids" - assert len(py_edges) == len(rs_edges), "LENGTH MISMATCH in edges" - # if sorted(py_edges) != sorted(rs_edges): - # for depr_edge, new_edge in zip(sorted(py_edges), sorted(rs_edges)): - # if depr_edge == new_edge: - # logger.info("edges EQUAL") - # logger.info(f"depr_edge: {depr_edge}") - # logger.info(f"new_edge : {new_edge}") - # else: - # logger.info("edges DIFFERENT") - # logger.info(f"depr_edge: {depr_edge}") - # logger.info(f"new_edge : {new_edge}") - assert sorted(py_edges) == sorted(rs_edges), "DIFF in edges" - ########## NEW OHJUN END ########## - ####################################### - - # We directly modify operation attributes in the DAG, so after we - # ran one iteration, the AON DAG holds updated attributes. - result = IterationResult( - iteration=iteration + 1, - cost_change=cost_change, - cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), - quant_time=get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ), - unit_time=self.unit_time, - ) - logger.info("%s", result) - profiling_iter = time.time() - profiling_iter - logger.info( - "PROFILING PhillipsDessouky::run single iteration time: %.10fs", - profiling_iter, - ) - yield result - - def reduce_durations( - self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] - ) -> float: - """Modify operation durations to reduce the DAG's execution time by 1.""" - speed_up_edges: list[Operation] = [] - for node_id in s_set: - for child_id in list(dag.successors(node_id)): - if child_id in t_set: - op: Operation = dag[node_id][child_id][self.attr_name] - speed_up_edges.append(op) - - slow_down_edges: list[Operation] = [] - for node_id in t_set: - for child_id in list(dag.successors(node_id)): - if child_id in s_set: - op: Operation = dag[node_id][child_id][self.attr_name] - slow_down_edges.append(op) - - if not speed_up_edges: - logger.info("No speed up candidate operations.") - return 0.0 - - cost_change = 0.0 - - # Reduce the duration of edges (speed up) by quant_time 1. - for op in speed_up_edges: - if op.is_dummy: - logger.info("Cannot speed up dummy operation.") - return float("inf") - if op.duration - 1 < op.min_duration: - logger.info("Operation %s has reached the limit of speed up", op) - return float("inf") - cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) - op_before_str = str(op) - op.duration -= 1 - logger.info("Sped up %s to %s", op_before_str, op) - - # Increase the duration of edges (slow down) by quant_time 1. - for op in slow_down_edges: - # Dummy edges can always be slowed down. - if op.is_dummy: - logger.info("Slowed down DummyOperation (didn't really slowdown).") - continue - elif op.duration + 1 > op.max_duration: - logger.info("Operation %s has reached the limit of slow down", op) - return float("inf") - cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) - before_op_str = str(op) - op.duration += 1 - logger.info("Slowed down %s to %s", before_op_str, op) - - return cost_change - - def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: - """In-place annotate the critical DAG with flow capacities.""" - # XXX(JW): Is this always large enough? - # It is necessary to monitor the `cost_change` value in `IterationResult` - # and make sure they are way smaller than this value. Even if the cost - # change is close or larger than this, users can scale down their cost - # value in `ExecutionOption`s. - inf = 10000.0 - for _, _, edge_attr in critical_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - duration = op.duration - # Dummy operations don't constrain the flow. - if op.is_dummy: # noqa: SIM114 - lb, ub = 0.0, inf - # Cannot be sped up or down. - elif duration == op.min_duration == op.max_duration: - lb, ub = 0.0, inf - # Cannot be sped up. - elif duration - 1 < op.min_duration: - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = inf - # Cannot be slowed down. - elif duration + 1 > op.max_duration: - lb = 0.0 - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) - else: - # In case the cost model is almost linear, give this edge some room. - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR - - # XXX(JW): This roundiing may not be necessary. - edge_attr["lb"] = lb // FP_ERROR * FP_ERROR - edge_attr["ub"] = ub // FP_ERROR * FP_ERROR diff --git a/lowtime/solver.py b/lowtime/solver.py index 4d72110..c3fb606 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -89,8 +89,6 @@ def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: object. Defaults to "op". """ self.attr_name = attr_name - # Set up Rust runner, will be initialized once critical DAG is formed - self.rust_runner = None # Run checks on the DAG and cache some properties. # Check: It's a DAG. @@ -173,27 +171,12 @@ def format_rust_inputs( tuple[ tuple[int, int], tuple[float, float, float, float], - tuple[bool, int, int, int, int, int, int, int] | None, ] ], ]: nodes = dag.nodes edges = [] for from_, to_, edge_attrs in dag.edges(data=True): - op_details = ( - None - if self.attr_name not in edge_attrs - else ( - edge_attrs[self.attr_name].is_dummy, - edge_attrs[self.attr_name].duration, - edge_attrs[self.attr_name].max_duration, - edge_attrs[self.attr_name].min_duration, - edge_attrs[self.attr_name].earliest_start, - edge_attrs[self.attr_name].latest_start, - edge_attrs[self.attr_name].earliest_finish, - edge_attrs[self.attr_name].latest_finish, - ) - ) edges.append( ( (from_, to_), @@ -203,7 +186,6 @@ def format_rust_inputs( edge_attrs.get("ub", 0), edge_attrs.get("lb", 0), ), - op_details, ) ) return nodes, edges @@ -260,17 +242,6 @@ def run(self) -> Generator[IterationResult, None, None]: "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup ) - # TODO(ohjun): this should be here once all functions are implemented - # # Preprocesses graph for Rust runner - # nodes, edges = self.format_rust_inputs(critical_dag) - # self.rust_runner = _lowtime_rs.PhillipsDessouky( - # FP_ERROR, - # nodes, - # critical_dag.graph["source_node"], - # critical_dag.graph["sink_node"], - # edges - # ) - # Iteratively reduce the execution time of the DAG. for iteration in range(sys.maxsize): logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) @@ -298,7 +269,6 @@ def run(self) -> Generator[IterationResult, None, None]: profiling_annotate = time.time() self.annotate_capacities(critical_dag) - # self.rust_runner.annotate_capacities() profiling_annotate = time.time() - profiling_annotate logger.info( "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", @@ -316,18 +286,17 @@ def run(self) -> Generator[IterationResult, None, None]: sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), ) - # Preprocesses graph for Rust runner - nodes, edges = self.format_rust_inputs(critical_dag) - self.rust_runner = _lowtime_rs.PhillipsDessouky( - FP_ERROR, - nodes, - critical_dag.graph["source_node"], - critical_dag.graph["sink_node"], - edges - ) try: profiling_min_cut = time.time() - s_set, t_set = self.rust_runner.find_min_cut() + nodes, edges = self.format_rust_inputs(critical_dag) + rust_runner = _lowtime_rs.PhillipsDessouky( + FP_ERROR, + nodes, + critical_dag.graph["source_node"], + critical_dag.graph["sink_node"], + edges + ) + s_set, t_set = rust_runner.find_min_cut() profiling_min_cut = time.time() - profiling_min_cut logger.info( "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", @@ -340,7 +309,6 @@ def run(self) -> Generator[IterationResult, None, None]: profiling_reduce = time.time() cost_change = self.reduce_durations(critical_dag, s_set, t_set) - # cost_change = self.rust_runner.reduce_durations() profiling_reduce = time.time() - profiling_reduce logger.info( "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", @@ -354,40 +322,6 @@ def run(self) -> Generator[IterationResult, None, None]: # Earliest/latest start/finish times on operations also annotated here. critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - ####################################### - ########## NEW OHJUN START ########## - # ohjun: rust version of aoa_to_critical_dag - aoa_nodes, aoa_edges = self.format_rust_inputs(aoa_dag) - aoa_source_node = aoa_dag.graph["source_node"] - aoa_sink_node = aoa_dag.graph["sink_node"] - self.rust_runner.temp_aoa_to_critical_dag( - aoa_nodes, - aoa_source_node, - aoa_sink_node, - aoa_edges, - ) - # ohjun: compare whether python vs rust versions are consistent - py_node_ids = critical_dag.nodes - py_edges = critical_dag.edges(data="capacity") - rs_node_ids = self.rust_runner.get_dag_node_ids() - rs_edges = self.rust_runner.get_dag_ek_processed_edges() - - assert len(py_node_ids) == len(rs_node_ids), "LENGTH MISMATCH in node_ids" - assert py_node_ids == rs_node_ids, "DIFF in node_ids" - assert len(py_edges) == len(rs_edges), "LENGTH MISMATCH in edges" - # if sorted(py_edges) != sorted(rs_edges): - # for depr_edge, new_edge in zip(sorted(py_edges), sorted(rs_edges)): - # if depr_edge == new_edge: - # logger.info("edges EQUAL") - # logger.info(f"depr_edge: {depr_edge}") - # logger.info(f"new_edge : {new_edge}") - # else: - # logger.info("edges DIFFERENT") - # logger.info(f"depr_edge: {depr_edge}") - # logger.info(f"new_edge : {new_edge}") - assert sorted(py_edges) == sorted(rs_edges), "DIFF in edges" - ########## NEW OHJUN END ########## - ####################################### # We directly modify operation attributes in the DAG, so after we # ran one iteration, the AON DAG holds updated attributes. diff --git a/src/graph_utils.rs b/src/graph_utils.rs deleted file mode 100644 index 7abd665..0000000 --- a/src/graph_utils.rs +++ /dev/null @@ -1,109 +0,0 @@ -use std::cmp; - -use log::{info, debug, Level}; - -use crate::lowtime_graph::{LowtimeGraph, LowtimeEdge}; - - -/// Convert the AOA DAG to a critical AOA DAG where only critical edges remain. -/// -/// This function modifies the earliest/latest start/end times of the `Operation`s -/// on the given graph. -/// -/// Assumptions: -/// - The graph is a DAG with `Operation`s annotated on edges. -/// - The graph has only one source node, annotated as "source_node" on the graph. -/// - The graph has only one sink node, annotated as "sink_node" on the graph. -// TODO(ohjun): adapt comment above for Rust version -pub fn aoa_to_critical_dag(mut aoa_dag: LowtimeGraph) -> LowtimeGraph { - // Note: Python version checked whether aoa_dag is a dag; this Rust version does - // not. This extra check could be added in the future. - - // Clear all earliest/latest start/end times. - aoa_dag.edges_mut().for_each(|(_, _, edge)| edge.get_op_mut().reset_times()); - - // Run the forward pass to set earliest start/end times. - for node_id in aoa_dag.get_topological_sorted_node_ids() { - if let Some(succs) = aoa_dag.successors(node_id) { - // let succs: Vec = succs.cloned().collect(); - for succ_id in succs { - let cur_op = aoa_dag.get_edge(node_id, *succ_id).get_op(); - if let Some(succ_succs) = aoa_dag.successors(*succ_id) { - // let succ_succs: Vec = succ_succs.cloned().collect(); - for succ_succ_id in succ_succs { - { - let next_op = aoa_dag.get_edge_mut(*succ_id, *succ_succ_id).get_op_mut(); - next_op.earliest_start = cmp::max(next_op.earliest_start, cur_op.earliest_finish); - next_op.earliest_finish = next_op.earliest_start + next_op.duration; - } - } - } - } - } - } - // for node_id in nx.topological_sort(aoa_dag): - // for succ_id in aoa_dag.successors(node_id): - // cur_op: Operation = aoa_dag[node_id][succ_id][attr_name] - - // for succ_succ_id in aoa_dag.successors(succ_id): - // next_op: Operation = aoa_dag[succ_id][succ_succ_id][attr_name] - - // next_op.earliest_start = max( - // next_op.earliest_start, - // cur_op.earliest_finish, - // ) - // next_op.earliest_finish = next_op.earliest_start + next_op.duration - - // # Run the backward pass to set latest start/end times. - // # For the forward pass, `reset_times` was called on all `Operation`s, so we - // # didn't have to think about the initial values of earliest/latest_start. - // # For the backward pass, we need to find the largest `earliest_finish` value - // # among operations on incoming edges to the sink node, which is when the entire - // # DAG will finish executing. Then, we set that value as the latest_finish value - // # for operations on incoming edges to the sink node. - // sink_node = aoa_dag.graph["sink_node"] - // dag_earliest_finish = 0 - // for node_id in aoa_dag.predecessors(sink_node): - // op: Operation = aoa_dag[node_id][sink_node][attr_name] - // dag_earliest_finish = max(dag_earliest_finish, op.earliest_finish) - // for node_id in aoa_dag.predecessors(sink_node): - // op: Operation = aoa_dag[node_id][sink_node][attr_name] - // op.latest_finish = dag_earliest_finish - // op.latest_start = op.latest_finish - op.duration - - // for node_id in reversed(list(nx.topological_sort(aoa_dag))): - // for pred_id in aoa_dag.predecessors(node_id): - // cur_op: Operation = aoa_dag[pred_id][node_id][attr_name] - - // for pred_pred_id in aoa_dag.predecessors(pred_id): - // prev_op: Operation = aoa_dag[pred_pred_id][pred_id][attr_name] - - // prev_op.latest_start = min( - // prev_op.latest_start, - // cur_op.latest_start - prev_op.duration, - // ) - // prev_op.latest_finish = prev_op.latest_start + prev_op.duration - - // # Remove all edges that are not on the critical path. - // critical_dag = nx.DiGraph(aoa_dag) - // for u, v, edge_attr in aoa_dag.edges(data=True): - // op: Operation = edge_attr[attr_name] - // if op.earliest_finish != op.latest_finish: - // critical_dag.remove_edge(u, v) - - // # Copy over source and sink node IDs. - // source_id = critical_dag.graph["source_node"] = aoa_dag.graph["source_node"] - // sink_id = critical_dag.graph["sink_node"] = aoa_dag.graph["sink_node"] - // if source_id not in critical_dag and source_id in aoa_dag: - // raise RuntimeError( - // "Source node was removed from the DAG when getting critical DAG." - // ) - // if sink_id not in critical_dag and sink_id in aoa_dag: - // raise RuntimeError( - // "Sink node was removed from the DAG when getting critical DAG." - // ) - - // return critical_dag - - return aoa_dag; -} diff --git a/src/lib.rs b/src/lib.rs index e80f2b0..30d4fae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,6 @@ use pyo3::prelude::*; -// mod cost_model; -mod graph_utils; mod lowtime_graph; -mod operation; mod phillips_dessouky; mod utils; diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index 11c17ee..17f4152 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -1,6 +1,4 @@ -use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; - use ordered_float::OrderedFloat; use pathfinding::directed::edmonds_karp::{ SparseCapacity, @@ -8,13 +6,6 @@ use pathfinding::directed::edmonds_karp::{ EKFlows, edmonds_karp, }; -use pathfinding::prelude::topological_sort; - -use std::time::Instant; -use log::{info, debug, Level}; - -use crate::operation::Operation; -use crate::utils; #[derive(Clone)] @@ -43,7 +34,7 @@ impl LowtimeGraph { mut node_ids: Vec, source_node_id: u32, sink_node_id: u32, - edges_raw: Vec<((u32, u32), (f64, f64, f64, f64), Option<(bool, u64, u64, u64, u64, u64, u64, u64)>)>, + edges_raw: Vec<((u32, u32), (f64, f64, f64, f64))>, ) -> Self { let mut graph = LowtimeGraph::new(); node_ids.sort(); @@ -54,51 +45,25 @@ impl LowtimeGraph { edges_raw.iter().for_each(|( (from, to), (capacity, flow, ub, lb), - op_details, )| { - let op = op_details.map(|( - is_dummy, - duration, - max_duration, - min_duration, - earliest_start, - latest_start, - earliest_finish, - latest_finish, - )| Operation::new( - is_dummy, - duration, - max_duration, - min_duration, - earliest_start, - latest_start, - earliest_finish, - latest_finish - )); - graph.add_edge(*from, *to, LowtimeEdge::new(op, *capacity, *flow, *ub, *lb)) + graph.add_edge(*from, *to, LowtimeEdge::new( + OrderedFloat(*capacity), + OrderedFloat(*flow), + OrderedFloat(*ub), + OrderedFloat(*lb), + )) }); graph } pub fn max_flow(&self) -> EKFlows> { let edges_edmonds_karp = self.get_ek_preprocessed_edges(); - - // TESTING(ohjun) - // info!("self.node_ids.len(): {}", self.node_ids.len()); - // info!("edges_edmonds_karp.len(): {}", edges_edmonds_karp.len()); - // info!("self.source_node_id.unwrap(): {}", self.source_node_id.unwrap()); - // info!("self.sink_node_id.unwrap(): {}", self.sink_node_id.unwrap()); - - let profiling_start = Instant::now(); let (flows, max_flow, min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( &self.node_ids, &self.source_node_id.unwrap(), &self.sink_node_id.unwrap(), edges_edmonds_karp, ); - let profiling_end = Instant::now(); - info!("PROFILING Rust_PhillipsDessouky::max_flow edmonds_karp time: {:.10}s", utils::profile_duration(profiling_start, profiling_end)); - (flows, max_flow, min_cut) } @@ -130,10 +95,6 @@ impl LowtimeGraph { self.edges.get(&node_id).map(|succs| succs.keys()) } - // pub fn successors_mut(&mut self, node_id: u32) -> Option> { - // self.edges.get(&node_id).map(|succs| succs.keys()) - // } - pub fn predecessors(&self, node_id: u32) -> Option> { self.preds.get(&node_id).map(|preds| preds.iter()) } @@ -154,15 +115,6 @@ impl LowtimeGraph { &self.node_ids } - pub fn get_topological_sorted_node_ids(&self) -> Vec { - topological_sort(&vec![self.get_source_node_id()], | node_id | { - match self.edges.get(&node_id) { - None => vec![], - Some(succs) => succs.keys().cloned().collect(), - } - }).unwrap() - } - pub fn add_node_id(&mut self, node_id: u32) -> () { assert!(self.node_ids.last().unwrap() < &node_id, "New node ids must be larger than all existing node ids"); self.node_ids.push(node_id) @@ -193,14 +145,7 @@ impl LowtimeGraph { self.num_edges += 1; } - // fn get_mut_op(&mut self, from: u32, to: u32) -> Option<&mut LowtimeEdge> { - // self.edges - // .get_mut(&from) - // .and_then(|to_edges| to_edges.get_mut(&to)) - // } - - // TESTING(ohjun): should make private when testing functions are deleted - pub fn get_ek_preprocessed_edges(&self, ) -> Vec>> { + fn get_ek_preprocessed_edges(&self, ) -> Vec>> { let mut processed_edges = Vec::with_capacity(self.num_edges); processed_edges.extend( self.edges.iter().flat_map(|(from, inner)| @@ -209,32 +154,10 @@ impl LowtimeGraph { ))); processed_edges } - - // TESTING(ohjun) - pub fn print_all_capacities(&self) -> () { - let mut processed_edges = self.get_ek_preprocessed_edges(); - processed_edges.sort_by(|((a_from, a_to), _a_cap): &((u32, u32), OrderedFloat), - ((b_from, b_to), _b_cap): &((u32, u32), OrderedFloat)| { - // a_from < b_from || (a_from == b_from && a_to < b_to) - let from_cmp = a_from.cmp(&b_from); - if from_cmp == Ordering::Equal { - a_to.cmp(&b_to) - } - else { - from_cmp - } - }); - info!("Rust Printing graph:"); - info!("Num edges: {}", processed_edges.len()); - processed_edges.iter().for_each(|((from, to), cap)| { - info!("{} -> {}: {}", from, to, cap); - }); - } } #[derive(Clone)] pub struct LowtimeEdge { - op: Option, capacity: OrderedFloat, flow: OrderedFloat, ub: OrderedFloat, @@ -243,51 +166,19 @@ pub struct LowtimeEdge { impl LowtimeEdge { pub fn new( - op: Option, - capacity: f64, - flow: f64, - ub: f64, - lb: f64, + capacity: OrderedFloat, + flow: OrderedFloat, + ub: OrderedFloat, + lb: OrderedFloat, ) -> Self { LowtimeEdge { - op, - capacity: OrderedFloat(capacity), - flow: OrderedFloat(flow), - ub: OrderedFloat(ub), - lb: OrderedFloat(lb), - } - } - - // TODO(ohjun): there's probably a better way to do this with default args - pub fn new_only_capacity(capacity: OrderedFloat) -> Self { - LowtimeEdge { - op: None, capacity, - flow: OrderedFloat(0.0), - ub: OrderedFloat(0.0), - lb: OrderedFloat(0.0), - } - } - - // TODO(ohjun): there's probably a better way to do this with default args - pub fn new_only_flow(flow: OrderedFloat) -> Self { - LowtimeEdge { - op: None, - capacity: OrderedFloat(0.0), flow, - ub: OrderedFloat(0.0), - lb: OrderedFloat(0.0), + ub, + lb, } } - pub fn get_op(&self) -> &Operation { - self.op.as_ref().unwrap() - } - - pub fn get_op_mut(&mut self) -> &mut Operation { - self.op.as_mut().unwrap() - } - pub fn get_capacity(&self) -> OrderedFloat { self.capacity } diff --git a/src/cost_model.rs b/src/not_used.cost_model.rs similarity index 100% rename from src/cost_model.rs rename to src/not_used.cost_model.rs diff --git a/src/operation.rs b/src/not_used.operation.rs similarity index 64% rename from src/operation.rs rename to src/not_used.operation.rs index 3154fe2..fc6454a 100644 --- a/src/operation.rs +++ b/src/not_used.operation.rs @@ -4,17 +4,13 @@ #[derive(Clone)] pub struct Operation { is_dummy: bool, - pub duration: u64, pub max_duration: u64, pub min_duration: u64, - pub earliest_start: u64, pub latest_start: u64, pub earliest_finish: u64, pub latest_finish: u64, - - // cost_model: CostModel, } impl Operation { @@ -48,25 +44,4 @@ impl Operation { self.earliest_finish = 0; self.latest_finish = u64::MAX; } - - // pub fn get_earliest_start(&self) -> u64 { - // self.earliest_start - // } - - // pub fn set_earliest_start(&mut self, new_earliest_start: u64) -> () { - // self.earliest_start = new_earliest_start - // } - - // pub fn get_latest_start(&self) -> u64 { - // self.latest_start - // } - - // pub fn set_latest_start(&mut self, new_latest_start: u64) -> () { - // self.latest_start = new_latest_start - // } - - - // fn get_cost(&mut self, duration: u32) -> f64 { - // self.cost_model.get_cost(duration) - // } } diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index 304cedf..2e09c68 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -1,20 +1,10 @@ use pyo3::prelude::*; use std::collections::{HashMap, HashSet, VecDeque}; - +use log::{debug, error, Level}; use ordered_float::OrderedFloat; -use pathfinding::directed::edmonds_karp::{ - SparseCapacity, - Edge, - edmonds_karp -}; - -use std::time::Instant; -use log::{info, debug, error, Level}; -use crate::graph_utils; use crate::lowtime_graph::{LowtimeGraph, LowtimeEdge}; -use crate::utils; #[pyclass] @@ -31,7 +21,7 @@ impl PhillipsDessouky { node_ids: Vec, source_node_id: u32, sink_node_id: u32, - edges_raw: Vec<((u32, u32), (f64, f64, f64, f64), Option<(bool, u64, u64, u64, u64, u64, u64, u64)>)>, + edges_raw: Vec<((u32, u32), (f64, f64, f64, f64))>, ) -> PyResult { Ok(PhillipsDessouky { dag: LowtimeGraph::of_python( @@ -44,21 +34,6 @@ impl PhillipsDessouky { }) } - // TESTING ohjun - fn get_dag_node_ids(&self) -> Vec { - self.dag.get_node_ids().clone() - } - - // TESTING ohjun - fn get_dag_ek_processed_edges(&self) -> Vec<((u32, u32), f64)> { - let rs_edges = self.dag.get_ek_preprocessed_edges(); - let py_edges: Vec<((u32, u32), f64)> = rs_edges.iter().map(|((from, to), cap)| { - ((*from, *to), cap.into_inner()) - }).collect(); - py_edges - } - - /// Find the min cut of the DAG annotated with lower/upper bound flow capacities. /// /// Assumptions: @@ -69,11 +44,10 @@ impl PhillipsDessouky { /// Returns: /// A tuple of (s_set, t_set) where s_set is the set of nodes on the source side /// of the min cut and t_set is the set of nodes on the sink side of the min cut. - /// Returns None if no feasible flow exists. - /// - /// Raises: - /// LowtimeFlowError: When no feasible flow exists. - // TODO(ohjun): fix documentation comment above to match rust version + /// + /// Panics if: + /// - Any of the assumptions are not true. + /// - No feasible flow exists. fn find_min_cut(&mut self) -> (HashSet, HashSet) { // In order to solve max flow on edges with both lower and upper bounds, // we first need to convert it to another DAG that only has upper bounds. @@ -99,7 +73,12 @@ impl PhillipsDessouky { acc + unbound_dag.get_edge(*pred_id, *u).get_lb() }); } - unbound_dag.add_edge(s_prime_id, *u, LowtimeEdge::new_only_capacity(capacity)); + unbound_dag.add_edge(s_prime_id, *u, LowtimeEdge::new( + capacity, + OrderedFloat(0.0), // flow + OrderedFloat(0.0), // ub + OrderedFloat(0.0), // lb + )); } // Add a new node t', which will become the new sink node. @@ -115,7 +94,12 @@ impl PhillipsDessouky { acc + unbound_dag.get_edge(*u, *succ_id).get_lb() }); } - unbound_dag.add_edge(*u, t_prime_id, LowtimeEdge::new_only_capacity(capacity)); + unbound_dag.add_edge(*u, t_prime_id, LowtimeEdge::new( + capacity, + OrderedFloat(0.0), // flow + OrderedFloat(0.0), // ub + OrderedFloat(0.0), // lb + )); } if log::log_enabled!(Level::Debug) { @@ -131,20 +115,18 @@ impl PhillipsDessouky { unbound_dag.add_edge( unbound_dag.get_sink_node_id(), unbound_dag.get_source_node_id(), - LowtimeEdge::new_only_capacity(OrderedFloat(f64::INFINITY)), + LowtimeEdge::new( + OrderedFloat(f64::INFINITY), // capacity + OrderedFloat(0.0), // flow + OrderedFloat(0.0), // ub + OrderedFloat(0.0), // lb + ), ); // Update source and sink on unbound_dag - // Note: This part is not in original Python solver, because the original solver - // never explicitly updates the source and sink; it simply passes in the - // new node_ids directly to max_flow. However, in this codebase, it makes - // sense to have LowtimeGraph be responsible for tracking its source/sink. - // I am noting this because it resulted in an extremely hard-to-find bug, - // and in the course of rewriting further a similar bug may appear again. unbound_dag.set_source_node_id(s_prime_id); unbound_dag.set_sink_node_id(t_prime_id); - // We're done with constructing the DAG with only flow upper bounds. // Find the maximum flow on this DAG. let (flows, _max_flow, _min_cut): ( @@ -186,7 +168,6 @@ impl PhillipsDessouky { flow_dict[&s_prime_id][u], unbound_dag.get_edge(s_prime_id, *u).get_capacity(), ); - // TODO(ohjun): integrate with pyo3 exceptions panic!("ERROR: Max flow on unbounded DAG didn't saturate."); } } @@ -205,7 +186,6 @@ impl PhillipsDessouky { flow_dict[u][&t_prime_id], unbound_dag.get_edge(*u, t_prime_id).get_capacity(), ); - // TODO(ohjun): integrate with pyo3 exceptions panic!("ERROR: Max flow on unbounded DAG didn't saturate."); } } @@ -242,7 +222,12 @@ impl PhillipsDessouky { match self.dag.has_edge(*v, *u) { true => residual_graph.get_edge_mut(*v, *u).set_capacity(vu_capacity), - false => residual_graph.add_edge(*v, *u, LowtimeEdge::new_only_capacity(vu_capacity)), + false => residual_graph.add_edge(*v, *u, LowtimeEdge::new( + vu_capacity, + OrderedFloat(0.0), // flow + OrderedFloat(0.0), // ub + OrderedFloat(0.0), // lb + )), } } @@ -278,8 +263,12 @@ impl PhillipsDessouky { let mut new_residual = self.dag.clone(); for (u, v, edge) in self.dag.edges() { new_residual.get_edge_mut(*u, *v).set_flow(edge.get_ub() - edge.get_flow()); - new_residual.add_edge(*v, *u, - LowtimeEdge::new_only_flow(edge.get_flow() - edge.get_lb())); + new_residual.add_edge(*v, *u, LowtimeEdge::new( + OrderedFloat(0.0), // capacity + edge.get_flow() - edge.get_lb(), // flow + OrderedFloat(0.0), // ub + OrderedFloat(0.0), // lb + )); } if log::log_enabled!(Level::Debug) { @@ -316,23 +305,4 @@ impl PhillipsDessouky { let t_set: HashSet = all_nodes.difference(&s_set).copied().collect(); (s_set, t_set) } - - fn temp_aoa_to_critical_dag(&mut self, - aoa_node_ids: Vec, - aoa_source_node_id: u32, - aoa_sink_node_id: u32, - aoa_edges_raw: Vec<((u32, u32), (f64, f64, f64, f64), Option<(bool, u64, u64, u64, u64, u64, u64, u64)>)>, - ) -> () { - let aoa_dag = LowtimeGraph::of_python( - aoa_node_ids, - aoa_source_node_id, - aoa_sink_node_id, - aoa_edges_raw, - ); - self.dag = graph_utils::aoa_to_critical_dag(aoa_dag); - } } - -// // not exposed to Python -// impl PhillipsDessouky { -// } diff --git a/src/utils.rs b/src/utils.rs index d22a298..391583f 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -3,7 +3,9 @@ use std::time::{ Duration, }; -pub fn profile_duration(start: Instant, end: Instant) -> f64 { +// This function is not used in the codebase, but it is left here +// to facilitate profiling during development. +pub fn get_duration(start: Instant, end: Instant) -> f64 { let duration: Duration = end.duration_since(start); let seconds = duration.as_secs(); let subsec_nanos = duration.subsec_nanos(); From 3874064c5e57fc00fb87fc9f5ee0adea71d87636 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 9 Dec 2024 13:18:39 -0500 Subject: [PATCH 32/55] remove unused files --- lowtime/solver.1.find-min-cut.full-fn.py | 755 ----------------- lowtime/solver.1.find_min_cut.final.py | 446 ---------- lowtime/solver.1.find_min_cut.up_to_first.py | 757 ----------------- lowtime/solver.1.find_min_cut.up_to_second.py | 759 ------------------ lowtime/solver.orig.py | 705 ---------------- src/not_used.cost_model.rs | 75 -- src/not_used.operation.rs | 47 -- 7 files changed, 3544 deletions(-) delete mode 100644 lowtime/solver.1.find-min-cut.full-fn.py delete mode 100644 lowtime/solver.1.find_min_cut.final.py delete mode 100644 lowtime/solver.1.find_min_cut.up_to_first.py delete mode 100644 lowtime/solver.1.find_min_cut.up_to_second.py delete mode 100644 lowtime/solver.orig.py delete mode 100644 src/not_used.cost_model.rs delete mode 100644 src/not_used.operation.rs diff --git a/lowtime/solver.1.find-min-cut.full-fn.py b/lowtime/solver.1.find-min-cut.full-fn.py deleted file mode 100644 index 2c0cd8c..0000000 --- a/lowtime/solver.1.find-min-cut.full-fn.py +++ /dev/null @@ -1,755 +0,0 @@ -# Copyright (C) 2023 Jae-Won Chung -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" - - -from __future__ import annotations - -import time -import sys -import logging -from collections import deque -from collections.abc import Generator - -import networkx as nx -from attrs import define, field - -from lowtime.operation import Operation -from lowtime.graph_utils import ( - aon_dag_to_aoa_dag, - aoa_to_critical_dag, - get_critical_aoa_dag_total_time, - get_total_cost, -) -from lowtime.exceptions import LowtimeFlowError -from lowtime import _lowtime_rs - -FP_ERROR = 1e-6 - -logger = logging.getLogger(__name__) - - -@define -class IterationResult: - """Holds results after one PD iteration. - - Attributes: - iteration: The number of optimization iterations experienced by the DAG. - cost_change: The increase in cost from reducing the DAG's - quantized execution time by 1. - cost: The current total cost of the DAG. - quant_time: The current quantized total execution time of the DAG. - unit_time: The unit time used for time quantization. - real_time: The current real (de-quantized) total execution time of the DAG. - """ - - iteration: int - cost_change: float - cost: float - quant_time: int - unit_time: float - real_time: float = field(init=False) - - def __attrs_post_init__(self) -> None: - """Set `real_time` after initialization.""" - self.real_time = self.quant_time * self.unit_time - - -class PhillipsDessouky: - """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" - - def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: - """Initialize the Phillips-Dessouky solver. - - Assumptions: - - The graph is a Directed Acyclic Graph (DAG). - - Operations are annotated on nodes with name `attr_name`. - - There is only one source (entry) node. The source node is annotated as - `dag.graph["source_node"]`. - - There is only one sink (exit) node. The sink node is annotated as - `dag.graph["sink_node"]`. - - The `unit_time` attribute of every operation is the same. - - Args: - dag: A networkx DiGraph object that represents the computation DAG. - The aforementioned assumptions should hold for the DAG. - attr_name: The name of the attribute on nodes that holds the operation - object. Defaults to "op". - """ - self.attr_name = attr_name - - # Run checks on the DAG and cache some properties. - # Check: It's a DAG. - if not nx.is_directed_acyclic_graph(dag): - raise ValueError("The graph should be a Directed Acyclic Graph.") - - # Check: Only one source node that matches annotation. - if (source_node := dag.graph.get("source_node")) is None: - raise ValueError("The graph should have a `source_node` attribute.") - source_node_candidates = [] - for node_id, in_degree in dag.in_degree(): - if in_degree == 0: - source_node_candidates.append(node_id) - if len(source_node_candidates) == 0: - raise ValueError( - "Found zero nodes with in-degree 0. Cannot determine source node." - ) - if len(source_node_candidates) > 1: - raise ValueError( - f"Expecting only one source node, found {source_node_candidates}." - ) - if (detected_source_node := source_node_candidates[0]) != source_node: - raise ValueError( - f"Detected source node ({detected_source_node}) does not match " - f"the annotated source node ({source_node})." - ) - - # Check: Only one sink node that matches annotation. - if (sink_node := dag.graph.get("sink_node")) is None: - raise ValueError("The graph should have a `sink_node` attribute.") - sink_node_candidates = [] - for node_id, out_degree in dag.out_degree(): - if out_degree == 0: - sink_node_candidates.append(node_id) - if len(sink_node_candidates) == 0: - raise ValueError( - "Found zero nodes with out-degree 0. Cannot determine sink node." - ) - if len(sink_node_candidates) > 1: - raise ValueError( - f"Expecting only one sink node, found {sink_node_candidates}." - ) - if (detected_sink_node := sink_node_candidates[0]) != sink_node: - raise ValueError( - f"Detected sink node ({detected_sink_node}) does not match " - f"the annotated sink node ({sink_node})." - ) - - # Check: The `unit_time` attributes of every operation should be the same. - unit_time_candidates = set[float]() - for _, node_attr in dag.nodes(data=True): - if self.attr_name in node_attr: - op: Operation = node_attr[self.attr_name] - if op.is_dummy: - continue - unit_time_candidates.update( - option.unit_time for option in op.spec.options.options - ) - if len(unit_time_candidates) == 0: - raise ValueError( - "Found zero operations in the graph. Make sure you " - f"added operations as node attributes with key `{self.attr_name}`.", - ) - if len(unit_time_candidates) > 1: - raise ValueError( - f"Expecting the same `unit_time` across all operations, " - f"found {unit_time_candidates}." - ) - - self.aon_dag = dag - self.unit_time = unit_time_candidates.pop() - - def run(self) -> Generator[IterationResult, None, None]: - """Run the algorithm and yield a DAG after each iteration. - - The solver will not deepcopy operations on the DAG but rather in-place modify - them for speed. The caller should deepcopy the operations or the DAG if needed - before running the next iteration. - - Upon yield, it is guaranteed that the earliest/latest start/finish time values - of all operations are up to date w.r.t. the `duration` of each operation. - """ - logger.info("Starting Phillips-Dessouky solver.") - profiling_setup = time.time() - - # Convert the original activity-on-node DAG to activity-on-arc DAG form. - # AOA DAGs are purely internal. All public input and output of this class - # should be in AON form. - aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) - - # Estimate the minimum execution time of the DAG by setting every operation - # to run at its minimum duration. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.min_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - min_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected minimum quantized execution time: %d", min_time) - - # Estimated the maximum execution time of the DAG by setting every operation - # to run at its maximum duration. This is also our initial start point. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.max_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - max_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected maximum quantized execution time: %d", max_time) - - num_iters = max_time - min_time + 1 - logger.info("Expected number of PD iterations: %d", num_iters) - - profiling_setup = time.time() - profiling_setup - logger.info( - "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup - ) - - # Iteratively reduce the execution time of the DAG. - for iteration in range(sys.maxsize): - logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) - profiling_iter = time.time() - - # At this point, `critical_dag` always exists and is what we want. - # For the first iteration, the critical DAG is computed before the for - # loop in order to estimate the number of iterations. For subsequent - # iterations, the critcal DAG is computed after each iteration in - # in order to construct `IterationResult`. - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Critical DAG:") - logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) - logger.debug("Number of edges: %d", critical_dag.number_of_edges()) - non_dummy_ops = [ - attr[self.attr_name] - for _, _, attr in critical_dag.edges(data=True) - if not attr[self.attr_name].is_dummy - ] - logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) - logger.debug( - "Sum of non-dummy durations: %d", - sum(op.duration for op in non_dummy_ops), - ) - - profiling_annotate = time.time() - self.annotate_capacities(critical_dag) - profiling_annotate = time.time() - profiling_annotate - logger.info( - "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", - profiling_annotate, - ) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Capacity DAG:") - logger.debug( - "Total lb value: %f", - sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), - ) - logger.debug( - "Total ub value: %f", - sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), - ) - - try: - profiling_min_cut = time.time() - s_set, t_set = self.find_min_cut(critical_dag) - profiling_min_cut = time.time() - profiling_min_cut - logger.info( - "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", - profiling_min_cut, - ) - except LowtimeFlowError as e: - logger.info("Could not find minimum cut: %s", e.message) - logger.info("Terminating PD iteration.") - break - - profiling_reduce = time.time() - cost_change = self.reduce_durations(critical_dag, s_set, t_set) - profiling_reduce = time.time() - profiling_reduce - logger.info( - "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", - profiling_reduce, - ) - - if cost_change == float("inf") or abs(cost_change) < FP_ERROR: - logger.info("No further time reduction possible.") - logger.info("Terminating PD iteration.") - break - - # Earliest/latest start/finish times on operations also annotated here. - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - - # We directly modify operation attributes in the DAG, so after we - # ran one iteration, the AON DAG holds updated attributes. - result = IterationResult( - iteration=iteration + 1, - cost_change=cost_change, - cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), - quant_time=get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ), - unit_time=self.unit_time, - ) - logger.info("%s", result) - profiling_iter = time.time() - profiling_iter - logger.info( - "PROFILING PhillipsDessouky::run single iteration time: %.10fs", - profiling_iter, - ) - yield result - - def reduce_durations( - self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] - ) -> float: - """Modify operation durations to reduce the DAG's execution time by 1.""" - speed_up_edges: list[Operation] = [] - for node_id in s_set: - for child_id in list(dag.successors(node_id)): - if child_id in t_set: - op: Operation = dag[node_id][child_id][self.attr_name] - speed_up_edges.append(op) - - slow_down_edges: list[Operation] = [] - for node_id in t_set: - for child_id in list(dag.successors(node_id)): - if child_id in s_set: - op: Operation = dag[node_id][child_id][self.attr_name] - slow_down_edges.append(op) - - if not speed_up_edges: - logger.info("No speed up candidate operations.") - return 0.0 - - cost_change = 0.0 - - # Reduce the duration of edges (speed up) by quant_time 1. - for op in speed_up_edges: - if op.is_dummy: - logger.info("Cannot speed up dummy operation.") - return float("inf") - if op.duration - 1 < op.min_duration: - logger.info("Operation %s has reached the limit of speed up", op) - return float("inf") - cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) - op_before_str = str(op) - op.duration -= 1 - logger.info("Sped up %s to %s", op_before_str, op) - - # Increase the duration of edges (slow down) by quant_time 1. - for op in slow_down_edges: - # Dummy edges can always be slowed down. - if op.is_dummy: - logger.info("Slowed down DummyOperation (didn't really slowdown).") - continue - elif op.duration + 1 > op.max_duration: - logger.info("Operation %s has reached the limit of slow down", op) - return float("inf") - cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) - before_op_str = str(op) - op.duration += 1 - logger.info("Slowed down %s to %s", before_op_str, op) - - return cost_change - - def find_min_cut(self, dag: nx.DiGraph) -> tuple[set[int], set[int]]: - """Find the min cut of the DAG annotated with lower/upper bound flow capacities. - - Assumptions: - - The capacity DAG is in AOA form. - - The capacity DAG has been annotated with `lb` and `ub` attributes on edges, - representing the lower and upper bounds of the flow on the edge. - - Returns: - A tuple of (s_set, t_set) where s_set is the set of nodes on the source side - of the min cut and t_set is the set of nodes on the sink side of the min cut. - Returns None if no feasible flow exists. - - Raises: - LowtimeFlowError: When no feasible flow exists. - """ - # Helper function for Rust interop - def format_rust_inputs( - dag: nx.DiGraph, - ) -> tuple[ - nx.classes.reportviews.NodeView, - list[ - tuple[ - tuple[int, int], - tuple[float, float, float, float], - tuple[bool, int, int, int, int, int, int, int] | None, - ] - ], - ]: - nodes = dag.nodes - edges = [] - for from_, to_, edge_attrs in dag.edges(data=True): - op_details = ( - None - if self.attr_name not in edge_attrs - else ( - edge_attrs[self.attr_name].is_dummy, - edge_attrs[self.attr_name].duration, - edge_attrs[self.attr_name].max_duration, - edge_attrs[self.attr_name].min_duration, - edge_attrs[self.attr_name].earliest_start, - edge_attrs[self.attr_name].latest_start, - edge_attrs[self.attr_name].earliest_finish, - edge_attrs[self.attr_name].latest_finish, - ) - ) - edges.append( - ( - (from_, to_), - ( - edge_attrs.get("capacity", 0), - edge_attrs.get("flow", 0), - edge_attrs.get("ub", 0), - edge_attrs.get("lb", 0), - ), - op_details, - ) - ) - return nodes, edges - - ########### NEW ########### - # Construct Rust runner with input (critical) dag - ohjun_nodes, ohjun_edges = format_rust_inputs(dag) - # print(ohjun_nodes) - # print(ohjun_edges) - ohjun_rust_runner = _lowtime_rs.PhillipsDessouky( - FP_ERROR, - ohjun_nodes, - dag.graph["source_node"], - dag.graph["sink_node"], - ohjun_edges - ) - ########### NEW ########### - - profiling_min_cut_setup = time.time() - source_node = dag.graph["source_node"] - sink_node = dag.graph["sink_node"] - - # In order to solve max flow on edges with both lower and upper bounds, - # we first need to convert it to another DAG that only has upper bounds. - unbound_dag = nx.DiGraph(dag) - - # For every edge, capacity = ub - lb. - for _, _, edge_attrs in unbound_dag.edges(data=True): - edge_attrs["capacity"] = edge_attrs["ub"] - edge_attrs["lb"] - - # Add a new node s', which will become the new source node. - # We constructed the AOA DAG, so we know that node IDs are integers. - node_ids: list[int] = list(unbound_dag.nodes) - s_prime_id = max(node_ids) + 1 - unbound_dag.add_node(s_prime_id) - - # For every node u in the original graph, add an edge (s', u) with capacity - # equal to the sum of all lower bounds of u's parents. - for u in dag.nodes: - capacity = 0.0 - for pred_id in dag.predecessors(u): - capacity += dag[pred_id][u]["lb"] - unbound_dag.add_edge(s_prime_id, u, capacity=capacity) - - # Add a new node t', which will become the new sink node. - t_prime_id = s_prime_id + 1 - unbound_dag.add_node(t_prime_id) - - # For every node u in the original graph, add an edge (u, t') with capacity - # equal to the sum of all lower bounds of u's children. - for u in dag.nodes: - capacity = 0.0 - for succ_id in dag.successors(u): - capacity += dag[u][succ_id]["lb"] - unbound_dag.add_edge(u, t_prime_id, capacity=capacity) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Unbound DAG") - logger.debug("Number of nodes: %d", unbound_dag.number_of_nodes()) - logger.debug("Number of edges: %d", unbound_dag.number_of_edges()) - logger.debug( - "Sum of capacities: %f", - sum(attr["capacity"] for _, _, attr in unbound_dag.edges(data=True)), - ) - - # Add an edge from t to s with infinite capacity. - unbound_dag.add_edge( - sink_node, - source_node, - capacity=float("inf"), - ) - - profiling_min_cut_setup = time.time() - profiling_min_cut_setup - logger.info( - "PROFILING PhillipsDessouky::find_min_cut setup time: %.10fs", - profiling_min_cut_setup, - ) - - # Helper function for Rust interop - # Rust's pathfinding::edmonds_karp does not return edges with 0 flow, - # but nx.max_flow does. So we fill in the 0s and empty nodes. - def reformat_rust_flow_to_dict( - flow_vec: list[tuple[tuple[int, int], float]], dag: nx.DiGraph - ) -> dict[int, dict[int, float]]: - flow_dict = dict() - for u in dag.nodes: - flow_dict[u] = dict() - for v in dag.successors(u): - flow_dict[u][v] = 0.0 - - for (u, v), cap in flow_vec: - flow_dict[u][v] = cap - - return flow_dict - - # We're done with constructing the DAG with only flow upper bounds. - # Find the maximum flow on this DAG. - profiling_max_flow = time.time() - nodes, edges = format_rust_inputs(unbound_dag) - - profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, s_prime_id, t_prime_id, edges) - profiling_data_transfer = time.time() - profiling_data_transfer - logger.info( - "PROFILING PhillipsDessouky::find_min_cut data transfer time: %.10fs", - profiling_data_transfer, - ) - - # ohjun: FIRST MAX FLOW - rust_flow_vec = rust_dag.max_flow_depr() - flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) - - profiling_max_flow = time.time() - profiling_max_flow - logger.info( - "PROFILING PhillipsDessouky::find_min_cut maximum_flow_1 time: %.10fs", - profiling_max_flow, - ) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("After first max flow") - total_flow = 0.0 - for d in flow_dict.values(): - for flow in d.values(): - total_flow += flow - logger.debug("Sum of all flow values: %f", total_flow) - - profiling_min_cut_between_max_flows = time.time() - - # Check if residual graph is saturated. If so, we have a feasible flow. - for u in unbound_dag.successors(s_prime_id): - if ( - abs(flow_dict[s_prime_id][u] - unbound_dag[s_prime_id][u]["capacity"]) - > FP_ERROR - ): - logger.error( - "s' -> %s unsaturated (flow: %s, capacity: %s)", - u, - flow_dict[s_prime_id][u], - unbound_dag[s_prime_id][u]["capacity"], - ) - raise LowtimeFlowError( - "ERROR: Max flow on unbounded DAG didn't saturate." - ) - for u in unbound_dag.predecessors(t_prime_id): - if ( - abs(flow_dict[u][t_prime_id] - unbound_dag[u][t_prime_id]["capacity"]) - > FP_ERROR - ): - logger.error( - "%s -> t' unsaturated (flow: %s, capacity: %s)", - u, - flow_dict[u][t_prime_id], - unbound_dag[u][t_prime_id]["capacity"], - ) - raise LowtimeFlowError( - "ERROR: Max flow on unbounded DAG didn't saturate." - ) - - # We have a feasible flow. Construct a new residual graph with the same - # shape as the capacity DAG so that we can find the min cut. - # First, retrieve the flow amounts to the original capacity graph, where for - # each edge u -> v, the flow amount is `flow + lb`. - for u, v in dag.edges: - dag[u][v]["flow"] = flow_dict[u][v] + dag[u][v]["lb"] - - # Construct a new residual graph (same shape as capacity DAG) with - # u -> v capacity `ub - flow` and v -> u capacity `flow - lb`. - residual_graph = nx.DiGraph(dag) - for u, v in dag.edges: - # Rounding small negative values to 0.0 avoids Rust-side - # pathfinding::edmonds_karp from entering unreachable code. - # This edge case did not exist in Python-side nx.max_flow call. - uv_capacity = residual_graph[u][v]["ub"] - residual_graph[u][v]["flow"] - uv_capacity = 0.0 if abs(uv_capacity) < FP_ERROR else uv_capacity - residual_graph[u][v]["capacity"] = uv_capacity - - vu_capacity = residual_graph[u][v]["flow"] - residual_graph[u][v]["lb"] - vu_capacity = 0.0 if abs(vu_capacity) < FP_ERROR else vu_capacity - if dag.has_edge(v, u): - residual_graph[v][u]["capacity"] = vu_capacity - else: - residual_graph.add_edge(v, u, capacity=vu_capacity) - - profiling_min_cut_between_max_flows = ( - time.time() - profiling_min_cut_between_max_flows - ) - logger.info( - "PROFILING PhillipsDessouky::find_min_cut between max flows time: %.10fs", - profiling_min_cut_between_max_flows, - ) - - # Run max flow on the new residual graph. - profiling_max_flow = time.time() - nodes, edges = format_rust_inputs(residual_graph) - - profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, source_node, sink_node, edges) - profiling_data_transfer = time.time() - profiling_data_transfer - logger.info( - "PROFILING PhillipsDessouky::find_min_cut data transfer 2 time: %.10fs", - profiling_data_transfer, - ) - - # ohjun: SECOND MAX FLOW - rust_flow_vec = rust_dag.max_flow_depr() - flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) - - profiling_max_flow = time.time() - profiling_max_flow - logger.info( - "PROFILING PhillipsDessouky::find_min_cut maximum_flow_2 time: %.10fs", - profiling_max_flow, - ) - - profiling_min_cut_after_max_flows = time.time() - - # Add additional flow we get to the original graph - for u, v in dag.edges: - dag[u][v]["flow"] += flow_dict[u][v] - dag[u][v]["flow"] -= flow_dict[v][u] - - # Construct the new residual graph. - new_residual = nx.DiGraph(dag) - for u, v in dag.edges: - new_residual[u][v]["flow"] = dag[u][v]["ub"] - dag[u][v]["flow"] - new_residual.add_edge(v, u, flow=dag[u][v]["flow"] - dag[u][v]["lb"]) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("New residual graph") - logger.debug("Number of nodes: %d", new_residual.number_of_nodes()) - logger.debug("Number of edges: %d", new_residual.number_of_edges()) - logger.debug( - "Sum of flow: %f", - sum(attr["flow"] for _, _, attr in new_residual.edges(data=True)), - ) - - # Find the s-t cut induced by the second maximum flow above. - # Only following `flow > 0` edges, find the set of nodes reachable from - # source node. That's the s-set, and the rest is the t-set. - s_set = set[int]() - q: deque[int] = deque() - q.append(source_node) - while q: - cur_id = q.pop() - s_set.add(cur_id) - if cur_id == sink_node: - break - for child_id in list(new_residual.successors(cur_id)): - if ( - child_id not in s_set - and abs(new_residual[cur_id][child_id]["flow"]) > FP_ERROR - ): - q.append(child_id) - t_set = set(new_residual.nodes) - s_set - - profiling_min_cut_after_max_flows = ( - time.time() - profiling_min_cut_after_max_flows - ) - logger.info( - "PROFILING PhillipsDessouky::find_min_cut after max flows time: %.10fs", - profiling_min_cut_after_max_flows, - ) - - ############# NEW ############# - ##### TESTING print all edges and capacities - # logger.info(f"s_prime_id: {s_prime_id}") - # logger.info(f"t_prime_id: {t_prime_id}") - # logger.info("Python Printing graph:") - # logger.info(f"Num edges: {len(unbound_dag.edges())}") - # for from_, to_, edge_attrs in unbound_dag.edges(data=True): - # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") - - # TEMP(ohjun): in current wip version, do everything until after 2nd max flow in Rust - ohjun_s_set, ohjun_t_set = ohjun_rust_runner.find_min_cut() - - depr_node_ids = list(new_residual.nodes) - depr_edges = list(new_residual.edges(data=False)) - new_node_ids = ohjun_rust_runner.get_new_residual_node_ids() - new_edges = ohjun_rust_runner.get_new_residual_ek_processed_edges() - new_edges = [from_to for (from_to, cap) in new_edges] - - # print(f"depr_node_ids: {depr_node_ids}") - # print(f"new_node_ids: {new_node_ids}") - assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" - assert depr_node_ids == new_node_ids, "DIFF in node_ids" - assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" - # for depr_edge, new_edge in zip(sorted(depr_edges), sorted(new_edges)): - # if depr_edge == new_edge: - # print("same:") - # else: - # print("DIFF:") - # print(f" depr_edge: {depr_edge}") - # print(f" new_edge: {new_edge}") - assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" - - # print(f"s_set: {s_set}") - # print(f"t_set: {t_set}") - # print(f"ohjun_s_set: {ohjun_s_set}") - # print(f"ohjun_t_set: {ohjun_t_set}") - assert len(s_set) == len(ohjun_s_set), "LENGTH MISMATCH in s_set" - assert len(t_set) == len(ohjun_t_set), "LENGTH MISMATCH in t_set" - assert s_set == ohjun_s_set, "DIFF in s_set" - assert t_set == ohjun_t_set, "DIFF in t_set" - - ############# NEW ############# - return s_set, t_set - - def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: - """In-place annotate the critical DAG with flow capacities.""" - # XXX(JW): Is this always large enough? - # It is necessary to monitor the `cost_change` value in `IterationResult` - # and make sure they are way smaller than this value. Even if the cost - # change is close or larger than this, users can scale down their cost - # value in `ExecutionOption`s. - inf = 10000.0 - for _, _, edge_attr in critical_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - duration = op.duration - # Dummy operations don't constrain the flow. - if op.is_dummy: # noqa: SIM114 - lb, ub = 0.0, inf - # Cannot be sped up or down. - elif duration == op.min_duration == op.max_duration: - lb, ub = 0.0, inf - # Cannot be sped up. - elif duration - 1 < op.min_duration: - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = inf - # Cannot be slowed down. - elif duration + 1 > op.max_duration: - lb = 0.0 - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) - else: - # In case the cost model is almost linear, give this edge some room. - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR - - # XXX(JW): This roundiing may not be necessary. - edge_attr["lb"] = lb // FP_ERROR * FP_ERROR - edge_attr["ub"] = ub // FP_ERROR * FP_ERROR diff --git a/lowtime/solver.1.find_min_cut.final.py b/lowtime/solver.1.find_min_cut.final.py deleted file mode 100644 index b5216a2..0000000 --- a/lowtime/solver.1.find_min_cut.final.py +++ /dev/null @@ -1,446 +0,0 @@ -# Copyright (C) 2023 Jae-Won Chung -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" - - -from __future__ import annotations - -import time -import sys -import logging -from collections import deque -from collections.abc import Generator - -import networkx as nx -from attrs import define, field - -from lowtime.operation import Operation -from lowtime.graph_utils import ( - aon_dag_to_aoa_dag, - aoa_to_critical_dag, - get_critical_aoa_dag_total_time, - get_total_cost, -) -from lowtime.exceptions import LowtimeFlowError -from lowtime import _lowtime_rs - -FP_ERROR = 1e-6 - -logger = logging.getLogger(__name__) - - -@define -class IterationResult: - """Holds results after one PD iteration. - - Attributes: - iteration: The number of optimization iterations experienced by the DAG. - cost_change: The increase in cost from reducing the DAG's - quantized execution time by 1. - cost: The current total cost of the DAG. - quant_time: The current quantized total execution time of the DAG. - unit_time: The unit time used for time quantization. - real_time: The current real (de-quantized) total execution time of the DAG. - """ - - iteration: int - cost_change: float - cost: float - quant_time: int - unit_time: float - real_time: float = field(init=False) - - def __attrs_post_init__(self) -> None: - """Set `real_time` after initialization.""" - self.real_time = self.quant_time * self.unit_time - - -class PhillipsDessouky: - """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" - - def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: - """Initialize the Phillips-Dessouky solver. - - Assumptions: - - The graph is a Directed Acyclic Graph (DAG). - - Operations are annotated on nodes with name `attr_name`. - - There is only one source (entry) node. The source node is annotated as - `dag.graph["source_node"]`. - - There is only one sink (exit) node. The sink node is annotated as - `dag.graph["sink_node"]`. - - The `unit_time` attribute of every operation is the same. - - Args: - dag: A networkx DiGraph object that represents the computation DAG. - The aforementioned assumptions should hold for the DAG. - attr_name: The name of the attribute on nodes that holds the operation - object. Defaults to "op". - """ - self.attr_name = attr_name - - # Run checks on the DAG and cache some properties. - # Check: It's a DAG. - if not nx.is_directed_acyclic_graph(dag): - raise ValueError("The graph should be a Directed Acyclic Graph.") - - # Check: Only one source node that matches annotation. - if (source_node := dag.graph.get("source_node")) is None: - raise ValueError("The graph should have a `source_node` attribute.") - source_node_candidates = [] - for node_id, in_degree in dag.in_degree(): - if in_degree == 0: - source_node_candidates.append(node_id) - if len(source_node_candidates) == 0: - raise ValueError( - "Found zero nodes with in-degree 0. Cannot determine source node." - ) - if len(source_node_candidates) > 1: - raise ValueError( - f"Expecting only one source node, found {source_node_candidates}." - ) - if (detected_source_node := source_node_candidates[0]) != source_node: - raise ValueError( - f"Detected source node ({detected_source_node}) does not match " - f"the annotated source node ({source_node})." - ) - - # Check: Only one sink node that matches annotation. - if (sink_node := dag.graph.get("sink_node")) is None: - raise ValueError("The graph should have a `sink_node` attribute.") - sink_node_candidates = [] - for node_id, out_degree in dag.out_degree(): - if out_degree == 0: - sink_node_candidates.append(node_id) - if len(sink_node_candidates) == 0: - raise ValueError( - "Found zero nodes with out-degree 0. Cannot determine sink node." - ) - if len(sink_node_candidates) > 1: - raise ValueError( - f"Expecting only one sink node, found {sink_node_candidates}." - ) - if (detected_sink_node := sink_node_candidates[0]) != sink_node: - raise ValueError( - f"Detected sink node ({detected_sink_node}) does not match " - f"the annotated sink node ({sink_node})." - ) - - # Check: The `unit_time` attributes of every operation should be the same. - unit_time_candidates = set[float]() - for _, node_attr in dag.nodes(data=True): - if self.attr_name in node_attr: - op: Operation = node_attr[self.attr_name] - if op.is_dummy: - continue - unit_time_candidates.update( - option.unit_time for option in op.spec.options.options - ) - if len(unit_time_candidates) == 0: - raise ValueError( - "Found zero operations in the graph. Make sure you " - f"added operations as node attributes with key `{self.attr_name}`.", - ) - if len(unit_time_candidates) > 1: - raise ValueError( - f"Expecting the same `unit_time` across all operations, " - f"found {unit_time_candidates}." - ) - - self.aon_dag = dag - self.unit_time = unit_time_candidates.pop() - - # Helper function for Rust interop - def format_rust_inputs( - self, - dag: nx.DiGraph, - ) -> tuple[ - nx.classes.reportviews.NodeView, - list[ - tuple[ - tuple[int, int], - tuple[float, float, float, float], - tuple[bool, int, int, int, int, int, int, int] | None, - ] - ], - ]: - nodes = dag.nodes - edges = [] - for from_, to_, edge_attrs in dag.edges(data=True): - op_details = ( - None - if self.attr_name not in edge_attrs - else ( - edge_attrs[self.attr_name].is_dummy, - edge_attrs[self.attr_name].duration, - edge_attrs[self.attr_name].max_duration, - edge_attrs[self.attr_name].min_duration, - edge_attrs[self.attr_name].earliest_start, - edge_attrs[self.attr_name].latest_start, - edge_attrs[self.attr_name].earliest_finish, - edge_attrs[self.attr_name].latest_finish, - ) - ) - edges.append( - ( - (from_, to_), - ( - edge_attrs.get("capacity", 0), - edge_attrs.get("flow", 0), - edge_attrs.get("ub", 0), - edge_attrs.get("lb", 0), - ), - op_details, - ) - ) - return nodes, edges - - def run(self) -> Generator[IterationResult, None, None]: - """Run the algorithm and yield a DAG after each iteration. - - The solver will not deepcopy operations on the DAG but rather in-place modify - them for speed. The caller should deepcopy the operations or the DAG if needed - before running the next iteration. - - Upon yield, it is guaranteed that the earliest/latest start/finish time values - of all operations are up to date w.r.t. the `duration` of each operation. - """ - logger.info("Starting Phillips-Dessouky solver.") - profiling_setup = time.time() - - # Convert the original activity-on-node DAG to activity-on-arc DAG form. - # AOA DAGs are purely internal. All public input and output of this class - # should be in AON form. - aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) - - # Estimate the minimum execution time of the DAG by setting every operation - # to run at its minimum duration. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.min_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - min_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected minimum quantized execution time: %d", min_time) - - # Estimated the maximum execution time of the DAG by setting every operation - # to run at its maximum duration. This is also our initial start point. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.max_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - max_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected maximum quantized execution time: %d", max_time) - - num_iters = max_time - min_time + 1 - logger.info("Expected number of PD iterations: %d", num_iters) - - profiling_setup = time.time() - profiling_setup - logger.info( - "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup - ) - - # Iteratively reduce the execution time of the DAG. - for iteration in range(sys.maxsize): - logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) - profiling_iter = time.time() - - # At this point, `critical_dag` always exists and is what we want. - # For the first iteration, the critical DAG is computed before the for - # loop in order to estimate the number of iterations. For subsequent - # iterations, the critcal DAG is computed after each iteration in - # in order to construct `IterationResult`. - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Critical DAG:") - logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) - logger.debug("Number of edges: %d", critical_dag.number_of_edges()) - non_dummy_ops = [ - attr[self.attr_name] - for _, _, attr in critical_dag.edges(data=True) - if not attr[self.attr_name].is_dummy - ] - logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) - logger.debug( - "Sum of non-dummy durations: %d", - sum(op.duration for op in non_dummy_ops), - ) - - profiling_annotate = time.time() - self.annotate_capacities(critical_dag) - profiling_annotate = time.time() - profiling_annotate - logger.info( - "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", - profiling_annotate, - ) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Capacity DAG:") - logger.debug( - "Total lb value: %f", - sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), - ) - logger.debug( - "Total ub value: %f", - sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), - ) - - try: - profiling_min_cut = time.time() - nodes, edges = self.format_rust_inputs(critical_dag) - rust_runner = _lowtime_rs.PhillipsDessouky( - FP_ERROR, - nodes, - critical_dag.graph["source_node"], - critical_dag.graph["sink_node"], - edges - ) - s_set, t_set = rust_runner.find_min_cut() - profiling_min_cut = time.time() - profiling_min_cut - logger.info( - "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", - profiling_min_cut, - ) - except LowtimeFlowError as e: - logger.info("Could not find minimum cut: %s", e.message) - logger.info("Terminating PD iteration.") - break - - profiling_reduce = time.time() - cost_change = self.reduce_durations(critical_dag, s_set, t_set) - profiling_reduce = time.time() - profiling_reduce - logger.info( - "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", - profiling_reduce, - ) - - if cost_change == float("inf") or abs(cost_change) < FP_ERROR: - logger.info("No further time reduction possible.") - logger.info("Terminating PD iteration.") - break - - # Earliest/latest start/finish times on operations also annotated here. - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - - # We directly modify operation attributes in the DAG, so after we - # ran one iteration, the AON DAG holds updated attributes. - result = IterationResult( - iteration=iteration + 1, - cost_change=cost_change, - cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), - quant_time=get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ), - unit_time=self.unit_time, - ) - logger.info("%s", result) - profiling_iter = time.time() - profiling_iter - logger.info( - "PROFILING PhillipsDessouky::run single iteration time: %.10fs", - profiling_iter, - ) - yield result - - def reduce_durations( - self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] - ) -> float: - """Modify operation durations to reduce the DAG's execution time by 1.""" - speed_up_edges: list[Operation] = [] - for node_id in s_set: - for child_id in list(dag.successors(node_id)): - if child_id in t_set: - op: Operation = dag[node_id][child_id][self.attr_name] - speed_up_edges.append(op) - - slow_down_edges: list[Operation] = [] - for node_id in t_set: - for child_id in list(dag.successors(node_id)): - if child_id in s_set: - op: Operation = dag[node_id][child_id][self.attr_name] - slow_down_edges.append(op) - - if not speed_up_edges: - logger.info("No speed up candidate operations.") - return 0.0 - - cost_change = 0.0 - - # Reduce the duration of edges (speed up) by quant_time 1. - for op in speed_up_edges: - if op.is_dummy: - logger.info("Cannot speed up dummy operation.") - return float("inf") - if op.duration - 1 < op.min_duration: - logger.info("Operation %s has reached the limit of speed up", op) - return float("inf") - cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) - op_before_str = str(op) - op.duration -= 1 - logger.info("Sped up %s to %s", op_before_str, op) - - # Increase the duration of edges (slow down) by quant_time 1. - for op in slow_down_edges: - # Dummy edges can always be slowed down. - if op.is_dummy: - logger.info("Slowed down DummyOperation (didn't really slowdown).") - continue - elif op.duration + 1 > op.max_duration: - logger.info("Operation %s has reached the limit of slow down", op) - return float("inf") - cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) - before_op_str = str(op) - op.duration += 1 - logger.info("Slowed down %s to %s", before_op_str, op) - - return cost_change - - def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: - """In-place annotate the critical DAG with flow capacities.""" - # XXX(JW): Is this always large enough? - # It is necessary to monitor the `cost_change` value in `IterationResult` - # and make sure they are way smaller than this value. Even if the cost - # change is close or larger than this, users can scale down their cost - # value in `ExecutionOption`s. - inf = 10000.0 - for _, _, edge_attr in critical_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - duration = op.duration - # Dummy operations don't constrain the flow. - if op.is_dummy: # noqa: SIM114 - lb, ub = 0.0, inf - # Cannot be sped up or down. - elif duration == op.min_duration == op.max_duration: - lb, ub = 0.0, inf - # Cannot be sped up. - elif duration - 1 < op.min_duration: - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = inf - # Cannot be slowed down. - elif duration + 1 > op.max_duration: - lb = 0.0 - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) - else: - # In case the cost model is almost linear, give this edge some room. - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR - - # XXX(JW): This roundiing may not be necessary. - edge_attr["lb"] = lb // FP_ERROR * FP_ERROR - edge_attr["ub"] = ub // FP_ERROR * FP_ERROR diff --git a/lowtime/solver.1.find_min_cut.up_to_first.py b/lowtime/solver.1.find_min_cut.up_to_first.py deleted file mode 100644 index c061c37..0000000 --- a/lowtime/solver.1.find_min_cut.up_to_first.py +++ /dev/null @@ -1,757 +0,0 @@ -# Copyright (C) 2023 Jae-Won Chung -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" - - -from __future__ import annotations - -import time -import sys -import logging -from collections import deque -from collections.abc import Generator - -import networkx as nx -from attrs import define, field - -from lowtime.operation import Operation -from lowtime.graph_utils import ( - aon_dag_to_aoa_dag, - aoa_to_critical_dag, - get_critical_aoa_dag_total_time, - get_total_cost, -) -from lowtime.exceptions import LowtimeFlowError -from lowtime import _lowtime_rs - -FP_ERROR = 1e-6 - -logger = logging.getLogger(__name__) - - -@define -class IterationResult: - """Holds results after one PD iteration. - - Attributes: - iteration: The number of optimization iterations experienced by the DAG. - cost_change: The increase in cost from reducing the DAG's - quantized execution time by 1. - cost: The current total cost of the DAG. - quant_time: The current quantized total execution time of the DAG. - unit_time: The unit time used for time quantization. - real_time: The current real (de-quantized) total execution time of the DAG. - """ - - iteration: int - cost_change: float - cost: float - quant_time: int - unit_time: float - real_time: float = field(init=False) - - def __attrs_post_init__(self) -> None: - """Set `real_time` after initialization.""" - self.real_time = self.quant_time * self.unit_time - - -class PhillipsDessouky: - """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" - - def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: - """Initialize the Phillips-Dessouky solver. - - Assumptions: - - The graph is a Directed Acyclic Graph (DAG). - - Operations are annotated on nodes with name `attr_name`. - - There is only one source (entry) node. The source node is annotated as - `dag.graph["source_node"]`. - - There is only one sink (exit) node. The sink node is annotated as - `dag.graph["sink_node"]`. - - The `unit_time` attribute of every operation is the same. - - Args: - dag: A networkx DiGraph object that represents the computation DAG. - The aforementioned assumptions should hold for the DAG. - attr_name: The name of the attribute on nodes that holds the operation - object. Defaults to "op". - """ - self.attr_name = attr_name - - # Run checks on the DAG and cache some properties. - # Check: It's a DAG. - if not nx.is_directed_acyclic_graph(dag): - raise ValueError("The graph should be a Directed Acyclic Graph.") - - # Check: Only one source node that matches annotation. - if (source_node := dag.graph.get("source_node")) is None: - raise ValueError("The graph should have a `source_node` attribute.") - source_node_candidates = [] - for node_id, in_degree in dag.in_degree(): - if in_degree == 0: - source_node_candidates.append(node_id) - if len(source_node_candidates) == 0: - raise ValueError( - "Found zero nodes with in-degree 0. Cannot determine source node." - ) - if len(source_node_candidates) > 1: - raise ValueError( - f"Expecting only one source node, found {source_node_candidates}." - ) - if (detected_source_node := source_node_candidates[0]) != source_node: - raise ValueError( - f"Detected source node ({detected_source_node}) does not match " - f"the annotated source node ({source_node})." - ) - - # Check: Only one sink node that matches annotation. - if (sink_node := dag.graph.get("sink_node")) is None: - raise ValueError("The graph should have a `sink_node` attribute.") - sink_node_candidates = [] - for node_id, out_degree in dag.out_degree(): - if out_degree == 0: - sink_node_candidates.append(node_id) - if len(sink_node_candidates) == 0: - raise ValueError( - "Found zero nodes with out-degree 0. Cannot determine sink node." - ) - if len(sink_node_candidates) > 1: - raise ValueError( - f"Expecting only one sink node, found {sink_node_candidates}." - ) - if (detected_sink_node := sink_node_candidates[0]) != sink_node: - raise ValueError( - f"Detected sink node ({detected_sink_node}) does not match " - f"the annotated sink node ({sink_node})." - ) - - # Check: The `unit_time` attributes of every operation should be the same. - unit_time_candidates = set[float]() - for _, node_attr in dag.nodes(data=True): - if self.attr_name in node_attr: - op: Operation = node_attr[self.attr_name] - if op.is_dummy: - continue - unit_time_candidates.update( - option.unit_time for option in op.spec.options.options - ) - if len(unit_time_candidates) == 0: - raise ValueError( - "Found zero operations in the graph. Make sure you " - f"added operations as node attributes with key `{self.attr_name}`.", - ) - if len(unit_time_candidates) > 1: - raise ValueError( - f"Expecting the same `unit_time` across all operations, " - f"found {unit_time_candidates}." - ) - - self.aon_dag = dag - self.unit_time = unit_time_candidates.pop() - - def run(self) -> Generator[IterationResult, None, None]: - """Run the algorithm and yield a DAG after each iteration. - - The solver will not deepcopy operations on the DAG but rather in-place modify - them for speed. The caller should deepcopy the operations or the DAG if needed - before running the next iteration. - - Upon yield, it is guaranteed that the earliest/latest start/finish time values - of all operations are up to date w.r.t. the `duration` of each operation. - """ - logger.info("Starting Phillips-Dessouky solver.") - profiling_setup = time.time() - - # Convert the original activity-on-node DAG to activity-on-arc DAG form. - # AOA DAGs are purely internal. All public input and output of this class - # should be in AON form. - aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) - - # Estimate the minimum execution time of the DAG by setting every operation - # to run at its minimum duration. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.min_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - min_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected minimum quantized execution time: %d", min_time) - - # Estimated the maximum execution time of the DAG by setting every operation - # to run at its maximum duration. This is also our initial start point. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.max_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - max_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected maximum quantized execution time: %d", max_time) - - num_iters = max_time - min_time + 1 - logger.info("Expected number of PD iterations: %d", num_iters) - - profiling_setup = time.time() - profiling_setup - logger.info( - "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup - ) - - # Iteratively reduce the execution time of the DAG. - for iteration in range(sys.maxsize): - logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) - profiling_iter = time.time() - - # At this point, `critical_dag` always exists and is what we want. - # For the first iteration, the critical DAG is computed before the for - # loop in order to estimate the number of iterations. For subsequent - # iterations, the critcal DAG is computed after each iteration in - # in order to construct `IterationResult`. - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Critical DAG:") - logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) - logger.debug("Number of edges: %d", critical_dag.number_of_edges()) - non_dummy_ops = [ - attr[self.attr_name] - for _, _, attr in critical_dag.edges(data=True) - if not attr[self.attr_name].is_dummy - ] - logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) - logger.debug( - "Sum of non-dummy durations: %d", - sum(op.duration for op in non_dummy_ops), - ) - - profiling_annotate = time.time() - self.annotate_capacities(critical_dag) - profiling_annotate = time.time() - profiling_annotate - logger.info( - "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", - profiling_annotate, - ) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Capacity DAG:") - logger.debug( - "Total lb value: %f", - sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), - ) - logger.debug( - "Total ub value: %f", - sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), - ) - - try: - profiling_min_cut = time.time() - s_set, t_set = self.find_min_cut(critical_dag) - profiling_min_cut = time.time() - profiling_min_cut - logger.info( - "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", - profiling_min_cut, - ) - except LowtimeFlowError as e: - logger.info("Could not find minimum cut: %s", e.message) - logger.info("Terminating PD iteration.") - break - - profiling_reduce = time.time() - cost_change = self.reduce_durations(critical_dag, s_set, t_set) - profiling_reduce = time.time() - profiling_reduce - logger.info( - "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", - profiling_reduce, - ) - - if cost_change == float("inf") or abs(cost_change) < FP_ERROR: - logger.info("No further time reduction possible.") - logger.info("Terminating PD iteration.") - break - - # Earliest/latest start/finish times on operations also annotated here. - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - - # We directly modify operation attributes in the DAG, so after we - # ran one iteration, the AON DAG holds updated attributes. - result = IterationResult( - iteration=iteration + 1, - cost_change=cost_change, - cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), - quant_time=get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ), - unit_time=self.unit_time, - ) - logger.info("%s", result) - profiling_iter = time.time() - profiling_iter - logger.info( - "PROFILING PhillipsDessouky::run single iteration time: %.10fs", - profiling_iter, - ) - yield result - - def reduce_durations( - self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] - ) -> float: - """Modify operation durations to reduce the DAG's execution time by 1.""" - speed_up_edges: list[Operation] = [] - for node_id in s_set: - for child_id in list(dag.successors(node_id)): - if child_id in t_set: - op: Operation = dag[node_id][child_id][self.attr_name] - speed_up_edges.append(op) - - slow_down_edges: list[Operation] = [] - for node_id in t_set: - for child_id in list(dag.successors(node_id)): - if child_id in s_set: - op: Operation = dag[node_id][child_id][self.attr_name] - slow_down_edges.append(op) - - if not speed_up_edges: - logger.info("No speed up candidate operations.") - return 0.0 - - cost_change = 0.0 - - # Reduce the duration of edges (speed up) by quant_time 1. - for op in speed_up_edges: - if op.is_dummy: - logger.info("Cannot speed up dummy operation.") - return float("inf") - if op.duration - 1 < op.min_duration: - logger.info("Operation %s has reached the limit of speed up", op) - return float("inf") - cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) - op_before_str = str(op) - op.duration -= 1 - logger.info("Sped up %s to %s", op_before_str, op) - - # Increase the duration of edges (slow down) by quant_time 1. - for op in slow_down_edges: - # Dummy edges can always be slowed down. - if op.is_dummy: - logger.info("Slowed down DummyOperation (didn't really slowdown).") - continue - elif op.duration + 1 > op.max_duration: - logger.info("Operation %s has reached the limit of slow down", op) - return float("inf") - cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) - before_op_str = str(op) - op.duration += 1 - logger.info("Slowed down %s to %s", before_op_str, op) - - return cost_change - - def find_min_cut(self, dag: nx.DiGraph) -> tuple[set[int], set[int]]: - """Find the min cut of the DAG annotated with lower/upper bound flow capacities. - - Assumptions: - - The capacity DAG is in AOA form. - - The capacity DAG has been annotated with `lb` and `ub` attributes on edges, - representing the lower and upper bounds of the flow on the edge. - - Returns: - A tuple of (s_set, t_set) where s_set is the set of nodes on the source side - of the min cut and t_set is the set of nodes on the sink side of the min cut. - Returns None if no feasible flow exists. - - Raises: - LowtimeFlowError: When no feasible flow exists. - """ - # Helper function for Rust interop - def format_rust_inputs( - dag: nx.DiGraph, - ) -> tuple[ - nx.classes.reportviews.NodeView, - list[ - tuple[ - tuple[int, int], - tuple[float, float, float, float], - tuple[bool, int, int, int, int, int, int, int] | None, - ] - ], - ]: - nodes = dag.nodes - edges = [] - for from_, to_, edge_attrs in dag.edges(data=True): - op_details = ( - None - if self.attr_name not in edge_attrs - else ( - edge_attrs[self.attr_name].is_dummy, - edge_attrs[self.attr_name].duration, - edge_attrs[self.attr_name].max_duration, - edge_attrs[self.attr_name].min_duration, - edge_attrs[self.attr_name].earliest_start, - edge_attrs[self.attr_name].latest_start, - edge_attrs[self.attr_name].earliest_finish, - edge_attrs[self.attr_name].latest_finish, - ) - ) - edges.append( - ( - (from_, to_), - ( - edge_attrs.get("capacity", 0), - edge_attrs.get("flow", 0), - edge_attrs.get("ub", 0), - edge_attrs.get("lb", 0), - ), - op_details, - ) - ) - return nodes, edges - - ########### NEW ########### - # Construct Rust runner with input (critical) dag - ohjun_nodes, ohjun_edges = format_rust_inputs(dag) - # print(ohjun_nodes) - # print(ohjun_edges) - ohjun_rust_runner = _lowtime_rs.PhillipsDessouky( - ohjun_nodes, - dag.graph["source_node"], - dag.graph["sink_node"], - ohjun_edges - ) - ########### NEW ########### - - profiling_min_cut_setup = time.time() - source_node = dag.graph["source_node"] - sink_node = dag.graph["sink_node"] - - # In order to solve max flow on edges with both lower and upper bounds, - # we first need to convert it to another DAG that only has upper bounds. - unbound_dag = nx.DiGraph(dag) - - # For every edge, capacity = ub - lb. - for _, _, edge_attrs in unbound_dag.edges(data=True): - edge_attrs["capacity"] = edge_attrs["ub"] - edge_attrs["lb"] - - # Add a new node s', which will become the new source node. - # We constructed the AOA DAG, so we know that node IDs are integers. - node_ids: list[int] = list(unbound_dag.nodes) - s_prime_id = max(node_ids) + 1 - unbound_dag.add_node(s_prime_id) - - # For every node u in the original graph, add an edge (s', u) with capacity - # equal to the sum of all lower bounds of u's parents. - for u in dag.nodes: - capacity = 0.0 - for pred_id in dag.predecessors(u): - capacity += dag[pred_id][u]["lb"] - unbound_dag.add_edge(s_prime_id, u, capacity=capacity) - - # Add a new node t', which will become the new sink node. - t_prime_id = s_prime_id + 1 - unbound_dag.add_node(t_prime_id) - - # For every node u in the original graph, add an edge (u, t') with capacity - # equal to the sum of all lower bounds of u's children. - for u in dag.nodes: - capacity = 0.0 - for succ_id in dag.successors(u): - capacity += dag[u][succ_id]["lb"] - unbound_dag.add_edge(u, t_prime_id, capacity=capacity) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Unbound DAG") - logger.debug("Number of nodes: %d", unbound_dag.number_of_nodes()) - logger.debug("Number of edges: %d", unbound_dag.number_of_edges()) - logger.debug( - "Sum of capacities: %f", - sum(attr["capacity"] for _, _, attr in unbound_dag.edges(data=True)), - ) - - # Add an edge from t to s with infinite capacity. - unbound_dag.add_edge( - sink_node, - source_node, - capacity=float("inf"), - ) - - profiling_min_cut_setup = time.time() - profiling_min_cut_setup - logger.info( - "PROFILING PhillipsDessouky::find_min_cut setup time: %.10fs", - profiling_min_cut_setup, - ) - - # Helper function for Rust interop - # Rust's pathfinding::edmonds_karp does not return edges with 0 flow, - # but nx.max_flow does. So we fill in the 0s and empty nodes. - def reformat_rust_flow_to_dict( - flow_vec: list[tuple[tuple[int, int], float]], dag: nx.DiGraph - ) -> dict[int, dict[int, float]]: - flow_dict = dict() - for u in dag.nodes: - flow_dict[u] = dict() - for v in dag.successors(u): - flow_dict[u][v] = 0.0 - - for (u, v), cap in flow_vec: - flow_dict[u][v] = cap - - return flow_dict - - # We're done with constructing the DAG with only flow upper bounds. - # Find the maximum flow on this DAG. - profiling_max_flow = time.time() - nodes, edges = format_rust_inputs(unbound_dag) - - profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(nodes, s_prime_id, t_prime_id, edges) - profiling_data_transfer = time.time() - profiling_data_transfer - logger.info( - "PROFILING PhillipsDessouky::find_min_cut data transfer time: %.10fs", - profiling_data_transfer, - ) - - ##### TESTING print all edges and capacities - # logger.info(f"s_prime_id: {s_prime_id}") - # logger.info(f"t_prime_id: {t_prime_id}") - # logger.info("Python Printing graph:") - # logger.info(f"Num edges: {len(unbound_dag.edges())}") - # for from_, to_, edge_attrs in unbound_dag.edges(data=True): - # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") - - # ohjun: FIRST MAX FLOW - rust_flow_vec = rust_dag.max_flow_depr() - flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) - - ############# NEW ############# - # TEMP(ohjun): in current wip version, do everything until after 1st max flow in Rust - ohjun_rust_flow_vec = ohjun_rust_runner.find_min_cut_wip() - ohjun_flow_dict = reformat_rust_flow_to_dict(ohjun_rust_flow_vec, unbound_dag) - - depr_node_ids = rust_dag.get_dag_node_ids() - depr_edges = rust_dag.get_dag_ek_processed_edges() - new_node_ids = ohjun_rust_runner.get_unbound_dag_node_ids() - new_edges = ohjun_rust_runner.get_unbound_dag_ek_processed_edges() - print(f"depr_node_ids: {depr_node_ids}") - print(f"new_node_ids: {new_node_ids}") - assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" - assert depr_node_ids == new_node_ids, "DIFF in node_ids" - assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" - if sorted(depr_edges) != sorted(new_edges): - for depr_edge, new_edge in zip(sorted(depr_edges), sorted(new_edges)): - if depr_edge == new_edge: - logger.info("edges EQUAL") - logger.info(f"depr_edge: {depr_edge}") - logger.info(f"new_edge : {new_edge}") - else: - logger.info("edges DIFFERENT") - logger.info(f"depr_edge: {depr_edge}") - logger.info(f"new_edge : {new_edge}") - assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" - - def print_dict(d): - for from_, inner in d.items(): - for to_, flow in inner.items(): - logger.info(f'{from_} -> {to_}: {flow}') - logger.info('flow_dict:') - # print_dict(flow_dict) - logger.info('ohjun_flow_dict:') - # print_dict(ohjun_flow_dict) - assert flow_dict == ohjun_flow_dict, "flow dicts were different :(" - ############# NEW ############# - - profiling_max_flow = time.time() - profiling_max_flow - logger.info( - "PROFILING PhillipsDessouky::find_min_cut maximum_flow_1 time: %.10fs", - profiling_max_flow, - ) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("After first max flow") - total_flow = 0.0 - for d in flow_dict.values(): - for flow in d.values(): - total_flow += flow - logger.debug("Sum of all flow values: %f", total_flow) - - profiling_min_cut_between_max_flows = time.time() - - # Check if residual graph is saturated. If so, we have a feasible flow. - for u in unbound_dag.successors(s_prime_id): - if ( - abs(flow_dict[s_prime_id][u] - unbound_dag[s_prime_id][u]["capacity"]) - > FP_ERROR - ): - logger.error( - "s' -> %s unsaturated (flow: %s, capacity: %s)", - u, - flow_dict[s_prime_id][u], - unbound_dag[s_prime_id][u]["capacity"], - ) - raise LowtimeFlowError( - "ERROR: Max flow on unbounded DAG didn't saturate." - ) - for u in unbound_dag.predecessors(t_prime_id): - if ( - abs(flow_dict[u][t_prime_id] - unbound_dag[u][t_prime_id]["capacity"]) - > FP_ERROR - ): - logger.error( - "%s -> t' unsaturated (flow: %s, capacity: %s)", - u, - flow_dict[u][t_prime_id], - unbound_dag[u][t_prime_id]["capacity"], - ) - raise LowtimeFlowError( - "ERROR: Max flow on unbounded DAG didn't saturate." - ) - - # We have a feasible flow. Construct a new residual graph with the same - # shape as the capacity DAG so that we can find the min cut. - # First, retrieve the flow amounts to the original capacity graph, where for - # each edge u -> v, the flow amount is `flow + lb`. - for u, v in dag.edges: - dag[u][v]["flow"] = flow_dict[u][v] + dag[u][v]["lb"] - - # Construct a new residual graph (same shape as capacity DAG) with - # u -> v capacity `ub - flow` and v -> u capacity `flow - lb`. - residual_graph = nx.DiGraph(dag) - for u, v in dag.edges: - # Rounding small negative values to 0.0 avoids Rust-side - # pathfinding::edmonds_karp from entering unreachable code. - # This edge case did not exist in Python-side nx.max_flow call. - uv_capacity = residual_graph[u][v]["ub"] - residual_graph[u][v]["flow"] - uv_capacity = 0.0 if abs(uv_capacity) < FP_ERROR else uv_capacity - residual_graph[u][v]["capacity"] = uv_capacity - - vu_capacity = residual_graph[u][v]["flow"] - residual_graph[u][v]["lb"] - vu_capacity = 0.0 if abs(vu_capacity) < FP_ERROR else vu_capacity - if dag.has_edge(v, u): - residual_graph[v][u]["capacity"] = vu_capacity - else: - residual_graph.add_edge(v, u, capacity=vu_capacity) - - profiling_min_cut_between_max_flows = ( - time.time() - profiling_min_cut_between_max_flows - ) - logger.info( - "PROFILING PhillipsDessouky::find_min_cut between max flows time: %.10fs", - profiling_min_cut_between_max_flows, - ) - - # Run max flow on the new residual graph. - profiling_max_flow = time.time() - nodes, edges = format_rust_inputs(residual_graph) - - profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(nodes, source_node, sink_node, edges) - profiling_data_transfer = time.time() - profiling_data_transfer - logger.info( - "PROFILING PhillipsDessouky::find_min_cut data transfer 2 time: %.10fs", - profiling_data_transfer, - ) - - # ohjun: SECOND MAX FLOW - rust_flow_vec = rust_dag.max_flow_depr() - flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) - - profiling_max_flow = time.time() - profiling_max_flow - logger.info( - "PROFILING PhillipsDessouky::find_min_cut maximum_flow_2 time: %.10fs", - profiling_max_flow, - ) - - profiling_min_cut_after_max_flows = time.time() - - # Add additional flow we get to the original graph - for u, v in dag.edges: - dag[u][v]["flow"] += flow_dict[u][v] - dag[u][v]["flow"] -= flow_dict[v][u] - - # Construct the new residual graph. - new_residual = nx.DiGraph(dag) - for u, v in dag.edges: - new_residual[u][v]["flow"] = dag[u][v]["ub"] - dag[u][v]["flow"] - new_residual.add_edge(v, u, flow=dag[u][v]["flow"] - dag[u][v]["lb"]) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("New residual graph") - logger.debug("Number of nodes: %d", new_residual.number_of_nodes()) - logger.debug("Number of edges: %d", new_residual.number_of_edges()) - logger.debug( - "Sum of flow: %f", - sum(attr["flow"] for _, _, attr in new_residual.edges(data=True)), - ) - - # Find the s-t cut induced by the second maximum flow above. - # Only following `flow > 0` edges, find the set of nodes reachable from - # source node. That's the s-set, and the rest is the t-set. - s_set = set[int]() - q: deque[int] = deque() - q.append(source_node) - while q: - cur_id = q.pop() - s_set.add(cur_id) - if cur_id == sink_node: - break - for child_id in list(new_residual.successors(cur_id)): - if ( - child_id not in s_set - and abs(new_residual[cur_id][child_id]["flow"]) > FP_ERROR - ): - q.append(child_id) - t_set = set(new_residual.nodes) - s_set - - profiling_min_cut_after_max_flows = ( - time.time() - profiling_min_cut_after_max_flows - ) - logger.info( - "PROFILING PhillipsDessouky::find_min_cut after max flows time: %.10fs", - profiling_min_cut_after_max_flows, - ) - - return s_set, t_set - - def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: - """In-place annotate the critical DAG with flow capacities.""" - # XXX(JW): Is this always large enough? - # It is necessary to monitor the `cost_change` value in `IterationResult` - # and make sure they are way smaller than this value. Even if the cost - # change is close or larger than this, users can scale down their cost - # value in `ExecutionOption`s. - inf = 10000.0 - for _, _, edge_attr in critical_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - duration = op.duration - # Dummy operations don't constrain the flow. - if op.is_dummy: # noqa: SIM114 - lb, ub = 0.0, inf - # Cannot be sped up or down. - elif duration == op.min_duration == op.max_duration: - lb, ub = 0.0, inf - # Cannot be sped up. - elif duration - 1 < op.min_duration: - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = inf - # Cannot be slowed down. - elif duration + 1 > op.max_duration: - lb = 0.0 - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) - else: - # In case the cost model is almost linear, give this edge some room. - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR - - # XXX(JW): This roundiing may not be necessary. - edge_attr["lb"] = lb // FP_ERROR * FP_ERROR - edge_attr["ub"] = ub // FP_ERROR * FP_ERROR diff --git a/lowtime/solver.1.find_min_cut.up_to_second.py b/lowtime/solver.1.find_min_cut.up_to_second.py deleted file mode 100644 index 0cece09..0000000 --- a/lowtime/solver.1.find_min_cut.up_to_second.py +++ /dev/null @@ -1,759 +0,0 @@ -# Copyright (C) 2023 Jae-Won Chung -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" - - -from __future__ import annotations - -import time -import sys -import logging -from collections import deque -from collections.abc import Generator - -import networkx as nx -from attrs import define, field - -from lowtime.operation import Operation -from lowtime.graph_utils import ( - aon_dag_to_aoa_dag, - aoa_to_critical_dag, - get_critical_aoa_dag_total_time, - get_total_cost, -) -from lowtime.exceptions import LowtimeFlowError -from lowtime import _lowtime_rs - -FP_ERROR = 1e-6 - -logger = logging.getLogger(__name__) - - -@define -class IterationResult: - """Holds results after one PD iteration. - - Attributes: - iteration: The number of optimization iterations experienced by the DAG. - cost_change: The increase in cost from reducing the DAG's - quantized execution time by 1. - cost: The current total cost of the DAG. - quant_time: The current quantized total execution time of the DAG. - unit_time: The unit time used for time quantization. - real_time: The current real (de-quantized) total execution time of the DAG. - """ - - iteration: int - cost_change: float - cost: float - quant_time: int - unit_time: float - real_time: float = field(init=False) - - def __attrs_post_init__(self) -> None: - """Set `real_time` after initialization.""" - self.real_time = self.quant_time * self.unit_time - - -class PhillipsDessouky: - """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" - - def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: - """Initialize the Phillips-Dessouky solver. - - Assumptions: - - The graph is a Directed Acyclic Graph (DAG). - - Operations are annotated on nodes with name `attr_name`. - - There is only one source (entry) node. The source node is annotated as - `dag.graph["source_node"]`. - - There is only one sink (exit) node. The sink node is annotated as - `dag.graph["sink_node"]`. - - The `unit_time` attribute of every operation is the same. - - Args: - dag: A networkx DiGraph object that represents the computation DAG. - The aforementioned assumptions should hold for the DAG. - attr_name: The name of the attribute on nodes that holds the operation - object. Defaults to "op". - """ - self.attr_name = attr_name - - # Run checks on the DAG and cache some properties. - # Check: It's a DAG. - if not nx.is_directed_acyclic_graph(dag): - raise ValueError("The graph should be a Directed Acyclic Graph.") - - # Check: Only one source node that matches annotation. - if (source_node := dag.graph.get("source_node")) is None: - raise ValueError("The graph should have a `source_node` attribute.") - source_node_candidates = [] - for node_id, in_degree in dag.in_degree(): - if in_degree == 0: - source_node_candidates.append(node_id) - if len(source_node_candidates) == 0: - raise ValueError( - "Found zero nodes with in-degree 0. Cannot determine source node." - ) - if len(source_node_candidates) > 1: - raise ValueError( - f"Expecting only one source node, found {source_node_candidates}." - ) - if (detected_source_node := source_node_candidates[0]) != source_node: - raise ValueError( - f"Detected source node ({detected_source_node}) does not match " - f"the annotated source node ({source_node})." - ) - - # Check: Only one sink node that matches annotation. - if (sink_node := dag.graph.get("sink_node")) is None: - raise ValueError("The graph should have a `sink_node` attribute.") - sink_node_candidates = [] - for node_id, out_degree in dag.out_degree(): - if out_degree == 0: - sink_node_candidates.append(node_id) - if len(sink_node_candidates) == 0: - raise ValueError( - "Found zero nodes with out-degree 0. Cannot determine sink node." - ) - if len(sink_node_candidates) > 1: - raise ValueError( - f"Expecting only one sink node, found {sink_node_candidates}." - ) - if (detected_sink_node := sink_node_candidates[0]) != sink_node: - raise ValueError( - f"Detected sink node ({detected_sink_node}) does not match " - f"the annotated sink node ({sink_node})." - ) - - # Check: The `unit_time` attributes of every operation should be the same. - unit_time_candidates = set[float]() - for _, node_attr in dag.nodes(data=True): - if self.attr_name in node_attr: - op: Operation = node_attr[self.attr_name] - if op.is_dummy: - continue - unit_time_candidates.update( - option.unit_time for option in op.spec.options.options - ) - if len(unit_time_candidates) == 0: - raise ValueError( - "Found zero operations in the graph. Make sure you " - f"added operations as node attributes with key `{self.attr_name}`.", - ) - if len(unit_time_candidates) > 1: - raise ValueError( - f"Expecting the same `unit_time` across all operations, " - f"found {unit_time_candidates}." - ) - - self.aon_dag = dag - self.unit_time = unit_time_candidates.pop() - - def run(self) -> Generator[IterationResult, None, None]: - """Run the algorithm and yield a DAG after each iteration. - - The solver will not deepcopy operations on the DAG but rather in-place modify - them for speed. The caller should deepcopy the operations or the DAG if needed - before running the next iteration. - - Upon yield, it is guaranteed that the earliest/latest start/finish time values - of all operations are up to date w.r.t. the `duration` of each operation. - """ - logger.info("Starting Phillips-Dessouky solver.") - profiling_setup = time.time() - - # Convert the original activity-on-node DAG to activity-on-arc DAG form. - # AOA DAGs are purely internal. All public input and output of this class - # should be in AON form. - aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) - - # Estimate the minimum execution time of the DAG by setting every operation - # to run at its minimum duration. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.min_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - min_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected minimum quantized execution time: %d", min_time) - - # Estimated the maximum execution time of the DAG by setting every operation - # to run at its maximum duration. This is also our initial start point. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.max_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - max_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected maximum quantized execution time: %d", max_time) - - num_iters = max_time - min_time + 1 - logger.info("Expected number of PD iterations: %d", num_iters) - - profiling_setup = time.time() - profiling_setup - logger.info( - "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup - ) - - # Iteratively reduce the execution time of the DAG. - for iteration in range(sys.maxsize): - logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) - profiling_iter = time.time() - - # At this point, `critical_dag` always exists and is what we want. - # For the first iteration, the critical DAG is computed before the for - # loop in order to estimate the number of iterations. For subsequent - # iterations, the critcal DAG is computed after each iteration in - # in order to construct `IterationResult`. - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Critical DAG:") - logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) - logger.debug("Number of edges: %d", critical_dag.number_of_edges()) - non_dummy_ops = [ - attr[self.attr_name] - for _, _, attr in critical_dag.edges(data=True) - if not attr[self.attr_name].is_dummy - ] - logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) - logger.debug( - "Sum of non-dummy durations: %d", - sum(op.duration for op in non_dummy_ops), - ) - - profiling_annotate = time.time() - self.annotate_capacities(critical_dag) - profiling_annotate = time.time() - profiling_annotate - logger.info( - "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", - profiling_annotate, - ) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Capacity DAG:") - logger.debug( - "Total lb value: %f", - sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), - ) - logger.debug( - "Total ub value: %f", - sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), - ) - - try: - profiling_min_cut = time.time() - s_set, t_set = self.find_min_cut(critical_dag) - profiling_min_cut = time.time() - profiling_min_cut - logger.info( - "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", - profiling_min_cut, - ) - except LowtimeFlowError as e: - logger.info("Could not find minimum cut: %s", e.message) - logger.info("Terminating PD iteration.") - break - - profiling_reduce = time.time() - cost_change = self.reduce_durations(critical_dag, s_set, t_set) - profiling_reduce = time.time() - profiling_reduce - logger.info( - "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", - profiling_reduce, - ) - - if cost_change == float("inf") or abs(cost_change) < FP_ERROR: - logger.info("No further time reduction possible.") - logger.info("Terminating PD iteration.") - break - - # Earliest/latest start/finish times on operations also annotated here. - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - - # We directly modify operation attributes in the DAG, so after we - # ran one iteration, the AON DAG holds updated attributes. - result = IterationResult( - iteration=iteration + 1, - cost_change=cost_change, - cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), - quant_time=get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ), - unit_time=self.unit_time, - ) - logger.info("%s", result) - profiling_iter = time.time() - profiling_iter - logger.info( - "PROFILING PhillipsDessouky::run single iteration time: %.10fs", - profiling_iter, - ) - yield result - - def reduce_durations( - self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] - ) -> float: - """Modify operation durations to reduce the DAG's execution time by 1.""" - speed_up_edges: list[Operation] = [] - for node_id in s_set: - for child_id in list(dag.successors(node_id)): - if child_id in t_set: - op: Operation = dag[node_id][child_id][self.attr_name] - speed_up_edges.append(op) - - slow_down_edges: list[Operation] = [] - for node_id in t_set: - for child_id in list(dag.successors(node_id)): - if child_id in s_set: - op: Operation = dag[node_id][child_id][self.attr_name] - slow_down_edges.append(op) - - if not speed_up_edges: - logger.info("No speed up candidate operations.") - return 0.0 - - cost_change = 0.0 - - # Reduce the duration of edges (speed up) by quant_time 1. - for op in speed_up_edges: - if op.is_dummy: - logger.info("Cannot speed up dummy operation.") - return float("inf") - if op.duration - 1 < op.min_duration: - logger.info("Operation %s has reached the limit of speed up", op) - return float("inf") - cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) - op_before_str = str(op) - op.duration -= 1 - logger.info("Sped up %s to %s", op_before_str, op) - - # Increase the duration of edges (slow down) by quant_time 1. - for op in slow_down_edges: - # Dummy edges can always be slowed down. - if op.is_dummy: - logger.info("Slowed down DummyOperation (didn't really slowdown).") - continue - elif op.duration + 1 > op.max_duration: - logger.info("Operation %s has reached the limit of slow down", op) - return float("inf") - cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) - before_op_str = str(op) - op.duration += 1 - logger.info("Slowed down %s to %s", before_op_str, op) - - return cost_change - - def find_min_cut(self, dag: nx.DiGraph) -> tuple[set[int], set[int]]: - """Find the min cut of the DAG annotated with lower/upper bound flow capacities. - - Assumptions: - - The capacity DAG is in AOA form. - - The capacity DAG has been annotated with `lb` and `ub` attributes on edges, - representing the lower and upper bounds of the flow on the edge. - - Returns: - A tuple of (s_set, t_set) where s_set is the set of nodes on the source side - of the min cut and t_set is the set of nodes on the sink side of the min cut. - Returns None if no feasible flow exists. - - Raises: - LowtimeFlowError: When no feasible flow exists. - """ - # Helper function for Rust interop - def format_rust_inputs( - dag: nx.DiGraph, - ) -> tuple[ - nx.classes.reportviews.NodeView, - list[ - tuple[ - tuple[int, int], - tuple[float, float, float, float], - tuple[bool, int, int, int, int, int, int, int] | None, - ] - ], - ]: - nodes = dag.nodes - edges = [] - for from_, to_, edge_attrs in dag.edges(data=True): - op_details = ( - None - if self.attr_name not in edge_attrs - else ( - edge_attrs[self.attr_name].is_dummy, - edge_attrs[self.attr_name].duration, - edge_attrs[self.attr_name].max_duration, - edge_attrs[self.attr_name].min_duration, - edge_attrs[self.attr_name].earliest_start, - edge_attrs[self.attr_name].latest_start, - edge_attrs[self.attr_name].earliest_finish, - edge_attrs[self.attr_name].latest_finish, - ) - ) - edges.append( - ( - (from_, to_), - ( - edge_attrs.get("capacity", 0), - edge_attrs.get("flow", 0), - edge_attrs.get("ub", 0), - edge_attrs.get("lb", 0), - ), - op_details, - ) - ) - return nodes, edges - - ########### NEW ########### - # Construct Rust runner with input (critical) dag - ohjun_nodes, ohjun_edges = format_rust_inputs(dag) - # print(ohjun_nodes) - # print(ohjun_edges) - ohjun_rust_runner = _lowtime_rs.PhillipsDessouky( - FP_ERROR, - ohjun_nodes, - dag.graph["source_node"], - dag.graph["sink_node"], - ohjun_edges - ) - ########### NEW ########### - - profiling_min_cut_setup = time.time() - source_node = dag.graph["source_node"] - sink_node = dag.graph["sink_node"] - - # In order to solve max flow on edges with both lower and upper bounds, - # we first need to convert it to another DAG that only has upper bounds. - unbound_dag = nx.DiGraph(dag) - - # For every edge, capacity = ub - lb. - for _, _, edge_attrs in unbound_dag.edges(data=True): - edge_attrs["capacity"] = edge_attrs["ub"] - edge_attrs["lb"] - - # Add a new node s', which will become the new source node. - # We constructed the AOA DAG, so we know that node IDs are integers. - node_ids: list[int] = list(unbound_dag.nodes) - s_prime_id = max(node_ids) + 1 - unbound_dag.add_node(s_prime_id) - - # For every node u in the original graph, add an edge (s', u) with capacity - # equal to the sum of all lower bounds of u's parents. - for u in dag.nodes: - capacity = 0.0 - for pred_id in dag.predecessors(u): - capacity += dag[pred_id][u]["lb"] - unbound_dag.add_edge(s_prime_id, u, capacity=capacity) - - # Add a new node t', which will become the new sink node. - t_prime_id = s_prime_id + 1 - unbound_dag.add_node(t_prime_id) - - # For every node u in the original graph, add an edge (u, t') with capacity - # equal to the sum of all lower bounds of u's children. - for u in dag.nodes: - capacity = 0.0 - for succ_id in dag.successors(u): - capacity += dag[u][succ_id]["lb"] - unbound_dag.add_edge(u, t_prime_id, capacity=capacity) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Unbound DAG") - logger.debug("Number of nodes: %d", unbound_dag.number_of_nodes()) - logger.debug("Number of edges: %d", unbound_dag.number_of_edges()) - logger.debug( - "Sum of capacities: %f", - sum(attr["capacity"] for _, _, attr in unbound_dag.edges(data=True)), - ) - - # Add an edge from t to s with infinite capacity. - unbound_dag.add_edge( - sink_node, - source_node, - capacity=float("inf"), - ) - - profiling_min_cut_setup = time.time() - profiling_min_cut_setup - logger.info( - "PROFILING PhillipsDessouky::find_min_cut setup time: %.10fs", - profiling_min_cut_setup, - ) - - # Helper function for Rust interop - # Rust's pathfinding::edmonds_karp does not return edges with 0 flow, - # but nx.max_flow does. So we fill in the 0s and empty nodes. - def reformat_rust_flow_to_dict( - flow_vec: list[tuple[tuple[int, int], float]], dag: nx.DiGraph - ) -> dict[int, dict[int, float]]: - flow_dict = dict() - for u in dag.nodes: - flow_dict[u] = dict() - for v in dag.successors(u): - flow_dict[u][v] = 0.0 - - for (u, v), cap in flow_vec: - flow_dict[u][v] = cap - - return flow_dict - - # We're done with constructing the DAG with only flow upper bounds. - # Find the maximum flow on this DAG. - profiling_max_flow = time.time() - nodes, edges = format_rust_inputs(unbound_dag) - - profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, s_prime_id, t_prime_id, edges) - profiling_data_transfer = time.time() - profiling_data_transfer - logger.info( - "PROFILING PhillipsDessouky::find_min_cut data transfer time: %.10fs", - profiling_data_transfer, - ) - - # ohjun: FIRST MAX FLOW - rust_flow_vec = rust_dag.max_flow_depr() - flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) - - profiling_max_flow = time.time() - profiling_max_flow - logger.info( - "PROFILING PhillipsDessouky::find_min_cut maximum_flow_1 time: %.10fs", - profiling_max_flow, - ) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("After first max flow") - total_flow = 0.0 - for d in flow_dict.values(): - for flow in d.values(): - total_flow += flow - logger.debug("Sum of all flow values: %f", total_flow) - - profiling_min_cut_between_max_flows = time.time() - - # Check if residual graph is saturated. If so, we have a feasible flow. - for u in unbound_dag.successors(s_prime_id): - if ( - abs(flow_dict[s_prime_id][u] - unbound_dag[s_prime_id][u]["capacity"]) - > FP_ERROR - ): - logger.error( - "s' -> %s unsaturated (flow: %s, capacity: %s)", - u, - flow_dict[s_prime_id][u], - unbound_dag[s_prime_id][u]["capacity"], - ) - raise LowtimeFlowError( - "ERROR: Max flow on unbounded DAG didn't saturate." - ) - for u in unbound_dag.predecessors(t_prime_id): - if ( - abs(flow_dict[u][t_prime_id] - unbound_dag[u][t_prime_id]["capacity"]) - > FP_ERROR - ): - logger.error( - "%s -> t' unsaturated (flow: %s, capacity: %s)", - u, - flow_dict[u][t_prime_id], - unbound_dag[u][t_prime_id]["capacity"], - ) - raise LowtimeFlowError( - "ERROR: Max flow on unbounded DAG didn't saturate." - ) - - # We have a feasible flow. Construct a new residual graph with the same - # shape as the capacity DAG so that we can find the min cut. - # First, retrieve the flow amounts to the original capacity graph, where for - # each edge u -> v, the flow amount is `flow + lb`. - for u, v in dag.edges: - dag[u][v]["flow"] = flow_dict[u][v] + dag[u][v]["lb"] - - # Construct a new residual graph (same shape as capacity DAG) with - # u -> v capacity `ub - flow` and v -> u capacity `flow - lb`. - residual_graph = nx.DiGraph(dag) - for u, v in dag.edges: - # Rounding small negative values to 0.0 avoids Rust-side - # pathfinding::edmonds_karp from entering unreachable code. - # This edge case did not exist in Python-side nx.max_flow call. - uv_capacity = residual_graph[u][v]["ub"] - residual_graph[u][v]["flow"] - uv_capacity = 0.0 if abs(uv_capacity) < FP_ERROR else uv_capacity - residual_graph[u][v]["capacity"] = uv_capacity - - vu_capacity = residual_graph[u][v]["flow"] - residual_graph[u][v]["lb"] - vu_capacity = 0.0 if abs(vu_capacity) < FP_ERROR else vu_capacity - if dag.has_edge(v, u): - residual_graph[v][u]["capacity"] = vu_capacity - else: - residual_graph.add_edge(v, u, capacity=vu_capacity) - - profiling_min_cut_between_max_flows = ( - time.time() - profiling_min_cut_between_max_flows - ) - logger.info( - "PROFILING PhillipsDessouky::find_min_cut between max flows time: %.10fs", - profiling_min_cut_between_max_flows, - ) - - # Run max flow on the new residual graph. - profiling_max_flow = time.time() - nodes, edges = format_rust_inputs(residual_graph) - - profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(FP_ERROR, nodes, source_node, sink_node, edges) - profiling_data_transfer = time.time() - profiling_data_transfer - logger.info( - "PROFILING PhillipsDessouky::find_min_cut data transfer 2 time: %.10fs", - profiling_data_transfer, - ) - - # ohjun: SECOND MAX FLOW - rust_flow_vec = rust_dag.max_flow_depr() - flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) - - ############# NEW ############# - ##### TESTING print all edges and capacities - # logger.info(f"s_prime_id: {s_prime_id}") - # logger.info(f"t_prime_id: {t_prime_id}") - # logger.info("Python Printing graph:") - # logger.info(f"Num edges: {len(unbound_dag.edges())}") - # for from_, to_, edge_attrs in unbound_dag.edges(data=True): - # logger.info(f"{from_} -> {to_}: {edge_attrs["capacity"]}") - - # TEMP(ohjun): in current wip version, do everything until after 2nd max flow in Rust - ohjun_rust_flow_vec = ohjun_rust_runner.find_min_cut_wip() - ohjun_flow_dict = reformat_rust_flow_to_dict(ohjun_rust_flow_vec, residual_graph) - - depr_node_ids = rust_dag.get_dag_node_ids() - depr_edges = rust_dag.get_dag_ek_processed_edges() - new_node_ids = ohjun_rust_runner.get_residual_graph_node_ids() - new_edges = ohjun_rust_runner.get_residual_graph_ek_processed_edges() - - # print(f"depr_node_ids: {depr_node_ids}") - # print(f"new_node_ids: {new_node_ids}") - assert len(depr_node_ids) == len(new_node_ids), "LENGTH MISMATCH in node_ids" - assert depr_node_ids == new_node_ids, "DIFF in node_ids" - assert len(depr_edges) == len(new_edges), "LENGTH MISMATCH in edges" - # if sorted(depr_edges) != sorted(new_edges): - # for depr_edge, new_edge in zip(sorted(depr_edges), sorted(new_edges)): - # if depr_edge == new_edge: - # logger.info("edges EQUAL") - # logger.info(f"depr_edge: {depr_edge}") - # logger.info(f"new_edge : {new_edge}") - # else: - # logger.info("edges DIFFERENT") - # logger.info(f"depr_edge: {depr_edge}") - # logger.info(f"new_edge : {new_edge}") - assert sorted(depr_edges) == sorted(new_edges), "DIFF in edges" - - # def print_dict(d): - # for from_, inner in d.items(): - # for to_, flow in inner.items(): - # logger.info(f'{from_} -> {to_}: {flow}') - # logger.info('flow_dict:') - # print_dict(flow_dict) - # logger.info('ohjun_flow_dict:') - # print_dict(ohjun_flow_dict) - assert flow_dict == ohjun_flow_dict, "flow dicts were different :(" - ############# NEW ############# - - profiling_max_flow = time.time() - profiling_max_flow - logger.info( - "PROFILING PhillipsDessouky::find_min_cut maximum_flow_2 time: %.10fs", - profiling_max_flow, - ) - - profiling_min_cut_after_max_flows = time.time() - - # Add additional flow we get to the original graph - for u, v in dag.edges: - dag[u][v]["flow"] += flow_dict[u][v] - dag[u][v]["flow"] -= flow_dict[v][u] - - # Construct the new residual graph. - new_residual = nx.DiGraph(dag) - for u, v in dag.edges: - new_residual[u][v]["flow"] = dag[u][v]["ub"] - dag[u][v]["flow"] - new_residual.add_edge(v, u, flow=dag[u][v]["flow"] - dag[u][v]["lb"]) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("New residual graph") - logger.debug("Number of nodes: %d", new_residual.number_of_nodes()) - logger.debug("Number of edges: %d", new_residual.number_of_edges()) - logger.debug( - "Sum of flow: %f", - sum(attr["flow"] for _, _, attr in new_residual.edges(data=True)), - ) - - # Find the s-t cut induced by the second maximum flow above. - # Only following `flow > 0` edges, find the set of nodes reachable from - # source node. That's the s-set, and the rest is the t-set. - s_set = set[int]() - q: deque[int] = deque() - q.append(source_node) - while q: - cur_id = q.pop() - s_set.add(cur_id) - if cur_id == sink_node: - break - for child_id in list(new_residual.successors(cur_id)): - if ( - child_id not in s_set - and abs(new_residual[cur_id][child_id]["flow"]) > FP_ERROR - ): - q.append(child_id) - t_set = set(new_residual.nodes) - s_set - - profiling_min_cut_after_max_flows = ( - time.time() - profiling_min_cut_after_max_flows - ) - logger.info( - "PROFILING PhillipsDessouky::find_min_cut after max flows time: %.10fs", - profiling_min_cut_after_max_flows, - ) - - return s_set, t_set - - def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: - """In-place annotate the critical DAG with flow capacities.""" - # XXX(JW): Is this always large enough? - # It is necessary to monitor the `cost_change` value in `IterationResult` - # and make sure they are way smaller than this value. Even if the cost - # change is close or larger than this, users can scale down their cost - # value in `ExecutionOption`s. - inf = 10000.0 - for _, _, edge_attr in critical_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - duration = op.duration - # Dummy operations don't constrain the flow. - if op.is_dummy: # noqa: SIM114 - lb, ub = 0.0, inf - # Cannot be sped up or down. - elif duration == op.min_duration == op.max_duration: - lb, ub = 0.0, inf - # Cannot be sped up. - elif duration - 1 < op.min_duration: - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = inf - # Cannot be slowed down. - elif duration + 1 > op.max_duration: - lb = 0.0 - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) - else: - # In case the cost model is almost linear, give this edge some room. - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR - - # XXX(JW): This roundiing may not be necessary. - edge_attr["lb"] = lb // FP_ERROR * FP_ERROR - edge_attr["ub"] = ub // FP_ERROR * FP_ERROR diff --git a/lowtime/solver.orig.py b/lowtime/solver.orig.py deleted file mode 100644 index 2bcefef..0000000 --- a/lowtime/solver.orig.py +++ /dev/null @@ -1,705 +0,0 @@ -# Copyright (C) 2023 Jae-Won Chung -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A time-cost trade-off solver based on the Phillips-Dessouky algorithm.""" - - -from __future__ import annotations - -import time -import sys -import logging -from collections import deque -from collections.abc import Generator - -import networkx as nx -from attrs import define, field - -from lowtime.operation import Operation -from lowtime.graph_utils import ( - aon_dag_to_aoa_dag, - aoa_to_critical_dag, - get_critical_aoa_dag_total_time, - get_total_cost, -) -from lowtime.exceptions import LowtimeFlowError -from lowtime import _lowtime_rs - -FP_ERROR = 1e-6 - -logger = logging.getLogger(__name__) - - -@define -class IterationResult: - """Holds results after one PD iteration. - - Attributes: - iteration: The number of optimization iterations experienced by the DAG. - cost_change: The increase in cost from reducing the DAG's - quantized execution time by 1. - cost: The current total cost of the DAG. - quant_time: The current quantized total execution time of the DAG. - unit_time: The unit time used for time quantization. - real_time: The current real (de-quantized) total execution time of the DAG. - """ - - iteration: int - cost_change: float - cost: float - quant_time: int - unit_time: float - real_time: float = field(init=False) - - def __attrs_post_init__(self) -> None: - """Set `real_time` after initialization.""" - self.real_time = self.quant_time * self.unit_time - - -class PhillipsDessouky: - """Implements the Phillips-Dessouky algorithm for the time-cost tradeoff problem.""" - - def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: - """Initialize the Phillips-Dessouky solver. - - Assumptions: - - The graph is a Directed Acyclic Graph (DAG). - - Operations are annotated on nodes with name `attr_name`. - - There is only one source (entry) node. The source node is annotated as - `dag.graph["source_node"]`. - - There is only one sink (exit) node. The sink node is annotated as - `dag.graph["sink_node"]`. - - The `unit_time` attribute of every operation is the same. - - Args: - dag: A networkx DiGraph object that represents the computation DAG. - The aforementioned assumptions should hold for the DAG. - attr_name: The name of the attribute on nodes that holds the operation - object. Defaults to "op". - """ - self.attr_name = attr_name - - # Run checks on the DAG and cache some properties. - # Check: It's a DAG. - if not nx.is_directed_acyclic_graph(dag): - raise ValueError("The graph should be a Directed Acyclic Graph.") - - # Check: Only one source node that matches annotation. - if (source_node := dag.graph.get("source_node")) is None: - raise ValueError("The graph should have a `source_node` attribute.") - source_node_candidates = [] - for node_id, in_degree in dag.in_degree(): - if in_degree == 0: - source_node_candidates.append(node_id) - if len(source_node_candidates) == 0: - raise ValueError( - "Found zero nodes with in-degree 0. Cannot determine source node." - ) - if len(source_node_candidates) > 1: - raise ValueError( - f"Expecting only one source node, found {source_node_candidates}." - ) - if (detected_source_node := source_node_candidates[0]) != source_node: - raise ValueError( - f"Detected source node ({detected_source_node}) does not match " - f"the annotated source node ({source_node})." - ) - - # Check: Only one sink node that matches annotation. - if (sink_node := dag.graph.get("sink_node")) is None: - raise ValueError("The graph should have a `sink_node` attribute.") - sink_node_candidates = [] - for node_id, out_degree in dag.out_degree(): - if out_degree == 0: - sink_node_candidates.append(node_id) - if len(sink_node_candidates) == 0: - raise ValueError( - "Found zero nodes with out-degree 0. Cannot determine sink node." - ) - if len(sink_node_candidates) > 1: - raise ValueError( - f"Expecting only one sink node, found {sink_node_candidates}." - ) - if (detected_sink_node := sink_node_candidates[0]) != sink_node: - raise ValueError( - f"Detected sink node ({detected_sink_node}) does not match " - f"the annotated sink node ({sink_node})." - ) - - # Check: The `unit_time` attributes of every operation should be the same. - unit_time_candidates = set[float]() - for _, node_attr in dag.nodes(data=True): - if self.attr_name in node_attr: - op: Operation = node_attr[self.attr_name] - if op.is_dummy: - continue - unit_time_candidates.update( - option.unit_time for option in op.spec.options.options - ) - if len(unit_time_candidates) == 0: - raise ValueError( - "Found zero operations in the graph. Make sure you " - f"added operations as node attributes with key `{self.attr_name}`.", - ) - if len(unit_time_candidates) > 1: - raise ValueError( - f"Expecting the same `unit_time` across all operations, " - f"found {unit_time_candidates}." - ) - - self.aon_dag = dag - self.unit_time = unit_time_candidates.pop() - - def run(self) -> Generator[IterationResult, None, None]: - """Run the algorithm and yield a DAG after each iteration. - - The solver will not deepcopy operations on the DAG but rather in-place modify - them for speed. The caller should deepcopy the operations or the DAG if needed - before running the next iteration. - - Upon yield, it is guaranteed that the earliest/latest start/finish time values - of all operations are up to date w.r.t. the `duration` of each operation. - """ - logger.info("Starting Phillips-Dessouky solver.") - profiling_setup = time.time() - - # Convert the original activity-on-node DAG to activity-on-arc DAG form. - # AOA DAGs are purely internal. All public input and output of this class - # should be in AON form. - aoa_dag = aon_dag_to_aoa_dag(self.aon_dag, attr_name=self.attr_name) - - # Estimate the minimum execution time of the DAG by setting every operation - # to run at its minimum duration. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.min_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - min_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected minimum quantized execution time: %d", min_time) - - # Estimated the maximum execution time of the DAG by setting every operation - # to run at its maximum duration. This is also our initial start point. - for _, _, edge_attr in aoa_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - if op.is_dummy: - continue - op.duration = op.max_duration - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - max_time = get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ) - logger.info("Expected maximum quantized execution time: %d", max_time) - - num_iters = max_time - min_time + 1 - logger.info("Expected number of PD iterations: %d", num_iters) - - profiling_setup = time.time() - profiling_setup - logger.info( - "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup - ) - - # Iteratively reduce the execution time of the DAG. - for iteration in range(sys.maxsize): - logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) - profiling_iter = time.time() - - # At this point, `critical_dag` always exists and is what we want. - # For the first iteration, the critical DAG is computed before the for - # loop in order to estimate the number of iterations. For subsequent - # iterations, the critcal DAG is computed after each iteration in - # in order to construct `IterationResult`. - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Critical DAG:") - logger.debug("Number of nodes: %d", critical_dag.number_of_nodes()) - logger.debug("Number of edges: %d", critical_dag.number_of_edges()) - non_dummy_ops = [ - attr[self.attr_name] - for _, _, attr in critical_dag.edges(data=True) - if not attr[self.attr_name].is_dummy - ] - logger.debug("Number of non-dummy operations: %d", len(non_dummy_ops)) - logger.debug( - "Sum of non-dummy durations: %d", - sum(op.duration for op in non_dummy_ops), - ) - - profiling_annotate = time.time() - self.annotate_capacities(critical_dag) - profiling_annotate = time.time() - profiling_annotate - logger.info( - "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", - profiling_annotate, - ) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Capacity DAG:") - logger.debug( - "Total lb value: %f", - sum([critical_dag[u][v]["lb"] for u, v in critical_dag.edges]), - ) - logger.debug( - "Total ub value: %f", - sum([critical_dag[u][v]["ub"] for u, v in critical_dag.edges]), - ) - - try: - profiling_min_cut = time.time() - s_set, t_set = self.find_min_cut(critical_dag) - profiling_min_cut = time.time() - profiling_min_cut - logger.info( - "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", - profiling_min_cut, - ) - except LowtimeFlowError as e: - logger.info("Could not find minimum cut: %s", e.message) - logger.info("Terminating PD iteration.") - break - - profiling_reduce = time.time() - cost_change = self.reduce_durations(critical_dag, s_set, t_set) - profiling_reduce = time.time() - profiling_reduce - logger.info( - "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", - profiling_reduce, - ) - - if cost_change == float("inf") or abs(cost_change) < FP_ERROR: - logger.info("No further time reduction possible.") - logger.info("Terminating PD iteration.") - break - - # Earliest/latest start/finish times on operations also annotated here. - critical_dag = aoa_to_critical_dag(aoa_dag, attr_name=self.attr_name) - - # We directly modify operation attributes in the DAG, so after we - # ran one iteration, the AON DAG holds updated attributes. - result = IterationResult( - iteration=iteration + 1, - cost_change=cost_change, - cost=get_total_cost(aoa_dag, mode="edge", attr_name=self.attr_name), - quant_time=get_critical_aoa_dag_total_time( - critical_dag, attr_name=self.attr_name - ), - unit_time=self.unit_time, - ) - logger.info("%s", result) - profiling_iter = time.time() - profiling_iter - logger.info( - "PROFILING PhillipsDessouky::run single iteration time: %.10fs", - profiling_iter, - ) - yield result - - def reduce_durations( - self, dag: nx.DiGraph, s_set: set[int], t_set: set[int] - ) -> float: - """Modify operation durations to reduce the DAG's execution time by 1.""" - speed_up_edges: list[Operation] = [] - for node_id in s_set: - for child_id in list(dag.successors(node_id)): - if child_id in t_set: - op: Operation = dag[node_id][child_id][self.attr_name] - speed_up_edges.append(op) - - slow_down_edges: list[Operation] = [] - for node_id in t_set: - for child_id in list(dag.successors(node_id)): - if child_id in s_set: - op: Operation = dag[node_id][child_id][self.attr_name] - slow_down_edges.append(op) - - if not speed_up_edges: - logger.info("No speed up candidate operations.") - return 0.0 - - cost_change = 0.0 - - # Reduce the duration of edges (speed up) by quant_time 1. - for op in speed_up_edges: - if op.is_dummy: - logger.info("Cannot speed up dummy operation.") - return float("inf") - if op.duration - 1 < op.min_duration: - logger.info("Operation %s has reached the limit of speed up", op) - return float("inf") - cost_change += abs(op.get_cost(op.duration - 1) - op.get_cost(op.duration)) - op_before_str = str(op) - op.duration -= 1 - logger.info("Sped up %s to %s", op_before_str, op) - - # Increase the duration of edges (slow down) by quant_time 1. - for op in slow_down_edges: - # Dummy edges can always be slowed down. - if op.is_dummy: - logger.info("Slowed down DummyOperation (didn't really slowdown).") - continue - elif op.duration + 1 > op.max_duration: - logger.info("Operation %s has reached the limit of slow down", op) - return float("inf") - cost_change -= abs(op.get_cost(op.duration) - op.get_cost(op.duration + 1)) - before_op_str = str(op) - op.duration += 1 - logger.info("Slowed down %s to %s", before_op_str, op) - - return cost_change - - def find_min_cut(self, dag: nx.DiGraph) -> tuple[set[int], set[int]]: - """Find the min cut of the DAG annotated with lower/upper bound flow capacities. - - Assumptions: - - The capacity DAG is in AOA form. - - The capacity DAG has been annotated with `lb` and `ub` attributes on edges, - representing the lower and upper bounds of the flow on the edge. - - Returns: - A tuple of (s_set, t_set) where s_set is the set of nodes on the source side - of the min cut and t_set is the set of nodes on the sink side of the min cut. - Returns None if no feasible flow exists. - - Raises: - LowtimeFlowError: When no feasible flow exists. - """ - profiling_min_cut_setup = time.time() - source_node = dag.graph["source_node"] - sink_node = dag.graph["sink_node"] - - # In order to solve max flow on edges with both lower and upper bounds, - # we first need to convert it to another DAG that only has upper bounds. - unbound_dag = nx.DiGraph(dag) - - # For every edge, capacity = ub - lb. - for _, _, edge_attrs in unbound_dag.edges(data=True): - edge_attrs["capacity"] = edge_attrs["ub"] - edge_attrs["lb"] - - # Add a new node s', which will become the new source node. - # We constructed the AOA DAG, so we know that node IDs are integers. - node_ids: list[int] = list(unbound_dag.nodes) - s_prime_id = max(node_ids) + 1 - unbound_dag.add_node(s_prime_id) - - # For every node u in the original graph, add an edge (s', u) with capacity - # equal to the sum of all lower bounds of u's parents. - for u in dag.nodes: - capacity = 0.0 - for pred_id in dag.predecessors(u): - capacity += dag[pred_id][u]["lb"] - unbound_dag.add_edge(s_prime_id, u, capacity=capacity) - - # Add a new node t', which will become the new sink node. - t_prime_id = s_prime_id + 1 - unbound_dag.add_node(t_prime_id) - - # For every node u in the original graph, add an edge (u, t') with capacity - # equal to the sum of all lower bounds of u's children. - for u in dag.nodes: - capacity = 0.0 - for succ_id in dag.successors(u): - capacity += dag[u][succ_id]["lb"] - unbound_dag.add_edge(u, t_prime_id, capacity=capacity) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Unbound DAG") - logger.debug("Number of nodes: %d", unbound_dag.number_of_nodes()) - logger.debug("Number of edges: %d", unbound_dag.number_of_edges()) - logger.debug( - "Sum of capacities: %f", - sum(attr["capacity"] for _, _, attr in unbound_dag.edges(data=True)), - ) - - # Add an edge from t to s with infinite capacity. - unbound_dag.add_edge( - sink_node, - source_node, - capacity=float("inf"), - ) - - profiling_min_cut_setup = time.time() - profiling_min_cut_setup - logger.info( - "PROFILING PhillipsDessouky::find_min_cut setup time: %.10fs", - profiling_min_cut_setup, - ) - - # Helper function for Rust interop - def format_rust_inputs( - dag: nx.DiGraph, - ) -> tuple[ - nx.classes.reportviews.NodeView, - list[ - tuple[ - tuple[int, int], - tuple[float, float, float, float], - tuple[bool, int, int, int, int, int, int, int] | None, - ] - ], - ]: - nodes = dag.nodes - edges = [] - for from_, to_, edge_attrs in dag.edges(data=True): - op_details = ( - None - if self.attr_name not in edge_attrs - else ( - edge_attrs[self.attr_name].is_dummy, - edge_attrs[self.attr_name].duration, - edge_attrs[self.attr_name].max_duration, - edge_attrs[self.attr_name].min_duration, - edge_attrs[self.attr_name].earliest_start, - edge_attrs[self.attr_name].latest_start, - edge_attrs[self.attr_name].earliest_finish, - edge_attrs[self.attr_name].latest_finish, - ) - ) - edges.append( - ( - (from_, to_), - ( - edge_attrs["capacity"], - edge_attrs.get("flow", 0), - edge_attrs.get("ub", 0), - edge_attrs.get("lb", 0), - ), - op_details, - ) - ) - return nodes, edges - - # Helper function for Rust interop - # Rust's pathfinding::edmonds_karp does not return edges with 0 flow, - # but nx.max_flow does. So we fill in the 0s and empty nodes. - def reformat_rust_flow_to_dict( - flow_vec: list[tuple[tuple[int, int], float]], dag: nx.DiGraph - ) -> dict[int, dict[int, float]]: - flow_dict = dict() - for u in dag.nodes: - flow_dict[u] = dict() - for v in dag.successors(u): - flow_dict[u][v] = 0.0 - - for (u, v), cap in flow_vec: - flow_dict[u][v] = cap - - return flow_dict - - # We're done with constructing the DAG with only flow upper bounds. - # Find the maximum flow on this DAG. - profiling_max_flow = time.time() - nodes, edges = format_rust_inputs(unbound_dag) - - profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(nodes, s_prime_id, t_prime_id, edges) - profiling_data_transfer = time.time() - profiling_data_transfer - logger.info( - "PROFILING PhillipsDessouky::find_min_cut data transfer time: %.10fs", - profiling_data_transfer, - ) - - # ohjun: FIRST MAX FLOW - rust_flow_vec = rust_dag.max_flow_depr() - flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, unbound_dag) - def print_dict(d): - for from_, inner in d.items(): - for to_, flow in inner.items(): - logger.info(f'{from_} -> {to_}: {flow}') - logger.info('correct flow_dict:') - print_dict(flow_dict) - - profiling_max_flow = time.time() - profiling_max_flow - logger.info( - "PROFILING PhillipsDessouky::find_min_cut maximum_flow_1 time: %.10fs", - profiling_max_flow, - ) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("After first max flow") - total_flow = 0.0 - for d in flow_dict.values(): - for flow in d.values(): - total_flow += flow - logger.debug("Sum of all flow values: %f", total_flow) - - profiling_min_cut_between_max_flows = time.time() - - # Check if residual graph is saturated. If so, we have a feasible flow. - for u in unbound_dag.successors(s_prime_id): - if ( - abs(flow_dict[s_prime_id][u] - unbound_dag[s_prime_id][u]["capacity"]) - > FP_ERROR - ): - logger.error( - "s' -> %s unsaturated (flow: %s, capacity: %s)", - u, - flow_dict[s_prime_id][u], - unbound_dag[s_prime_id][u]["capacity"], - ) - raise LowtimeFlowError( - "ERROR: Max flow on unbounded DAG didn't saturate." - ) - for u in unbound_dag.predecessors(t_prime_id): - if ( - abs(flow_dict[u][t_prime_id] - unbound_dag[u][t_prime_id]["capacity"]) - > FP_ERROR - ): - logger.error( - "%s -> t' unsaturated (flow: %s, capacity: %s)", - u, - flow_dict[u][t_prime_id], - unbound_dag[u][t_prime_id]["capacity"], - ) - raise LowtimeFlowError( - "ERROR: Max flow on unbounded DAG didn't saturate." - ) - - # We have a feasible flow. Construct a new residual graph with the same - # shape as the capacity DAG so that we can find the min cut. - # First, retrieve the flow amounts to the original capacity graph, where for - # each edge u -> v, the flow amount is `flow + lb`. - for u, v in dag.edges: - dag[u][v]["flow"] = flow_dict[u][v] + dag[u][v]["lb"] - - # Construct a new residual graph (same shape as capacity DAG) with - # u -> v capacity `ub - flow` and v -> u capacity `flow - lb`. - residual_graph = nx.DiGraph(dag) - for u, v in dag.edges: - # Rounding small negative values to 0.0 avoids Rust-side - # pathfinding::edmonds_karp from entering unreachable code. - # This edge case did not exist in Python-side nx.max_flow call. - uv_capacity = residual_graph[u][v]["ub"] - residual_graph[u][v]["flow"] - uv_capacity = 0.0 if abs(uv_capacity) < FP_ERROR else uv_capacity - residual_graph[u][v]["capacity"] = uv_capacity - - vu_capacity = residual_graph[u][v]["flow"] - residual_graph[u][v]["lb"] - vu_capacity = 0.0 if abs(vu_capacity) < FP_ERROR else vu_capacity - if dag.has_edge(v, u): - residual_graph[v][u]["capacity"] = vu_capacity - else: - residual_graph.add_edge(v, u, capacity=vu_capacity) - - profiling_min_cut_between_max_flows = ( - time.time() - profiling_min_cut_between_max_flows - ) - logger.info( - "PROFILING PhillipsDessouky::find_min_cut between max flows time: %.10fs", - profiling_min_cut_between_max_flows, - ) - - # Run max flow on the new residual graph. - profiling_max_flow = time.time() - nodes, edges = format_rust_inputs(residual_graph) - - profiling_data_transfer = time.time() - rust_dag = _lowtime_rs.PhillipsDessouky(nodes, source_node, sink_node, edges) - profiling_data_transfer = time.time() - profiling_data_transfer - logger.info( - "PROFILING PhillipsDessouky::find_min_cut data transfer 2 time: %.10fs", - profiling_data_transfer, - ) - - # ohjun: SECOND MAX FLOW - rust_flow_vec = rust_dag.max_flow_depr() - flow_dict = reformat_rust_flow_to_dict(rust_flow_vec, residual_graph) - - profiling_max_flow = time.time() - profiling_max_flow - logger.info( - "PROFILING PhillipsDessouky::find_min_cut maximum_flow_2 time: %.10fs", - profiling_max_flow, - ) - - profiling_min_cut_after_max_flows = time.time() - - # Add additional flow we get to the original graph - for u, v in dag.edges: - dag[u][v]["flow"] += flow_dict[u][v] - dag[u][v]["flow"] -= flow_dict[v][u] - - # Construct the new residual graph. - new_residual = nx.DiGraph(dag) - for u, v in dag.edges: - new_residual[u][v]["flow"] = dag[u][v]["ub"] - dag[u][v]["flow"] - new_residual.add_edge(v, u, flow=dag[u][v]["flow"] - dag[u][v]["lb"]) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug("New residual graph") - logger.debug("Number of nodes: %d", new_residual.number_of_nodes()) - logger.debug("Number of edges: %d", new_residual.number_of_edges()) - logger.debug( - "Sum of flow: %f", - sum(attr["flow"] for _, _, attr in new_residual.edges(data=True)), - ) - - # Find the s-t cut induced by the second maximum flow above. - # Only following `flow > 0` edges, find the set of nodes reachable from - # source node. That's the s-set, and the rest is the t-set. - s_set = set[int]() - q: deque[int] = deque() - q.append(source_node) - while q: - cur_id = q.pop() - s_set.add(cur_id) - if cur_id == sink_node: - break - for child_id in list(new_residual.successors(cur_id)): - if ( - child_id not in s_set - and abs(new_residual[cur_id][child_id]["flow"]) > FP_ERROR - ): - q.append(child_id) - t_set = set(new_residual.nodes) - s_set - - profiling_min_cut_after_max_flows = ( - time.time() - profiling_min_cut_after_max_flows - ) - logger.info( - "PROFILING PhillipsDessouky::find_min_cut after max flows time: %.10fs", - profiling_min_cut_after_max_flows, - ) - - return s_set, t_set - - def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: - """In-place annotate the critical DAG with flow capacities.""" - # XXX(JW): Is this always large enough? - # It is necessary to monitor the `cost_change` value in `IterationResult` - # and make sure they are way smaller than this value. Even if the cost - # change is close or larger than this, users can scale down their cost - # value in `ExecutionOption`s. - inf = 10000.0 - for _, _, edge_attr in critical_dag.edges(data=True): - op: Operation = edge_attr[self.attr_name] - duration = op.duration - # Dummy operations don't constrain the flow. - if op.is_dummy: # noqa: SIM114 - lb, ub = 0.0, inf - # Cannot be sped up or down. - elif duration == op.min_duration == op.max_duration: - lb, ub = 0.0, inf - # Cannot be sped up. - elif duration - 1 < op.min_duration: - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = inf - # Cannot be slowed down. - elif duration + 1 > op.max_duration: - lb = 0.0 - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) - else: - # In case the cost model is almost linear, give this edge some room. - lb = abs(op.get_cost(duration) - op.get_cost(duration + 1)) - ub = abs(op.get_cost(duration - 1) - op.get_cost(duration)) + FP_ERROR - - # XXX(JW): This roundiing may not be necessary. - edge_attr["lb"] = lb // FP_ERROR * FP_ERROR - edge_attr["ub"] = ub // FP_ERROR * FP_ERROR diff --git a/src/not_used.cost_model.rs b/src/not_used.cost_model.rs deleted file mode 100644 index 2979a0f..0000000 --- a/src/not_used.cost_model.rs +++ /dev/null @@ -1,75 +0,0 @@ -use pyo3::prelude::*; - -use std::collections::HashMap; - - -pub trait ModelCost { - fn get_cost(&self, duration: u32) -> f64; -} - -#[pyclass] -pub struct CostModel { - inner: Box, - cache: HashMap, -} - -#[pymethods] -impl CostModel { - // Issues with traits of input type - // #[new] - // pub fn new(model: Box) -> PyResult { - // Ok(CostModel { - // inner: model, - // cache: HashMap::new(), - // }) - // } - - // Python functions cannot have generic type parameters - // #[new] - // pub fn new(model: C) -> PyResult - // where - // C: ModelCost + Send - // { - // Ok(CostModel { - // inner: Box::new(model), - // cache: HashMap::new(), - // }) - // } - - // Some error message about f64 not implementing some convoluted trait - // pub fn new_exponential(a: f64, b: f64, c: f64) -> PyResult { - // Ok(CostModel { - // inner: Box::new(ExponentialModel::new(a, b, c)), - // cache: HashMap::new(), - // }) - // } - - pub fn get_cost(&mut self, duration: u32) -> f64 { - match self.cache.get(&duration) { - Some(cost) => *cost, - None => { - let cost = self.inner.get_cost(duration); - self.cache.insert(duration, cost); - cost - } - } - } -} - -struct ExponentialModel { - a: f64, - b: f64, - c: f64, -} - -impl ExponentialModel { - fn new(a: f64, b: f64, c: f64) -> Self { - ExponentialModel {a, b, c} - } -} - -impl ModelCost for ExponentialModel { - fn get_cost(&self, duration: u32) -> f64 { - self.a * f64::exp(self.b * duration as f64) + self.c - } -} diff --git a/src/not_used.operation.rs b/src/not_used.operation.rs deleted file mode 100644 index fc6454a..0000000 --- a/src/not_used.operation.rs +++ /dev/null @@ -1,47 +0,0 @@ -// use crate::cost_model::CostModel; - - -#[derive(Clone)] -pub struct Operation { - is_dummy: bool, - pub duration: u64, - pub max_duration: u64, - pub min_duration: u64, - pub earliest_start: u64, - pub latest_start: u64, - pub earliest_finish: u64, - pub latest_finish: u64, -} - -impl Operation { - pub fn new( - is_dummy: bool, - duration: u64, - max_duration: u64, - min_duration: u64, - earliest_start: u64, - latest_start: u64, - earliest_finish: u64, - latest_finish: u64, - // cost_model: CostModel, - ) -> Self { - Operation { - is_dummy, - duration, - max_duration, - min_duration, - earliest_start, - latest_start, - earliest_finish, - latest_finish, - // cost_model, - } - } - - pub fn reset_times(&mut self) -> () { - self.earliest_start = 0; - self.latest_start = u64::MAX; - self.earliest_finish = 0; - self.latest_finish = u64::MAX; - } -} From c143e333879a47a7e6fa78684073284c66e2aff4 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Mon, 9 Dec 2024 13:35:19 -0500 Subject: [PATCH 33/55] update pyo3 version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 720e6b5..6c15a01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ name = "_lowtime_rs" crate-type = ["cdylib"] [dependencies] -pyo3 = "0.22.0" +pyo3 = "0.23.3" pyo3-log = "0.11.0" log = "0.4" ordered-float = { version = "4.0", default-features = false } From bbf932cb214a8324c2c3df25775b34eb221da938 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 10 Dec 2024 15:54:09 -0500 Subject: [PATCH 34/55] use prior pyo3 v --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6c15a01..6ee87ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ name = "_lowtime_rs" crate-type = ["cdylib"] [dependencies] -pyo3 = "0.23.3" +pyo3 = { version = "0.22.0", features = ["extension-module"] } pyo3-log = "0.11.0" log = "0.4" ordered-float = { version = "4.0", default-features = false } From c4e64e2027e7343b86fc11e32f01d5fe8f67f852 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 10 Dec 2024 15:57:08 -0500 Subject: [PATCH 35/55] black solver formatting --- lowtime/solver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lowtime/solver.py b/lowtime/solver.py index c3fb606..4769e97 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -294,7 +294,7 @@ def run(self) -> Generator[IterationResult, None, None]: nodes, critical_dag.graph["source_node"], critical_dag.graph["sink_node"], - edges + edges, ) s_set, t_set = rust_runner.find_min_cut() profiling_min_cut = time.time() - profiling_min_cut From 01887723d8c488309e388e3e705e0155074fa1bf Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 10 Dec 2024 15:58:30 -0500 Subject: [PATCH 36/55] fix pyi interface --- lowtime/_lowtime_rs.pyi | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lowtime/_lowtime_rs.pyi b/lowtime/_lowtime_rs.pyi index e8bbc2d..79a0c0e 100644 --- a/lowtime/_lowtime_rs.pyi +++ b/lowtime/_lowtime_rs.pyi @@ -16,6 +16,4 @@ class PhillipsDessouky: ] ], ) -> None: ... - def find_min_cut_wip(self) -> list[tuple[tuple[int, int], float]]: ... - -# TODO(ohjun): add CostModel interface + def find_min_cut(self) -> tuple[set[int], set[int]]: ... From b52c6d8497ed52f0668847174a9c84d268b3a114 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 10 Dec 2024 16:00:41 -0500 Subject: [PATCH 37/55] fix solver style --- lowtime/solver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lowtime/solver.py b/lowtime/solver.py index 4769e97..c965b64 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -20,7 +20,6 @@ import time import sys import logging -from collections import deque from collections.abc import Generator import networkx as nx @@ -161,7 +160,6 @@ def __init__(self, dag: nx.DiGraph, attr_name: str = "op") -> None: self.aon_dag = dag self.unit_time = unit_time_candidates.pop() - # Helper function for Rust interop def format_rust_inputs( self, dag: nx.DiGraph, @@ -174,6 +172,7 @@ def format_rust_inputs( ] ], ]: + """Convert Python-side nx.DiGraph into format compatible with Rust-side LowtimeGraph.""" nodes = dag.nodes edges = [] for from_, to_, edge_attrs in dag.edges(data=True): From 83d98732e0805b6a3a8dfefed8ca562d6d5d4c40 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 10 Dec 2024 16:07:08 -0500 Subject: [PATCH 38/55] fix pyi pyright --- lowtime/_lowtime_rs.pyi | 1 + 1 file changed, 1 insertion(+) diff --git a/lowtime/_lowtime_rs.pyi b/lowtime/_lowtime_rs.pyi index 79a0c0e..bd90be1 100644 --- a/lowtime/_lowtime_rs.pyi +++ b/lowtime/_lowtime_rs.pyi @@ -5,6 +5,7 @@ import networkx as nx class PhillipsDessouky: def __init__( self, + fp_error: float, node_ids: list[int] | nx.classes.reportviews.NodeView, source_node_id: int, sink_node_id: int, From d97295c8452db9b8db8127c1c36f79dfb9c2ce58 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 10 Dec 2024 16:09:02 -0500 Subject: [PATCH 39/55] actaully fix pyi pyright --- lowtime/_lowtime_rs.pyi | 1 - 1 file changed, 1 deletion(-) diff --git a/lowtime/_lowtime_rs.pyi b/lowtime/_lowtime_rs.pyi index bd90be1..7b19ab9 100644 --- a/lowtime/_lowtime_rs.pyi +++ b/lowtime/_lowtime_rs.pyi @@ -13,7 +13,6 @@ class PhillipsDessouky: tuple[ tuple[int, int], tuple[float, float, float, float], - tuple[bool, int, int, int, int, int, int, int] | None, ] ], ) -> None: ... From 1c646428f5419284f980d6fb07210852ceece511 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 10 Dec 2024 16:19:22 -0500 Subject: [PATCH 40/55] upgrade pyo3 --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6ee87ab..f143f97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,8 +9,8 @@ name = "_lowtime_rs" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.22.0", features = ["extension-module"] } -pyo3-log = "0.11.0" +pyo3 = { version = "0.23.3", features = ["extension-module"] } +pyo3-log = "0.12.0" log = "0.4" ordered-float = { version = "4.0", default-features = false } pathfinding = "4.11.0" From 03fc9fb8c3b47e648dbc64dd9b484a1287b28479 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 10 Dec 2024 16:51:49 -0500 Subject: [PATCH 41/55] try removing macos-12 from maturin.yml --- .github/workflows/maturin.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/maturin.yml b/.github/workflows/maturin.yml index ad3ec43..8385ada 100644 --- a/.github/workflows/maturin.yml +++ b/.github/workflows/maturin.yml @@ -117,8 +117,6 @@ jobs: strategy: matrix: platform: - - runner: macos-12 - target: x86_64 - runner: macos-14 target: aarch64 steps: From e4894441586eab5f451ba7b44c0e8cc3f1f58b52 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 10 Dec 2024 16:58:25 -0500 Subject: [PATCH 42/55] keep macos-12 x86_64 commented in yml --- .github/workflows/maturin.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/maturin.yml b/.github/workflows/maturin.yml index 8385ada..bd517bb 100644 --- a/.github/workflows/maturin.yml +++ b/.github/workflows/maturin.yml @@ -117,6 +117,8 @@ jobs: strategy: matrix: platform: + # - runner: macos-12 + # target: x86_64 - runner: macos-14 target: aarch64 steps: From 55d98da5325d50ec33b969aa382ad517897bf61a Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 10 Dec 2024 17:09:45 -0500 Subject: [PATCH 43/55] remove temporary profiling logs --- lowtime/solver.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/lowtime/solver.py b/lowtime/solver.py index c965b64..7f2f0ae 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -200,7 +200,6 @@ def run(self) -> Generator[IterationResult, None, None]: of all operations are up to date w.r.t. the `duration` of each operation. """ logger.info("Starting Phillips-Dessouky solver.") - profiling_setup = time.time() # Convert the original activity-on-node DAG to activity-on-arc DAG form. # AOA DAGs are purely internal. All public input and output of this class @@ -236,15 +235,9 @@ def run(self) -> Generator[IterationResult, None, None]: num_iters = max_time - min_time + 1 logger.info("Expected number of PD iterations: %d", num_iters) - profiling_setup = time.time() - profiling_setup - logger.info( - "PROFILING PhillipsDessouky::run set up time: %.10fs", profiling_setup - ) - # Iteratively reduce the execution time of the DAG. for iteration in range(sys.maxsize): logger.info(">>> Beginning iteration %d/%d", iteration + 1, num_iters) - profiling_iter = time.time() # At this point, `critical_dag` always exists and is what we want. # For the first iteration, the critical DAG is computed before the for @@ -266,13 +259,7 @@ def run(self) -> Generator[IterationResult, None, None]: sum(op.duration for op in non_dummy_ops), ) - profiling_annotate = time.time() self.annotate_capacities(critical_dag) - profiling_annotate = time.time() - profiling_annotate - logger.info( - "PROFILING PhillipsDessouky::annotate_capacities time: %.10fs", - profiling_annotate, - ) if logger.isEnabledFor(logging.DEBUG): logger.debug("Capacity DAG:") @@ -286,7 +273,6 @@ def run(self) -> Generator[IterationResult, None, None]: ) try: - profiling_min_cut = time.time() nodes, edges = self.format_rust_inputs(critical_dag) rust_runner = _lowtime_rs.PhillipsDessouky( FP_ERROR, @@ -296,23 +282,12 @@ def run(self) -> Generator[IterationResult, None, None]: edges, ) s_set, t_set = rust_runner.find_min_cut() - profiling_min_cut = time.time() - profiling_min_cut - logger.info( - "PROFILING PhillipsDessouky::find_min_cut time: %.10fs", - profiling_min_cut, - ) except LowtimeFlowError as e: logger.info("Could not find minimum cut: %s", e.message) logger.info("Terminating PD iteration.") break - profiling_reduce = time.time() cost_change = self.reduce_durations(critical_dag, s_set, t_set) - profiling_reduce = time.time() - profiling_reduce - logger.info( - "PROFILING PhillipsDessouky::reduce_durations time: %.10fs", - profiling_reduce, - ) if cost_change == float("inf") or abs(cost_change) < FP_ERROR: logger.info("No further time reduction possible.") @@ -334,11 +309,6 @@ def run(self) -> Generator[IterationResult, None, None]: unit_time=self.unit_time, ) logger.info("%s", result) - profiling_iter = time.time() - profiling_iter - logger.info( - "PROFILING PhillipsDessouky::run single iteration time: %.10fs", - profiling_iter, - ) yield result def reduce_durations( From c9a0723285075054511acb65b4f88194e9eebd9e Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Tue, 10 Dec 2024 17:13:07 -0500 Subject: [PATCH 44/55] remove unused time library --- lowtime/solver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lowtime/solver.py b/lowtime/solver.py index 7f2f0ae..cf214a2 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -17,7 +17,6 @@ from __future__ import annotations -import time import sys import logging from collections.abc import Generator From ce21569963b6bd2a3bd1b10bd476e7ae6edef5e9 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 11 Dec 2024 18:17:15 -0500 Subject: [PATCH 45/55] Update src/lowtime_graph.rs Co-authored-by: Jae-Won Chung --- src/lowtime_graph.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index 17f4152..e630707 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -58,13 +58,12 @@ impl LowtimeGraph { pub fn max_flow(&self) -> EKFlows> { let edges_edmonds_karp = self.get_ek_preprocessed_edges(); - let (flows, max_flow, min_cut) = edmonds_karp::<_, _, _, SparseCapacity<_>>( + edmonds_karp::<_, _, _, SparseCapacity<_>>( &self.node_ids, &self.source_node_id.unwrap(), &self.sink_node_id.unwrap(), edges_edmonds_karp, - ); - (flows, max_flow, min_cut) + ) } pub fn get_source_node_id(&self) -> u32 { From b97e6e07abdc28255f54acd2d879325265fa901a Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 11 Dec 2024 18:18:26 -0500 Subject: [PATCH 46/55] add newline to eof --- src/lowtime_graph.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index e630707..1b9109f 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -209,5 +209,4 @@ impl LowtimeEdge { pub fn get_lb(&self) -> OrderedFloat { self.lb } - -} \ No newline at end of file +} From 82199981dc7000bf6a39a6e1df230f204e809f9d Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 11 Dec 2024 18:19:04 -0500 Subject: [PATCH 47/55] Update src/phillips_dessouky.rs Co-authored-by: Jae-Won Chung --- src/phillips_dessouky.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index 2e09c68..be15122 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -286,8 +286,7 @@ impl PhillipsDessouky { let mut s_set: HashSet = HashSet::new(); let mut q: VecDeque = VecDeque::new(); q.push_back(new_residual.get_source_node_id()); - while !q.is_empty() { - let cur_id = q.pop_back().unwrap(); + while let Some(cur_id) = q.pop_back() { s_set.insert(cur_id); if cur_id == new_residual.get_sink_node_id() { break; From 48d1725a22ade49ee8f619d998474aaeb5e93096 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 11 Dec 2024 18:27:44 -0500 Subject: [PATCH 48/55] make source/sink raw u32 instead of Option --- src/lowtime_graph.rs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index 1b9109f..9953615 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -11,19 +11,19 @@ use pathfinding::directed::edmonds_karp::{ #[derive(Clone)] pub struct LowtimeGraph { node_ids: Vec, - source_node_id: Option, - sink_node_id: Option, + source_node_id: u32, + sink_node_id: u32, edges: HashMap>, preds: HashMap>, num_edges: usize, } impl LowtimeGraph { - pub fn new() -> Self { + pub fn new(source_node_id: u32, sink_node_id: u32) -> Self { LowtimeGraph { node_ids: Vec::new(), - source_node_id: None, - sink_node_id: None, + source_node_id, + sink_node_id, edges: HashMap::new(), preds: HashMap::new(), num_edges: 0, @@ -36,11 +36,9 @@ impl LowtimeGraph { sink_node_id: u32, edges_raw: Vec<((u32, u32), (f64, f64, f64, f64))>, ) -> Self { - let mut graph = LowtimeGraph::new(); + let mut graph = LowtimeGraph::new(source_node_id, sink_node_id); node_ids.sort(); graph.node_ids = node_ids.clone(); - graph.source_node_id = Some(source_node_id); - graph.sink_node_id = Some(sink_node_id); edges_raw.iter().for_each(|( (from, to), @@ -60,26 +58,26 @@ impl LowtimeGraph { let edges_edmonds_karp = self.get_ek_preprocessed_edges(); edmonds_karp::<_, _, _, SparseCapacity<_>>( &self.node_ids, - &self.source_node_id.unwrap(), - &self.sink_node_id.unwrap(), + &self.source_node_id, + &self.sink_node_id, edges_edmonds_karp, ) } pub fn get_source_node_id(&self) -> u32 { - self.source_node_id.unwrap() + self.source_node_id } pub fn set_source_node_id(&mut self, new_source_node_id: u32) -> () { - self.source_node_id = Some(new_source_node_id); + self.source_node_id = new_source_node_id; } pub fn get_sink_node_id(&self) -> u32 { - self.sink_node_id.unwrap() + self.sink_node_id } pub fn set_sink_node_id(&mut self, new_sink_node_id: u32) -> () { - self.sink_node_id = Some(new_sink_node_id); + self.sink_node_id = new_sink_node_id; } pub fn num_nodes(&self) -> usize { From eeef91d55f11d1ca07db5de94d6c2b9617c85b59 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 11 Dec 2024 18:33:25 -0500 Subject: [PATCH 49/55] add original python find_min_cut, edit docstring to specify why it is not used anymore --- lowtime/solver.py | 199 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) diff --git a/lowtime/solver.py b/lowtime/solver.py index cf214a2..4f896e2 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -363,6 +363,205 @@ def reduce_durations( return cost_change + def find_min_cut(self, dag: nx.DiGraph) -> tuple[set[int], set[int]]: + """Find the min cut of the DAG annotated with lower/upper bound flow capacities. + + Note: this function is not used and instead accelerated by calling + rust_runner.find_min_cut. It is left here for reference in case someone wants + to modify the algorithm in Python for research. + + Assumptions: + - The capacity DAG is in AOA form. + - The capacity DAG has been annotated with `lb` and `ub` attributes on edges, + representing the lower and upper bounds of the flow on the edge. + + Returns: + A tuple of (s_set, t_set) where s_set is the set of nodes on the source side + of the min cut and t_set is the set of nodes on the sink side of the min cut. + Returns None if no feasible flow exists. + + Raises: + LowtimeFlowError: When no feasible flow exists. + """ + source_node = dag.graph["source_node"] + sink_node = dag.graph["sink_node"] + + # In order to solve max flow on edges with both lower and upper bounds, + # we first need to convert it to another DAG that only has upper bounds. + unbound_dag = nx.DiGraph(dag) + + # For every edge, capacity = ub - lb. + for _, _, edge_attrs in unbound_dag.edges(data=True): + edge_attrs["capacity"] = edge_attrs["ub"] - edge_attrs["lb"] + + # Add a new node s', which will become the new source node. + # We constructed the AOA DAG, so we know that node IDs are integers. + node_ids: list[int] = list(unbound_dag.nodes) + s_prime_id = max(node_ids) + 1 + unbound_dag.add_node(s_prime_id) + + # For every node u in the original graph, add an edge (s', u) with capacity + # equal to the sum of all lower bounds of u's parents. + for u in dag.nodes: + capacity = 0.0 + for pred_id in dag.predecessors(u): + capacity += dag[pred_id][u]["lb"] + unbound_dag.add_edge(s_prime_id, u, capacity=capacity) + + # Add a new node t', which will become the new sink node. + t_prime_id = s_prime_id + 1 + unbound_dag.add_node(t_prime_id) + + # For every node u in the original graph, add an edge (u, t') with capacity + # equal to the sum of all lower bounds of u's children. + for u in dag.nodes: + capacity = 0.0 + for succ_id in dag.successors(u): + capacity += dag[u][succ_id]["lb"] + unbound_dag.add_edge(u, t_prime_id, capacity=capacity) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Unbound DAG") + logger.debug("Number of nodes: %d", unbound_dag.number_of_nodes()) + logger.debug("Number of edges: %d", unbound_dag.number_of_edges()) + logger.debug( + "Sum of capacities: %f", + sum(attr["capacity"] for _, _, attr in unbound_dag.edges(data=True)), + ) + + # Add an edge from t to s with infinite capacity. + unbound_dag.add_edge( + sink_node, + source_node, + capacity=float("inf"), + ) + + # We're done with constructing the DAG with only flow upper bounds. + # Find the maximum flow on this DAG. + try: + _, flow_dict = nx.maximum_flow( + unbound_dag, + s_prime_id, + t_prime_id, + capacity="capacity", + flow_func=edmonds_karp, + ) + except nx.NetworkXUnbounded as e: + raise LowtimeFlowError("ERROR: Infinite flow for unbounded DAG.") from e + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("After first max flow") + total_flow = 0.0 + for d in flow_dict.values(): + for flow in d.values(): + total_flow += flow + logger.debug("Sum of all flow values: %f", total_flow) + + # Check if residual graph is saturated. If so, we have a feasible flow. + for u in unbound_dag.successors(s_prime_id): + if ( + abs(flow_dict[s_prime_id][u] - unbound_dag[s_prime_id][u]["capacity"]) + > FP_ERROR + ): + logger.error( + "s' -> %s unsaturated (flow: %s, capacity: %s)", + u, + flow_dict[s_prime_id][u], + unbound_dag[s_prime_id][u]["capacity"], + ) + raise LowtimeFlowError( + "ERROR: Max flow on unbounded DAG didn't saturate." + ) + for u in unbound_dag.predecessors(t_prime_id): + if ( + abs(flow_dict[u][t_prime_id] - unbound_dag[u][t_prime_id]["capacity"]) + > FP_ERROR + ): + logger.error( + "%s -> t' unsaturated (flow: %s, capacity: %s)", + u, + flow_dict[u][t_prime_id], + unbound_dag[u][t_prime_id]["capacity"], + ) + raise LowtimeFlowError( + "ERROR: Max flow on unbounded DAG didn't saturate." + ) + + # We have a feasible flow. Construct a new residual graph with the same + # shape as the capacity DAG so that we can find the min cut. + # First, retrieve the flow amounts to the original capacity graph, where for + # each edge u -> v, the flow amount is `flow + lb`. + for u, v in dag.edges: + dag[u][v]["flow"] = flow_dict[u][v] + dag[u][v]["lb"] + + # Construct a new residual graph (same shape as capacity DAG) with + # u -> v capacity `ub - flow` and v -> u capacity `flow - lb`. + residual_graph = nx.DiGraph(dag) + for u, v in dag.edges: + residual_graph[u][v]["capacity"] = ( + residual_graph[u][v]["ub"] - residual_graph[u][v]["flow"] + ) + capacity = residual_graph[u][v]["flow"] - residual_graph[u][v]["lb"] + if dag.has_edge(v, u): + residual_graph[v][u]["capacity"] = capacity + else: + residual_graph.add_edge(v, u, capacity=capacity) + + # Run max flow on the new residual graph. + try: + _, flow_dict = nx.maximum_flow( + residual_graph, + source_node, + sink_node, + capacity="capacity", + flow_func=edmonds_karp, + ) + except nx.NetworkXUnbounded as e: + raise LowtimeFlowError( + "ERROR: Infinite flow on capacity residual graph." + ) from e + + # Add additional flow we get to the original graph + for u, v in dag.edges: + dag[u][v]["flow"] += flow_dict[u][v] + dag[u][v]["flow"] -= flow_dict[v][u] + + # Construct the new residual graph. + new_residual = nx.DiGraph(dag) + for u, v in dag.edges: + new_residual[u][v]["flow"] = dag[u][v]["ub"] - dag[u][v]["flow"] + new_residual.add_edge(v, u, flow=dag[u][v]["flow"] - dag[u][v]["lb"]) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("New residual graph") + logger.debug("Number of nodes: %d", new_residual.number_of_nodes()) + logger.debug("Number of edges: %d", new_residual.number_of_edges()) + logger.debug( + "Sum of flow: %f", + sum(attr["flow"] for _, _, attr in new_residual.edges(data=True)), + ) + + # Find the s-t cut induced by the second maximum flow above. + # Only following `flow > 0` edges, find the set of nodes reachable from + # source node. That's the s-set, and the rest is the t-set. + s_set = set[int]() + q: deque[int] = deque() + q.append(source_node) + while q: + cur_id = q.pop() + s_set.add(cur_id) + if cur_id == sink_node: + break + for child_id in list(new_residual.successors(cur_id)): + if ( + child_id not in s_set + and abs(new_residual[cur_id][child_id]["flow"]) > FP_ERROR + ): + q.append(child_id) + t_set = set(new_residual.nodes) - s_set + + return s_set, t_set + def annotate_capacities(self, critical_dag: nx.DiGraph) -> None: """In-place annotate the critical DAG with flow capacities.""" # XXX(JW): Is this always large enough? From 46c78e063c5751562d069773ad7cd89e8d30da3f Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 11 Dec 2024 18:45:07 -0500 Subject: [PATCH 50/55] make LowtimeEdge vars public, remove getters/setters --- src/lowtime_graph.rs | 42 +++++----------------------------------- src/phillips_dessouky.rs | 42 ++++++++++++++++++++-------------------- 2 files changed, 26 insertions(+), 58 deletions(-) diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index 9953615..17d7cc7 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -147,7 +147,7 @@ impl LowtimeGraph { processed_edges.extend( self.edges.iter().flat_map(|(from, inner)| inner.iter().map(|(to, edge)| - ((*from, *to), edge.get_capacity()) + ((*from, *to), edge.capacity) ))); processed_edges } @@ -155,10 +155,10 @@ impl LowtimeGraph { #[derive(Clone)] pub struct LowtimeEdge { - capacity: OrderedFloat, - flow: OrderedFloat, - ub: OrderedFloat, - lb: OrderedFloat, + pub capacity: OrderedFloat, + pub flow: OrderedFloat, + pub ub: OrderedFloat, + pub lb: OrderedFloat, } impl LowtimeEdge { @@ -175,36 +175,4 @@ impl LowtimeEdge { lb, } } - - pub fn get_capacity(&self) -> OrderedFloat { - self.capacity - } - - pub fn set_capacity(&mut self, new_capacity: OrderedFloat) -> () { - self.capacity = new_capacity - } - - pub fn get_flow(&self) -> OrderedFloat { - self.flow - } - - pub fn set_flow(&mut self, new_flow: OrderedFloat) -> () { - self.flow = new_flow; - } - - pub fn incr_flow(&mut self, flow: OrderedFloat) -> () { - self.flow += flow; - } - - pub fn decr_flow(&mut self, flow: OrderedFloat) -> () { - self.flow -= flow; - } - - pub fn get_ub(&self) -> OrderedFloat { - self.ub - } - - pub fn get_lb(&self) -> OrderedFloat { - self.lb - } } diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index be15122..fe3c4ef 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -55,7 +55,7 @@ impl PhillipsDessouky { // For every edge, capacity = ub - lb. unbound_dag.edges_mut().for_each(|(_from, _to, edge)| - edge.set_capacity(edge.get_ub() - edge.get_lb()) + edge.capacity = edge.ub - edge.lb ); // Add a new node s', which will become the new source node. @@ -70,7 +70,7 @@ impl PhillipsDessouky { let mut capacity = OrderedFloat(0.0); if let Some(preds) = unbound_dag.predecessors(*u) { capacity = preds.fold(OrderedFloat(0.0), |acc, pred_id| { - acc + unbound_dag.get_edge(*pred_id, *u).get_lb() + acc + unbound_dag.get_edge(*pred_id, *u).lb }); } unbound_dag.add_edge(s_prime_id, *u, LowtimeEdge::new( @@ -91,7 +91,7 @@ impl PhillipsDessouky { let mut capacity = OrderedFloat(0.0); if let Some(succs) = unbound_dag.successors(*u) { capacity = succs.fold(OrderedFloat(0.0), |acc, succ_id| { - acc + unbound_dag.get_edge(*u, *succ_id).get_lb() + acc + unbound_dag.get_edge(*u, *succ_id).lb }); } unbound_dag.add_edge(*u, t_prime_id, LowtimeEdge::new( @@ -107,7 +107,7 @@ impl PhillipsDessouky { debug!("Number of nodes: {}", unbound_dag.num_nodes()); debug!("Number of edges: {}", unbound_dag.num_edges()); let total_capacity = unbound_dag.edges() - .fold(OrderedFloat(0.0), |acc, (_from, _to, edge)| acc + edge.get_capacity()); + .fold(OrderedFloat(0.0), |acc, (_from, _to, edge)| acc + edge.capacity); debug!("Sum of capacities: {}", total_capacity); } @@ -159,14 +159,14 @@ impl PhillipsDessouky { let flow = flow_dict.get(&s_prime_id) .and_then(|inner| inner.get(u)) .unwrap_or(&OrderedFloat(0.0)); - let cap = unbound_dag.get_edge(s_prime_id, *u).get_capacity(); + let cap = unbound_dag.get_edge(s_prime_id, *u).capacity; let diff = (flow - cap).into_inner().abs(); if diff > self.fp_error { error!( "s' -> {} unsaturated (flow: {}, capacity: {})", u, flow_dict[&s_prime_id][u], - unbound_dag.get_edge(s_prime_id, *u).get_capacity(), + unbound_dag.get_edge(s_prime_id, *u).capacity, ); panic!("ERROR: Max flow on unbounded DAG didn't saturate."); } @@ -177,14 +177,14 @@ impl PhillipsDessouky { let flow = flow_dict.get(u) .and_then(|inner| inner.get(&t_prime_id)) .unwrap_or(&OrderedFloat(0.0)); - let cap = unbound_dag.get_edge(*u, t_prime_id).get_capacity(); + let cap = unbound_dag.get_edge(*u, t_prime_id).capacity; let diff = (flow - cap).into_inner().abs(); if diff > self.fp_error { error!( "{} -> t' unsaturated (flow: {}, capacity: {})", u, flow_dict[u][&t_prime_id], - unbound_dag.get_edge(*u, t_prime_id).get_capacity(), + unbound_dag.get_edge(*u, t_prime_id).capacity, ); panic!("ERROR: Max flow on unbounded DAG didn't saturate."); } @@ -199,7 +199,7 @@ impl PhillipsDessouky { let flow = flow_dict.get(u) .and_then(|inner| inner.get(v)) .unwrap_or(&OrderedFloat(0.0)); - edge.set_flow(flow + edge.get_lb()); + edge.flow = flow + edge.lb; } // Construct a new residual graph (same shape as capacity DAG) with @@ -209,19 +209,19 @@ impl PhillipsDessouky { // Rounding small negative values to 0.0 avoids pathfinding::edmonds_karp // from entering unreachable code. Has no impact on correctness in test runs. let residual_uv_edge = residual_graph.get_edge_mut(*u, *v); - let mut uv_capacity = residual_uv_edge.get_ub() - residual_uv_edge.get_flow(); + let mut uv_capacity = residual_uv_edge.ub - residual_uv_edge.flow; if uv_capacity.into_inner().abs() < self.fp_error { uv_capacity = OrderedFloat(0.0); } - residual_uv_edge.set_capacity(uv_capacity); + residual_uv_edge.capacity = uv_capacity; - let mut vu_capacity = residual_uv_edge.get_flow() - residual_uv_edge.get_lb(); + let mut vu_capacity = residual_uv_edge.flow - residual_uv_edge.lb; if vu_capacity.into_inner().abs() < self.fp_error { vu_capacity = OrderedFloat(0.0); } match self.dag.has_edge(*v, *u) { - true => residual_graph.get_edge_mut(*v, *u).set_capacity(vu_capacity), + true => residual_graph.get_edge_mut(*v, *u).capacity = vu_capacity, false => residual_graph.add_edge(*v, *u, LowtimeEdge::new( vu_capacity, OrderedFloat(0.0), // flow @@ -251,21 +251,21 @@ impl PhillipsDessouky { // Add additional flow we get to the original graph for (u, v, edge) in self.dag.edges_mut() { - edge.incr_flow(*flow_dict.get(u) + edge.flow += *flow_dict.get(u) .and_then(|inner| inner.get(v)) - .unwrap_or(&OrderedFloat(0.0))); - edge.decr_flow(*flow_dict.get(v) + .unwrap_or(&OrderedFloat(0.0)); + edge.flow -= *flow_dict.get(v) .and_then(|inner| inner.get(u)) - .unwrap_or(&OrderedFloat(0.0))); + .unwrap_or(&OrderedFloat(0.0)); } // Construct the new residual graph. let mut new_residual = self.dag.clone(); for (u, v, edge) in self.dag.edges() { - new_residual.get_edge_mut(*u, *v).set_flow(edge.get_ub() - edge.get_flow()); + new_residual.get_edge_mut(*u, *v).flow = edge.ub - edge.flow; new_residual.add_edge(*v, *u, LowtimeEdge::new( OrderedFloat(0.0), // capacity - edge.get_flow() - edge.get_lb(), // flow + edge.flow - edge.lb, // flow OrderedFloat(0.0), // ub OrderedFloat(0.0), // lb )); @@ -276,7 +276,7 @@ impl PhillipsDessouky { debug!("Number of nodes: {}", new_residual.num_nodes()); debug!("Number of edges: {}", new_residual.num_edges()); let total_flow = unbound_dag.edges() - .fold(OrderedFloat(0.0), |acc, (_from, _to, edge)| acc + edge.get_flow()); + .fold(OrderedFloat(0.0), |acc, (_from, _to, edge)| acc + edge.flow); debug!("Sum of capacities: {}", total_flow); } @@ -293,7 +293,7 @@ impl PhillipsDessouky { } if let Some(succs) = new_residual.successors(cur_id) { for child_id in succs { - let flow = new_residual.get_edge(cur_id, *child_id).get_flow().into_inner(); + let flow = new_residual.get_edge(cur_id, *child_id).flow.into_inner(); if !s_set.contains(child_id) && flow.abs() > self.fp_error { q.push_back(*child_id); } From 86752f9c5693b42c4957ff4bc3ca9378d2ec938a Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 11 Dec 2024 18:50:39 -0500 Subject: [PATCH 51/55] make LowtimeGraph.node_ids pub --- src/lowtime_graph.rs | 6 +----- src/phillips_dessouky.rs | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index 17d7cc7..d163532 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -10,7 +10,7 @@ use pathfinding::directed::edmonds_karp::{ #[derive(Clone)] pub struct LowtimeGraph { - node_ids: Vec, + pub node_ids: Vec, source_node_id: u32, sink_node_id: u32, edges: HashMap>, @@ -108,10 +108,6 @@ impl LowtimeGraph { }) } - pub fn get_node_ids(&self) -> &Vec { - &self.node_ids - } - pub fn add_node_id(&mut self, node_id: u32) -> () { assert!(self.node_ids.last().unwrap() < &node_id, "New node ids must be larger than all existing node ids"); self.node_ids.push(node_id) diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index fe3c4ef..4f7f2e6 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -60,12 +60,12 @@ impl PhillipsDessouky { // Add a new node s', which will become the new source node. // We constructed the AOA DAG, so we know that node IDs are integers. - let s_prime_id = unbound_dag.get_node_ids().last().unwrap() + 1; + let s_prime_id = unbound_dag.node_ids.last().unwrap() + 1; unbound_dag.add_node_id(s_prime_id); // For every node u in the original graph, add an edge (s', u) with capacity // equal to the sum of all lower bounds of u's parents. - let orig_node_ids = self.dag.get_node_ids(); + let orig_node_ids = &self.dag.node_ids; for u in orig_node_ids.iter() { let mut capacity = OrderedFloat(0.0); if let Some(preds) = unbound_dag.predecessors(*u) { @@ -300,7 +300,7 @@ impl PhillipsDessouky { } } } - let all_nodes: HashSet = new_residual.get_node_ids().into_iter().copied().collect(); + let all_nodes: HashSet = new_residual.node_ids.iter().copied().collect(); let t_set: HashSet = all_nodes.difference(&s_set).copied().collect(); (s_set, t_set) } From cb06b5317fa17d4a55a8465254e7f342b47da413 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 11 Dec 2024 18:52:29 -0500 Subject: [PATCH 52/55] make LowtimeGraph.source/sink_node_id pub --- src/lowtime_graph.rs | 20 ++------------------ src/phillips_dessouky.rs | 12 ++++++------ 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index d163532..9183503 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -11,8 +11,8 @@ use pathfinding::directed::edmonds_karp::{ #[derive(Clone)] pub struct LowtimeGraph { pub node_ids: Vec, - source_node_id: u32, - sink_node_id: u32, + pub source_node_id: u32, + pub sink_node_id: u32, edges: HashMap>, preds: HashMap>, num_edges: usize, @@ -64,22 +64,6 @@ impl LowtimeGraph { ) } - pub fn get_source_node_id(&self) -> u32 { - self.source_node_id - } - - pub fn set_source_node_id(&mut self, new_source_node_id: u32) -> () { - self.source_node_id = new_source_node_id; - } - - pub fn get_sink_node_id(&self) -> u32 { - self.sink_node_id - } - - pub fn set_sink_node_id(&mut self, new_sink_node_id: u32) -> () { - self.sink_node_id = new_sink_node_id; - } - pub fn num_nodes(&self) -> usize { self.node_ids.len() } diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index 4f7f2e6..ac0f809 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -113,8 +113,8 @@ impl PhillipsDessouky { // Add an edge from t to s with infinite capacity. unbound_dag.add_edge( - unbound_dag.get_sink_node_id(), - unbound_dag.get_source_node_id(), + unbound_dag.sink_node_id, + unbound_dag.source_node_id, LowtimeEdge::new( OrderedFloat(f64::INFINITY), // capacity OrderedFloat(0.0), // flow @@ -124,8 +124,8 @@ impl PhillipsDessouky { ); // Update source and sink on unbound_dag - unbound_dag.set_source_node_id(s_prime_id); - unbound_dag.set_sink_node_id(t_prime_id); + unbound_dag.source_node_id = s_prime_id; + unbound_dag.sink_node_id = t_prime_id; // We're done with constructing the DAG with only flow upper bounds. // Find the maximum flow on this DAG. @@ -285,10 +285,10 @@ impl PhillipsDessouky { // source node. That's the s-set, and the rest is the t-set. let mut s_set: HashSet = HashSet::new(); let mut q: VecDeque = VecDeque::new(); - q.push_back(new_residual.get_source_node_id()); + q.push_back(new_residual.source_node_id); while let Some(cur_id) = q.pop_back() { s_set.insert(cur_id); - if cur_id == new_residual.get_sink_node_id() { + if cur_id == new_residual.sink_node_id { break; } if let Some(succs) = new_residual.successors(cur_id) { From e41933eec439e5bba588103463cd6d684d726dcc Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 11 Dec 2024 19:17:15 -0500 Subject: [PATCH 53/55] make non-capacity vars in LowtimeEdge raw f64 instead of OrderedFloat --- src/lowtime_graph.rs | 18 +++++------ src/phillips_dessouky.rs | 68 ++++++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/lowtime_graph.rs b/src/lowtime_graph.rs index 9183503..bbc89bc 100644 --- a/src/lowtime_graph.rs +++ b/src/lowtime_graph.rs @@ -46,9 +46,9 @@ impl LowtimeGraph { )| { graph.add_edge(*from, *to, LowtimeEdge::new( OrderedFloat(*capacity), - OrderedFloat(*flow), - OrderedFloat(*ub), - OrderedFloat(*lb), + *flow, + *ub, + *lb, )) }); graph @@ -136,17 +136,17 @@ impl LowtimeGraph { #[derive(Clone)] pub struct LowtimeEdge { pub capacity: OrderedFloat, - pub flow: OrderedFloat, - pub ub: OrderedFloat, - pub lb: OrderedFloat, + pub flow: f64, + pub ub: f64, + pub lb: f64, } impl LowtimeEdge { pub fn new( capacity: OrderedFloat, - flow: OrderedFloat, - ub: OrderedFloat, - lb: OrderedFloat, + flow: f64, + ub: f64, + lb: f64, ) -> Self { LowtimeEdge { capacity, diff --git a/src/phillips_dessouky.rs b/src/phillips_dessouky.rs index ac0f809..fde1183 100644 --- a/src/phillips_dessouky.rs +++ b/src/phillips_dessouky.rs @@ -55,7 +55,7 @@ impl PhillipsDessouky { // For every edge, capacity = ub - lb. unbound_dag.edges_mut().for_each(|(_from, _to, edge)| - edge.capacity = edge.ub - edge.lb + edge.capacity = OrderedFloat(edge.ub - edge.lb) ); // Add a new node s', which will become the new source node. @@ -69,15 +69,15 @@ impl PhillipsDessouky { for u in orig_node_ids.iter() { let mut capacity = OrderedFloat(0.0); if let Some(preds) = unbound_dag.predecessors(*u) { - capacity = preds.fold(OrderedFloat(0.0), |acc, pred_id| { + capacity = OrderedFloat(preds.fold(0.0, |acc, pred_id| { acc + unbound_dag.get_edge(*pred_id, *u).lb - }); + })); } unbound_dag.add_edge(s_prime_id, *u, LowtimeEdge::new( capacity, - OrderedFloat(0.0), // flow - OrderedFloat(0.0), // ub - OrderedFloat(0.0), // lb + 0.0, // flow + 0.0, // ub + 0.0, // lb )); } @@ -90,15 +90,15 @@ impl PhillipsDessouky { for u in orig_node_ids.iter() { let mut capacity = OrderedFloat(0.0); if let Some(succs) = unbound_dag.successors(*u) { - capacity = succs.fold(OrderedFloat(0.0), |acc, succ_id| { + capacity = OrderedFloat(succs.fold(0.0, |acc, succ_id| { acc + unbound_dag.get_edge(*u, *succ_id).lb - }); + })); } unbound_dag.add_edge(*u, t_prime_id, LowtimeEdge::new( capacity, - OrderedFloat(0.0), // flow - OrderedFloat(0.0), // ub - OrderedFloat(0.0), // lb + 0.0, // flow + 0.0, // ub + 0.0, // lb )); } @@ -117,9 +117,9 @@ impl PhillipsDessouky { unbound_dag.source_node_id, LowtimeEdge::new( OrderedFloat(f64::INFINITY), // capacity - OrderedFloat(0.0), // flow - OrderedFloat(0.0), // ub - OrderedFloat(0.0), // lb + 0.0, // flow + 0.0, // ub + 0.0, // lb ), ); @@ -145,10 +145,10 @@ impl PhillipsDessouky { // Convert flows to dict for faster lookup let flow_dict = flows.iter().fold( HashMap::new(), - |mut acc: HashMap>>, ((from, to), flow)| { + |mut acc: HashMap>, ((from, to), flow)| { acc.entry(*from) .or_insert_with(HashMap::new) - .insert(*to, *flow); + .insert(*to, flow.into_inner()); acc } ); @@ -158,9 +158,9 @@ impl PhillipsDessouky { for u in succs { let flow = flow_dict.get(&s_prime_id) .and_then(|inner| inner.get(u)) - .unwrap_or(&OrderedFloat(0.0)); + .unwrap_or(&0.0); let cap = unbound_dag.get_edge(s_prime_id, *u).capacity; - let diff = (flow - cap).into_inner().abs(); + let diff = (flow - cap.into_inner()).abs(); if diff > self.fp_error { error!( "s' -> {} unsaturated (flow: {}, capacity: {})", @@ -176,9 +176,9 @@ impl PhillipsDessouky { for u in preds { let flow = flow_dict.get(u) .and_then(|inner| inner.get(&t_prime_id)) - .unwrap_or(&OrderedFloat(0.0)); + .unwrap_or(&0.0); let cap = unbound_dag.get_edge(*u, t_prime_id).capacity; - let diff = (flow - cap).into_inner().abs(); + let diff = (flow - cap.into_inner()).abs(); if diff > self.fp_error { error!( "{} -> t' unsaturated (flow: {}, capacity: {})", @@ -198,7 +198,7 @@ impl PhillipsDessouky { for (u, v, edge) in self.dag.edges_mut() { let flow = flow_dict.get(u) .and_then(|inner| inner.get(v)) - .unwrap_or(&OrderedFloat(0.0)); + .unwrap_or(&0.0); edge.flow = flow + edge.lb; } @@ -209,13 +209,13 @@ impl PhillipsDessouky { // Rounding small negative values to 0.0 avoids pathfinding::edmonds_karp // from entering unreachable code. Has no impact on correctness in test runs. let residual_uv_edge = residual_graph.get_edge_mut(*u, *v); - let mut uv_capacity = residual_uv_edge.ub - residual_uv_edge.flow; + let mut uv_capacity = OrderedFloat(residual_uv_edge.ub - residual_uv_edge.flow); if uv_capacity.into_inner().abs() < self.fp_error { uv_capacity = OrderedFloat(0.0); } residual_uv_edge.capacity = uv_capacity; - let mut vu_capacity = residual_uv_edge.flow - residual_uv_edge.lb; + let mut vu_capacity = OrderedFloat(residual_uv_edge.flow - residual_uv_edge.lb); if vu_capacity.into_inner().abs() < self.fp_error { vu_capacity = OrderedFloat(0.0); } @@ -224,9 +224,9 @@ impl PhillipsDessouky { true => residual_graph.get_edge_mut(*v, *u).capacity = vu_capacity, false => residual_graph.add_edge(*v, *u, LowtimeEdge::new( vu_capacity, - OrderedFloat(0.0), // flow - OrderedFloat(0.0), // ub - OrderedFloat(0.0), // lb + 0.0, // flow + 0.0, // ub + 0.0, // lb )), } } @@ -241,10 +241,10 @@ impl PhillipsDessouky { // Convert flows to dict for faster lookup let flow_dict = flows.iter().fold( HashMap::new(), - |mut acc: HashMap>>, ((from, to), flow)| { + |mut acc: HashMap>, ((from, to), flow)| { acc.entry(*from) .or_insert_with(HashMap::new) - .insert(*to, *flow); + .insert(*to, flow.into_inner()); acc } ); @@ -253,10 +253,10 @@ impl PhillipsDessouky { for (u, v, edge) in self.dag.edges_mut() { edge.flow += *flow_dict.get(u) .and_then(|inner| inner.get(v)) - .unwrap_or(&OrderedFloat(0.0)); + .unwrap_or(&0.0); edge.flow -= *flow_dict.get(v) .and_then(|inner| inner.get(u)) - .unwrap_or(&OrderedFloat(0.0)); + .unwrap_or(&0.0); } // Construct the new residual graph. @@ -266,8 +266,8 @@ impl PhillipsDessouky { new_residual.add_edge(*v, *u, LowtimeEdge::new( OrderedFloat(0.0), // capacity edge.flow - edge.lb, // flow - OrderedFloat(0.0), // ub - OrderedFloat(0.0), // lb + 0.0, // ub + 0.0, // lb )); } @@ -276,7 +276,7 @@ impl PhillipsDessouky { debug!("Number of nodes: {}", new_residual.num_nodes()); debug!("Number of edges: {}", new_residual.num_edges()); let total_flow = unbound_dag.edges() - .fold(OrderedFloat(0.0), |acc, (_from, _to, edge)| acc + edge.flow); + .fold(0.0, |acc, (_from, _to, edge)| acc + edge.flow); debug!("Sum of capacities: {}", total_flow); } @@ -293,7 +293,7 @@ impl PhillipsDessouky { } if let Some(succs) = new_residual.successors(cur_id) { for child_id in succs { - let flow = new_residual.get_edge(cur_id, *child_id).flow.into_inner(); + let flow = new_residual.get_edge(cur_id, *child_id).flow; if !s_set.contains(child_id) && flow.abs() > self.fp_error { q.push_back(*child_id); } From 4a094f8cb714c9a0e51ab24b297d4565775dfd1a Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 11 Dec 2024 19:57:57 -0500 Subject: [PATCH 54/55] add deque to solver, even though find_min_cut never called --- lowtime/solver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lowtime/solver.py b/lowtime/solver.py index 4f896e2..1e18883 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -19,6 +19,7 @@ import sys import logging +from collections import deque from collections.abc import Generator import networkx as nx From 723d80a05e1bf237f7b3f968ad36be85c7944f44 Mon Sep 17 00:00:00 2001 From: Oh Jun Kweon Date: Wed, 11 Dec 2024 20:00:47 -0500 Subject: [PATCH 55/55] add nx edmonds_karp even though find_min_cut never called --- lowtime/solver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lowtime/solver.py b/lowtime/solver.py index 1e18883..ab7a060 100644 --- a/lowtime/solver.py +++ b/lowtime/solver.py @@ -23,6 +23,7 @@ from collections.abc import Generator import networkx as nx +from networkx.algorithms.flow import edmonds_karp from attrs import define, field from lowtime.operation import Operation