From 86d21e86a84587e134aaf19aeb7cf0ce4aa7bcbc Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 18 Aug 2022 18:11:26 +0200 Subject: [PATCH 01/59] Add channels --- compiler/src/vm/channel.rs | 44 ++++++++++++++++++++++++++++++++++++++ compiler/src/vm/mod.rs | 1 + 2 files changed, 45 insertions(+) create mode 100644 compiler/src/vm/channel.rs diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs new file mode 100644 index 000000000..148ebe712 --- /dev/null +++ b/compiler/src/vm/channel.rs @@ -0,0 +1,44 @@ +use super::{Heap, Pointer}; +use std::collections::VecDeque; + +/// A conveyer belt or pipe that flows between send and receive ports in the +/// program. Using send ports, you can put packets into a channel. Using receive +/// ports, you can get packets out again. +/// +/// Channels always have a maximum capacity of packets that they can hold +/// simultaneously – you can set it to something large, but having no capacity +/// enables buggy code that leaks memory. +#[derive(Clone)] +pub struct Channel { + pub capacity: usize, + packets: VecDeque, +} + +/// A self-contained value that is sent over a channel. +#[derive(Clone)] +pub struct Packet { + heap: Heap, + value: Pointer, +} + +impl Channel { + pub fn new(capacity: usize) -> Self { + Self { + capacity, + packets: Default::default(), + } + } + + pub fn is_full(&self) -> bool { + self.packets.len() == self.capacity + } + + pub fn send(&mut self, packet: Packet) { + assert!(!self.is_full()); + self.packets.push_back(packet); + } + + pub fn receive(&mut self) -> Option { + self.packets.pop_front() + } +} diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index e4d57e679..af6ab459d 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -1,4 +1,5 @@ mod builtin_functions; +mod channel; mod heap; pub mod tracer; pub mod use_provider; From 5062d561680ab5136b711a9f91f8911e8f8687b0 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 18 Aug 2022 18:12:20 +0200 Subject: [PATCH 02/59] Add ports --- compiler/src/vm/heap/mod.rs | 32 ++++++++++++++++++++++++----- compiler/src/vm/heap/object.rs | 37 +++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/compiler/src/vm/heap/mod.rs b/compiler/src/vm/heap/mod.rs index e6c7d1fd7..54471dce3 100644 --- a/compiler/src/vm/heap/mod.rs +++ b/compiler/src/vm/heap/mod.rs @@ -2,7 +2,9 @@ mod object; mod pointer; pub use self::{ - object::{Builtin, Closure, Data, Int, Object, Struct, Symbol, Text}, + object::{ + Builtin, ChannelId, Closure, Data, Int, Object, ReceivePort, SendPort, Struct, Symbol, Text, + }, pointer::Pointer, }; use crate::builtin_functions::BuiltinFunction; @@ -123,6 +125,7 @@ impl Heap { &self, other: &mut Heap, addresses: &[Pointer], + channel_map: &HashMap, ) -> Vec { let mut objects_to_refcounts = HashMap::new(); for address in addresses { @@ -144,7 +147,7 @@ impl Heap { address_map[&address], Object { reference_count: refcount, - data: Self::map_addresses_in_data(&address_map, &self.get(address).data), + data: Self::map_data(&address_map, channel_map, &self.get(address).data), }, ); } @@ -165,7 +168,11 @@ impl Heap { self.gather_objects_to_clone(objects_to_refcounts, child); } } - fn map_addresses_in_data(address_map: &HashMap, data: &Data) -> Data { + fn map_data( + address_map: &HashMap, + channel_map: &HashMap, + data: &Data, + ) -> Data { match data { Data::Int(int) => Data::Int(int.clone()), Data::Text(text) => Data::Text(text.clone()), @@ -187,10 +194,19 @@ impl Heap { body: closure.body.clone(), }), Data::Builtin(builtin) => Data::Builtin(builtin.clone()), + Data::SendPort(port) => Data::SendPort(SendPort::new(channel_map[&port.channel])), + Data::ReceivePort(port) => { + Data::ReceivePort(ReceivePort::new(channel_map[&port.channel])) + } } } - pub fn clone_single_to_other_heap(&self, other: &mut Heap, address: Pointer) -> Pointer { - self.clone_multiple_to_other_heap(other, &[address]) + pub fn clone_single_to_other_heap( + &self, + other: &mut Heap, + address: Pointer, + channel_map: &HashMap, + ) -> Pointer { + self.clone_multiple_to_other_heap(other, &[address], channel_map) .pop() .unwrap() } @@ -213,6 +229,12 @@ impl Heap { pub fn create_builtin(&mut self, builtin: BuiltinFunction) -> Pointer { self.create(Data::Builtin(Builtin { function: builtin })) } + pub fn create_send_port(&mut self, channel: ChannelId) -> Pointer { + self.create(Data::SendPort(SendPort::new(channel))) + } + pub fn create_receive_port(&mut self, channel: ChannelId) -> Pointer { + self.create(Data::ReceivePort(ReceivePort::new(channel))) + } pub fn create_nothing(&mut self) -> Pointer { self.create_symbol("Nothing".to_string()) } diff --git a/compiler/src/vm/heap/object.rs b/compiler/src/vm/heap/object.rs index 7415d2a22..03be55974 100644 --- a/compiler/src/vm/heap/object.rs +++ b/compiler/src/vm/heap/object.rs @@ -29,6 +29,8 @@ pub enum Data { Struct(Struct), Closure(Closure), Builtin(Builtin), + SendPort(SendPort), + ReceivePort(ReceivePort), } #[derive(Clone)] @@ -155,6 +157,28 @@ impl Closure { } } +#[derive(Clone)] +pub struct SendPort { + pub channel: ChannelId, +} +#[derive(Clone)] +pub struct ReceivePort { + pub channel: ChannelId, +} + +pub type ChannelId = usize; + +impl SendPort { + pub fn new(channel: ChannelId) -> Self { + Self { channel } + } +} +impl ReceivePort { + pub fn new(channel: ChannelId) -> Self { + Self { channel } + } +} + impl Data { fn hash(&self, heap: &Heap) -> u64 { let mut state = DefaultHasher::new(); @@ -183,6 +207,8 @@ impl Data { closure.body.hash(state); } Data::Builtin(builtin) => builtin.function.hash(state), + Data::SendPort(port) => port.channel.hash(state), + Data::ReceivePort(port) => port.channel.hash(state), } } @@ -194,6 +220,8 @@ impl Data { (Data::Struct(a), Data::Struct(b)) => a.equals(heap, b), (Data::Closure(_), Data::Closure(_)) => false, (Data::Builtin(a), Data::Builtin(b)) => a.function == b.function, + (Data::SendPort(a), Data::SendPort(b)) => a.channel == b.channel, + (Data::ReceivePort(a), Data::ReceivePort(b)) => a.channel == b.channel, _ => false, } } @@ -215,12 +243,19 @@ impl Data { ), Data::Closure(_) => "{...}".to_string(), Data::Builtin(builtin) => format!("builtin{:?}", builtin.function), + Data::SendPort(port) => format!("", port.channel), + Data::ReceivePort(port) => format!("", port.channel), } } pub fn children(&self) -> Vec { match self { - Data::Int(_) | Data::Text(_) | Data::Symbol(_) | Data::Builtin(_) => vec![], + Data::Int(_) + | Data::Text(_) + | Data::Symbol(_) + | Data::Builtin(_) + | Data::SendPort(_) + | Data::ReceivePort(_) => vec![], Data::Struct(struct_) => struct_ .iter() .flat_map(|(a, b)| vec![a, b].into_iter()) From d5d921ab55b521871ebebac7437f1261c532ad17 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 18 Aug 2022 18:12:54 +0200 Subject: [PATCH 03/59] Refactor Heap::create_list --- compiler/src/vm/builtin_functions.rs | 4 ++-- compiler/src/vm/heap/mod.rs | 6 +++--- compiler/src/vm/use_provider.rs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index da08aa604..fc5ecd390 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -236,7 +236,7 @@ impl Heap { } fn struct_get_keys(&mut self, args: &[Pointer]) -> BuiltinResult { unpack_and_later_drop!(self, args, (struct_: Struct), { - Return(self.create_list(struct_.iter().map(|(key, _)| key).collect())) + Return(self.create_list(&struct_.iter().map(|(key, _)| key).collect_vec())) }) } fn struct_has_key(&mut self, args: &[Pointer]) -> BuiltinResult { @@ -252,7 +252,7 @@ impl Heap { for c in text.value.graphemes(true) { character_addresses.push(self.create_text(c.to_string())); } - Return(self.create_list(character_addresses)) + Return(self.create_list(&character_addresses)) }) } fn text_concatenate(&mut self, args: &[Pointer]) -> BuiltinResult { diff --git a/compiler/src/vm/heap/mod.rs b/compiler/src/vm/heap/mod.rs index 54471dce3..e72983230 100644 --- a/compiler/src/vm/heap/mod.rs +++ b/compiler/src/vm/heap/mod.rs @@ -238,10 +238,10 @@ impl Heap { pub fn create_nothing(&mut self) -> Pointer { self.create_symbol("Nothing".to_string()) } - pub fn create_list(&mut self, items: Vec) -> Pointer { + pub fn create_list(&mut self, items: &[Pointer]) -> Pointer { let mut fields = vec![]; - for (index, item) in items.into_iter().enumerate() { - fields.push((self.create_int(index.into()), item)); + for (index, item) in items.iter().enumerate() { + fields.push((self.create_int(index.into()), *item)); } self.create_struct(fields.into_iter().collect()) } diff --git a/compiler/src/vm/use_provider.rs b/compiler/src/vm/use_provider.rs index 3e0d5a7a2..969164746 100644 --- a/compiler/src/vm/use_provider.rs +++ b/compiler/src/vm/use_provider.rs @@ -54,7 +54,7 @@ impl Vm { .iter() .map(|byte| self.heap.create_int((*byte).into())) .collect_vec(); - let list = self.heap.create_list(bytes); + let list = self.heap.create_list(&bytes); self.data_stack.push(list); } UseResult::Code(lir) => { From 54a04292c7425692cf20883df70661dc57834e60 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 18 Aug 2022 18:13:10 +0200 Subject: [PATCH 04/59] Start implementing builtins --- compiler/src/builtin_functions.rs | 3 ++ compiler/src/vm/builtin_functions.rs | 48 +++++++++++++++++++++++++++- compiler/src/vm/vm.rs | 42 +++++++++++++++++++++--- 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/compiler/src/builtin_functions.rs b/compiler/src/builtin_functions.rs index 1540a801e..f76261123 100644 --- a/compiler/src/builtin_functions.rs +++ b/compiler/src/builtin_functions.rs @@ -4,6 +4,9 @@ use strum_macros::EnumIter; #[derive(Debug, EnumIter, PartialEq, Eq, Clone, Hash, Copy)] pub enum BuiltinFunction { + ChannelCreate, // capacity -> [sendPort, receivePort] + ChannelSend, // channel any -> Nothing + ChannelReceive, // channel -> any Equals, // any any -> booleanSymbol FunctionRun, // (lambdaWith0Arguments) -> (returnValue: any) GetArgumentCount, // closure -> argumentCount diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index fc5ecd390..eec7761d0 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -1,9 +1,12 @@ use super::{ - heap::{Closure, Data, Int, Pointer, Struct, Symbol, Text}, + channel::Channel, + heap::{Closure, Data, Int, Pointer, SendPort, Struct, Symbol, Text}, use_provider::UseProvider, + vm::ChannelOperation, Heap, Vm, }; use crate::{builtin_functions::BuiltinFunction, compiler::lir::Instruction}; +use itertools::Itertools; use num_bigint::BigInt; use num_integer::Integer; use num_traits::ToPrimitive; @@ -20,6 +23,9 @@ impl Vm { ) { let result = span!(Level::TRACE, "Running builtin{builtin_function:?}").in_scope(|| { match &builtin_function { + BuiltinFunction::ChannelCreate => self.channel_create(args), + BuiltinFunction::ChannelSend => self.channel_send(args), + BuiltinFunction::ChannelReceive => self.channel_receive(args), BuiltinFunction::Equals => self.heap.equals(args), BuiltinFunction::FunctionRun => self.heap.function_run(args), BuiltinFunction::GetArgumentCount => self.heap.get_argument_count(args), @@ -103,6 +109,44 @@ macro_rules! unpack_and_later_drop { }; } +impl Vm { + fn channel_create(&mut self, args: &[Pointer]) -> BuiltinResult { + unpack_and_later_drop!(self.heap, args, (capacity: Int), { + let id = self.next_internal_channel_id + 1; + self.next_internal_channel_id += 1; + self.internal_channels.insert( + id, + Channel::new(capacity.value.try_into().expect( + "you tried to create a channel with a capacity bigger than the maximum usize", + )), + ); + + let send_port = self.heap.create_send_port(id); + let receive_port = self.heap.create_receive_port(id); + Return(self.heap.create_list(&[send_port, receive_port])) + }) + } + + fn channel_send(&mut self, args: &[Pointer]) -> BuiltinResult { + unpack_and_later_drop!(self.heap, args, (port: SendPort, packet: Any), { + if let Some(channel) = self.internal_channels.get(&port.channel) { + // An internal channel. + } else { + // An external channel. + let external_id = self.internal_to_external_channels[&port.channel]; + self.channel_operations + .entry(external_id) + .or_default() + .push(ChannelOperation::Send { packet: () }) + } + }) + } + + fn channel_receive(&mut self, args: &[Pointer]) -> BuiltinResult { + todo!() + } +} + impl Heap { fn equals(&mut self, args: &[Pointer]) -> BuiltinResult { unpack_and_later_drop!(self, args, (a: Any, b: Any), { @@ -328,6 +372,8 @@ impl Heap { Data::Struct(_) => "Struct", Data::Closure(_) => "Function", Data::Builtin(_) => "Builtin", + Data::SendPort(_) => "SendPort", + Data::ReceivePort(_) => "ReceivePort", }; Return(self.create_symbol(symbol.to_string())) }) diff --git a/compiler/src/vm/vm.rs b/compiler/src/vm/vm.rs index 86f0390db..6ed2337e3 100644 --- a/compiler/src/vm/vm.rs +++ b/compiler/src/vm/vm.rs @@ -1,5 +1,6 @@ use super::{ - heap::{Builtin, Closure, Data, Heap, Pointer}, + channel::{Channel, Packet}, + heap::{Builtin, ChannelId, Closure, Data, Heap, Pointer}, tracer::{TraceEntry, Tracer}, use_provider::{DbUseProvider, UseProvider}, }; @@ -9,20 +10,40 @@ use crate::{ module::Module, }; use itertools::Itertools; -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use tracing::{info, trace}; const TRACE: bool = false; -/// A VM can execute some byte code. +/// A VM is a Candy program that thinks it's currently running. Because VMs are +/// first-class Rust structs, they enable other code to store "freezed" programs +/// and to remain in control about when and for how long they run. #[derive(Clone)] pub struct Vm { + // Core functionality to run code. VMs are stack-based machines that run + // instructions from a LIR. All values are stored on a heap. pub status: Status, next_instruction: InstructionPointer, - pub heap: Heap, pub data_stack: Vec, pub call_stack: Vec, pub import_stack: Vec, + pub heap: Heap, + + // Channel functionality. VMs communicate with the outer world using + // channels. Each channel is identified using an ID that is valid inside + // this particular VM. Channels created by the program are managed ("owned") + // by the VM itself. For channels owned by the outside world (such as those + // referenced in the environment argument), the VM maintains a mapping + // between internal and external IDs. + pub internal_channels: HashMap, + pub next_internal_channel_id: ChannelId, + pub external_to_internal_channels: HashMap, + pub internal_to_external_channels: HashMap, + pub channel_operations: HashMap>, + + // Debug properties. These are not essential to a correct working of the VM, + // but they enable advanced functionality like stack traces or finding out + // who's fault a panic is. pub tracer: Tracer, pub fuzzable_closures: Vec<(Id, Pointer)>, pub num_instructions_executed: usize, @@ -58,6 +79,12 @@ impl InstructionPointer { } } +#[derive(Clone)] +pub enum ChannelOperation { + Send { packet: Packet }, + Receive, +} + pub struct TearDownResult { pub heap: Heap, pub result: Result, @@ -70,10 +97,15 @@ impl Vm { Self { status: Status::Done, next_instruction: InstructionPointer::null_pointer(), - heap, data_stack: vec![], call_stack: vec![], import_stack: vec![], + heap, + internal_channels: HashMap::new(), + next_internal_channel_id: 0, + external_to_internal_channels: HashMap::new(), + internal_to_external_channels: HashMap::new(), + channel_operations: Default::default(), tracer: Tracer::default(), fuzzable_closures: vec![], num_instructions_executed: 0, From ba2475ef4c502a7f42d0b7affc6c3d42708ac181 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 18 Aug 2022 23:04:12 +0200 Subject: [PATCH 05/59] Implement channels --- compiler/src/vm/builtin_functions.rs | 170 ++++++----- compiler/src/vm/channel.rs | 19 +- compiler/src/vm/fiber.rs | 436 +++++++++++++++++++++++++++ compiler/src/vm/heap/mod.rs | 24 +- compiler/src/vm/mod.rs | 6 +- compiler/src/vm/use_provider.rs | 4 +- compiler/src/vm/vm.rs | 423 ++++---------------------- packages/Core/.candy | 4 + packages/Core/Channel.candy | 18 ++ packages/Core/Concurrency.candy | 5 + 10 files changed, 645 insertions(+), 464 deletions(-) create mode 100644 compiler/src/vm/fiber.rs create mode 100644 packages/Core/Channel.candy create mode 100644 packages/Core/Concurrency.candy diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index eec7761d0..62f8e4f70 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -1,9 +1,9 @@ use super::{ - channel::Channel, - heap::{Closure, Data, Int, Pointer, SendPort, Struct, Symbol, Text}, + channel::{Capacity, Packet}, + fiber::{Fiber, Status}, + heap::{ChannelId, Closure, Data, Int, Pointer, ReceivePort, SendPort, Struct, Symbol, Text}, use_provider::UseProvider, - vm::ChannelOperation, - Heap, Vm, + Heap, }; use crate::{builtin_functions::BuiltinFunction, compiler::lir::Instruction}; use itertools::Itertools; @@ -14,59 +14,61 @@ use std::{ops::Deref, str::FromStr}; use tracing::{info, span, Level}; use unicode_segmentation::UnicodeSegmentation; -impl Vm { +impl Fiber { pub(super) fn run_builtin_function( &mut self, use_provider: &U, builtin_function: &BuiltinFunction, args: &[Pointer], ) { - let result = span!(Level::TRACE, "Running builtin{builtin_function:?}").in_scope(|| { - match &builtin_function { - BuiltinFunction::ChannelCreate => self.channel_create(args), - BuiltinFunction::ChannelSend => self.channel_send(args), - BuiltinFunction::ChannelReceive => self.channel_receive(args), - BuiltinFunction::Equals => self.heap.equals(args), - BuiltinFunction::FunctionRun => self.heap.function_run(args), - BuiltinFunction::GetArgumentCount => self.heap.get_argument_count(args), - BuiltinFunction::IfElse => self.heap.if_else(args), - BuiltinFunction::IntAdd => self.heap.int_add(args), - BuiltinFunction::IntBitLength => self.heap.int_bit_length(args), - BuiltinFunction::IntBitwiseAnd => self.heap.int_bitwise_and(args), - BuiltinFunction::IntBitwiseOr => self.heap.int_bitwise_or(args), - BuiltinFunction::IntBitwiseXor => self.heap.int_bitwise_xor(args), - BuiltinFunction::IntCompareTo => self.heap.int_compare_to(args), - BuiltinFunction::IntDivideTruncating => self.heap.int_divide_truncating(args), - BuiltinFunction::IntModulo => self.heap.int_modulo(args), - BuiltinFunction::IntMultiply => self.heap.int_multiply(args), - BuiltinFunction::IntParse => self.heap.int_parse(args), - BuiltinFunction::IntRemainder => self.heap.int_remainder(args), - BuiltinFunction::IntShiftLeft => self.heap.int_shift_left(args), - BuiltinFunction::IntShiftRight => self.heap.int_shift_right(args), - BuiltinFunction::IntSubtract => self.heap.int_subtract(args), - BuiltinFunction::Print => self.heap.print(args), - BuiltinFunction::StructGet => self.heap.struct_get(args), - BuiltinFunction::StructGetKeys => self.heap.struct_get_keys(args), - BuiltinFunction::StructHasKey => self.heap.struct_has_key(args), - BuiltinFunction::TextCharacters => self.heap.text_characters(args), - BuiltinFunction::TextConcatenate => self.heap.text_concatenate(args), - BuiltinFunction::TextContains => self.heap.text_contains(args), - BuiltinFunction::TextEndsWith => self.heap.text_ends_with(args), - BuiltinFunction::TextGetRange => self.heap.text_get_range(args), - BuiltinFunction::TextIsEmpty => self.heap.text_is_empty(args), - BuiltinFunction::TextLength => self.heap.text_length(args), - BuiltinFunction::TextStartsWith => self.heap.text_starts_with(args), - BuiltinFunction::TextTrimEnd => self.heap.text_trim_end(args), - BuiltinFunction::TextTrimStart => self.heap.text_trim_start(args), - BuiltinFunction::TypeOf => self.heap.type_of(args), - } + let result = span!(Level::TRACE, "Running builtin").in_scope(|| match &builtin_function { + BuiltinFunction::ChannelCreate => self.heap.channel_create(args), + BuiltinFunction::ChannelSend => self.heap.channel_send(args), + BuiltinFunction::ChannelReceive => self.heap.channel_receive(args), + BuiltinFunction::Equals => self.heap.equals(args), + BuiltinFunction::FunctionRun => self.heap.function_run(args), + BuiltinFunction::GetArgumentCount => self.heap.get_argument_count(args), + BuiltinFunction::IfElse => self.heap.if_else(args), + BuiltinFunction::IntAdd => self.heap.int_add(args), + BuiltinFunction::IntBitLength => self.heap.int_bit_length(args), + BuiltinFunction::IntBitwiseAnd => self.heap.int_bitwise_and(args), + BuiltinFunction::IntBitwiseOr => self.heap.int_bitwise_or(args), + BuiltinFunction::IntBitwiseXor => self.heap.int_bitwise_xor(args), + BuiltinFunction::IntCompareTo => self.heap.int_compare_to(args), + BuiltinFunction::IntDivideTruncating => self.heap.int_divide_truncating(args), + BuiltinFunction::IntModulo => self.heap.int_modulo(args), + BuiltinFunction::IntMultiply => self.heap.int_multiply(args), + BuiltinFunction::IntParse => self.heap.int_parse(args), + BuiltinFunction::IntRemainder => self.heap.int_remainder(args), + BuiltinFunction::IntShiftLeft => self.heap.int_shift_left(args), + BuiltinFunction::IntShiftRight => self.heap.int_shift_right(args), + BuiltinFunction::IntSubtract => self.heap.int_subtract(args), + BuiltinFunction::Parallel => self.heap.parallel(args), + BuiltinFunction::Print => self.heap.print(args), + BuiltinFunction::StructGet => self.heap.struct_get(args), + BuiltinFunction::StructGetKeys => self.heap.struct_get_keys(args), + BuiltinFunction::StructHasKey => self.heap.struct_has_key(args), + BuiltinFunction::TextCharacters => self.heap.text_characters(args), + BuiltinFunction::TextConcatenate => self.heap.text_concatenate(args), + BuiltinFunction::TextContains => self.heap.text_contains(args), + BuiltinFunction::TextEndsWith => self.heap.text_ends_with(args), + BuiltinFunction::TextGetRange => self.heap.text_get_range(args), + BuiltinFunction::TextIsEmpty => self.heap.text_is_empty(args), + BuiltinFunction::TextLength => self.heap.text_length(args), + BuiltinFunction::TextStartsWith => self.heap.text_starts_with(args), + BuiltinFunction::TextTrimEnd => self.heap.text_trim_end(args), + BuiltinFunction::TextTrimStart => self.heap.text_trim_start(args), + BuiltinFunction::TypeOf => self.heap.type_of(args), }); match result { Ok(Return(value)) => self.data_stack.push(value), - Ok(DivergeControlFlow(closure_address)) => { - self.data_stack.push(closure_address); + Ok(DivergeControlFlow { closure }) => { + self.data_stack.push(closure); self.run_instruction(use_provider, Instruction::Call { num_args: 0 }); } + Ok(CreateChannel { capacity }) => self.status = Status::CreatingChannel { capacity }, + Ok(Send { channel, packet }) => self.status = Status::Sending { channel, packet }, + Ok(Receive { channel }) => self.status = Status::Receiving { channel }, Err(reason) => self.panic(reason), } } @@ -75,7 +77,10 @@ impl Vm { type BuiltinResult = Result; enum SuccessfulBehavior { Return(Pointer), - DivergeControlFlow(Pointer), + DivergeControlFlow { closure: Pointer }, + CreateChannel { capacity: Capacity }, + Send { channel: ChannelId, packet: Packet }, + Receive { channel: ChannelId }, } use SuccessfulBehavior::*; @@ -109,45 +114,36 @@ macro_rules! unpack_and_later_drop { }; } -impl Vm { +impl Heap { fn channel_create(&mut self, args: &[Pointer]) -> BuiltinResult { - unpack_and_later_drop!(self.heap, args, (capacity: Int), { - let id = self.next_internal_channel_id + 1; - self.next_internal_channel_id += 1; - self.internal_channels.insert( - id, - Channel::new(capacity.value.try_into().expect( + unpack_and_later_drop!(self, args, (capacity: Int), { + CreateChannel { + capacity: capacity.value.clone().try_into().expect( "you tried to create a channel with a capacity bigger than the maximum usize", - )), - ); - - let send_port = self.heap.create_send_port(id); - let receive_port = self.heap.create_receive_port(id); - Return(self.heap.create_list(&[send_port, receive_port])) + ), + } }) } fn channel_send(&mut self, args: &[Pointer]) -> BuiltinResult { - unpack_and_later_drop!(self.heap, args, (port: SendPort, packet: Any), { - if let Some(channel) = self.internal_channels.get(&port.channel) { - // An internal channel. - } else { - // An external channel. - let external_id = self.internal_to_external_channels[&port.channel]; - self.channel_operations - .entry(external_id) - .or_default() - .push(ChannelOperation::Send { packet: () }) + unpack_and_later_drop!(self, args, (port: SendPort, packet: Any), { + let mut heap = Heap::default(); + let value = self.clone_single_to_other_heap(&mut heap, packet.address); + Send { + channel: port.channel, + packet: Packet { heap, value }, } }) } fn channel_receive(&mut self, args: &[Pointer]) -> BuiltinResult { - todo!() + unpack_and_later_drop!(self, args, (port: ReceivePort), { + Receive { + channel: port.channel, + } + }) } -} -impl Heap { fn equals(&mut self, args: &[Pointer]) -> BuiltinResult { unpack_and_later_drop!(self, args, (a: Any, b: Any), { let is_equal = a.equals(self, &b); @@ -159,7 +155,9 @@ impl Heap { unpack_and_later_drop!(self, args, (closure: Closure), { closure.should_take_no_arguments()?; self.dup(closure.address); - DivergeControlFlow(closure.address) + DivergeControlFlow { + closure: closure.address, + } }) } @@ -177,7 +175,9 @@ impl Heap { { let closure_to_run = if *condition { &then } else { &else_ }.address; self.dup(closure_to_run); - DivergeControlFlow(closure_to_run) + DivergeControlFlow { + closure: closure_to_run, + } } ) } @@ -472,6 +472,26 @@ impl TryInto for Data { } } } +impl TryInto for Data { + type Error = String; + + fn try_into(self) -> Result { + match self { + Data::SendPort(port) => Ok(port), + _ => Err("a builtin function expected a send port".to_string()), + } + } +} +impl TryInto for Data { + type Error = String; + + fn try_into(self) -> Result { + match self { + Data::ReceivePort(port) => Ok(port), + _ => Err("a builtin function expected a receive port".to_string()), + } + } +} impl TryInto for Data { type Error = String; diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index 148ebe712..e0b844da2 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -10,19 +10,21 @@ use std::collections::VecDeque; /// enables buggy code that leaks memory. #[derive(Clone)] pub struct Channel { - pub capacity: usize, + pub capacity: Capacity, packets: VecDeque, } +pub type Capacity = usize; + /// A self-contained value that is sent over a channel. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Packet { - heap: Heap, - value: Pointer, + pub heap: Heap, + pub value: Pointer, } impl Channel { - pub fn new(capacity: usize) -> Self { + pub fn new(capacity: Capacity) -> Self { Self { capacity, packets: Default::default(), @@ -33,9 +35,12 @@ impl Channel { self.packets.len() == self.capacity } - pub fn send(&mut self, packet: Packet) { - assert!(!self.is_full()); + pub fn send(&mut self, packet: Packet) -> bool { + if self.is_full() { + return false; + } self.packets.push_back(packet); + return true; } pub fn receive(&mut self) -> Option { diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs new file mode 100644 index 000000000..6f3f2f48f --- /dev/null +++ b/compiler/src/vm/fiber.rs @@ -0,0 +1,436 @@ +use super::{ + channel::{Capacity, Packet}, + heap::{Builtin, ChannelId, Closure, Data, Heap, Pointer}, + tracer::{TraceEntry, Tracer}, + use_provider::UseProvider, +}; +use crate::{ + compiler::{hir::Id, lir::Instruction}, + module::Module, +}; +use itertools::Itertools; +use std::collections::HashMap; +use tracing::trace; + +const TRACE: bool = false; + +/// A fiber is one execution thread of a program. A fiber is always owned and +/// managed by a VM. A VM can own multiple fibers and run them concurrently. +#[derive(Clone)] +pub struct Fiber { + // Core functionality to run code. Fibers are stack-based machines that run + // instructions from a LIR. All values are stored on a heap. + pub status: Status, + next_instruction: InstructionPointer, + pub data_stack: Vec, + pub call_stack: Vec, + pub import_stack: Vec, + pub heap: Heap, + + // Debug stuff. This is not essential to a correct working of the fiber, but + // enables advanced functionality like stack traces or finding out who's + // fault a panic is. + pub tracer: Tracer, + pub fuzzable_closures: Vec<(Id, Pointer)>, + pub num_instructions_executed: usize, +} + +#[derive(Clone, Debug)] +pub enum Status { + Running, + CreatingChannel { capacity: Capacity }, + Sending { channel: ChannelId, packet: Packet }, + Receiving { channel: ChannelId }, + Done, + Panicked { reason: String }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct InstructionPointer { + /// Pointer to the closure object that is currently running code. + closure: Pointer, + + /// Index of the next instruction to run. + instruction: usize, +} +impl InstructionPointer { + fn null_pointer() -> Self { + Self { + closure: Pointer::null(), + instruction: 0, + } + } + fn start_of_closure(closure: Pointer) -> Self { + Self { + closure, + instruction: 0, + } + } +} + +pub struct TearDownResult { + pub heap: Heap, + pub result: Result, + pub fuzzable_closures: Vec<(Id, Pointer)>, + pub tracer: Tracer, +} + +impl Fiber { + fn new_with_heap(heap: Heap) -> Self { + Self { + status: Status::Done, + next_instruction: InstructionPointer::null_pointer(), + data_stack: vec![], + call_stack: vec![], + import_stack: vec![], + heap, + tracer: Tracer::default(), + fuzzable_closures: vec![], + num_instructions_executed: 0, + } + } + pub fn new_for_running_closure( + heap: Heap, + use_provider: &U, + closure: Pointer, + arguments: &[Pointer], + ) -> Self { + let mut fiber = Self::new_with_heap(heap); + + fiber.data_stack.extend(arguments); + fiber.data_stack.push(closure); + + fiber.status = Status::Running; + fiber.run_instruction( + use_provider, + Instruction::Call { + num_args: arguments.len(), + }, + ); + fiber + } + pub fn new_for_running_module_closure( + use_provider: &U, + closure: Closure, + ) -> Self { + assert_eq!(closure.captured.len(), 0, "Called start_module_closure with a closure that is not a module closure (it captures stuff)."); + assert_eq!(closure.num_args, 0, "Called start_module_closure with a closure that is not a module closure (it has arguments)."); + let mut heap = Heap::default(); + let closure = heap.create_closure(closure); + Self::new_for_running_closure(heap, use_provider, closure, &[]) + } + pub fn tear_down(mut self) -> TearDownResult { + let result = match self.status { + Status::Done => Ok(self.data_stack.pop().unwrap()), + Status::Panicked { reason } => Err(reason), + _ => panic!("Called `tear_down` on a fiber that's still running."), + }; + TearDownResult { + heap: self.heap, + result, + fuzzable_closures: self.fuzzable_closures, + tracer: self.tracer, + } + } + + pub fn status(&self) -> Status { + self.status.clone() + } + pub fn panic(&mut self, reason: String) { + self.status = Status::Panicked { reason }; + } + pub fn complete_channel_create(&mut self, channel: ChannelId) { + let send_port = self.heap.create_send_port(channel); + let receive_port = self.heap.create_receive_port(channel); + self.data_stack + .push(self.heap.create_list(&[send_port, receive_port])); + self.status = Status::Running; + } + pub fn complete_send(&mut self) { + self.data_stack.push(self.heap.create_nothing()); + self.status = Status::Running; + } + pub fn complete_receive(&mut self, packet: Packet) { + let address_in_local_heap = packet + .heap + .clone_single_to_other_heap(&mut self.heap, packet.value); + self.data_stack.push(address_in_local_heap); + self.status = Status::Running; + } + + fn get_from_data_stack(&self, offset: usize) -> Pointer { + self.data_stack[self.data_stack.len() - 1 - offset as usize] + } + + pub fn run(&mut self, use_provider: &U, mut num_instructions: usize) { + assert!( + matches!(self.status, Status::Running), + "Called Fiber::run on a fiber that is not ready to run." + ); + while matches!(self.status, Status::Running) && num_instructions > 0 { + num_instructions -= 1; + + let current_closure = self.heap.get(self.next_instruction.closure); + let current_body = if let Data::Closure(Closure { body, .. }) = ¤t_closure.data { + body + } else { + panic!("The instruction pointer points to a non-closure."); + }; + let instruction = current_body[self.next_instruction.instruction].clone(); + + if TRACE { + trace!( + "Data stack: {}", + self.data_stack + .iter() + .map(|it| it.format(&self.heap)) + .join(", ") + ); + trace!( + "Call stack: {}", + self.call_stack + .iter() + .map(|ip| format!("{}:{}", ip.closure, ip.instruction)) + .join(", ") + ); + trace!( + "Instruction pointer: {}:{}", + self.next_instruction.closure, + self.next_instruction.instruction + ); + trace!("Heap: {:?}", self.heap); + trace!("Running instruction: {instruction:?}"); + } + + self.next_instruction.instruction += 1; + self.run_instruction(use_provider, instruction); + self.num_instructions_executed += 1; + + if self.next_instruction == InstructionPointer::null_pointer() { + self.status = Status::Done; + } + } + } + pub fn run_instruction(&mut self, use_provider: &U, instruction: Instruction) { + match instruction { + Instruction::CreateInt(int) => { + let address = self.heap.create_int(int.into()); + self.data_stack.push(address); + } + Instruction::CreateText(text) => { + let address = self.heap.create_text(text); + self.data_stack.push(address); + } + Instruction::CreateSymbol(symbol) => { + let address = self.heap.create_symbol(symbol); + self.data_stack.push(address); + } + Instruction::CreateStruct { num_entries } => { + let mut key_value_addresses = vec![]; + for _ in 0..(2 * num_entries) { + key_value_addresses.push(self.data_stack.pop().unwrap()); + } + let mut entries = HashMap::new(); + for mut key_and_value in &key_value_addresses.into_iter().rev().chunks(2) { + let key = key_and_value.next().unwrap(); + let value = key_and_value.next().unwrap(); + assert_eq!(key_and_value.next(), None); + entries.insert(key, value); + } + let address = self.heap.create_struct(entries); + self.data_stack.push(address); + } + Instruction::CreateClosure { + num_args, + body, + captured, + } => { + let captured = captured + .iter() + .map(|offset| self.get_from_data_stack(*offset)) + .collect_vec(); + for address in &captured { + self.heap.dup(*address); + } + let address = self.heap.create_closure(Closure { + captured, + num_args, + body, + }); + self.data_stack.push(address); + } + Instruction::CreateBuiltin(builtin) => { + let address = self.heap.create_builtin(builtin); + self.data_stack.push(address); + } + Instruction::PushFromStack(offset) => { + let address = self.get_from_data_stack(offset); + self.heap.dup(address); + self.data_stack.push(address); + } + Instruction::PopMultipleBelowTop(n) => { + let top = self.data_stack.pop().unwrap(); + for _ in 0..n { + let address = self.data_stack.pop().unwrap(); + self.heap.drop(address); + } + self.data_stack.push(top); + } + Instruction::Call { num_args } => { + let closure_address = self.data_stack.pop().unwrap(); + let mut args = vec![]; + for _ in 0..num_args { + args.push(self.data_stack.pop().unwrap()); + } + args.reverse(); + + match self.heap.get(closure_address).data.clone() { + Data::Closure(Closure { + captured, + num_args: expected_num_args, + .. + }) => { + if num_args != expected_num_args { + self.panic(format!("Closure expects {expected_num_args} parameters, but you called it with {num_args} arguments.")); + return; + } + + self.call_stack.push(self.next_instruction); + self.data_stack.append(&mut captured.clone()); + for captured in captured { + self.heap.dup(captured); + } + self.data_stack.append(&mut args); + self.next_instruction = + InstructionPointer::start_of_closure(closure_address); + } + Data::Builtin(Builtin { function: builtin }) => { + self.heap.drop(closure_address); + self.run_builtin_function(use_provider, &builtin, &args); + } + _ => { + self.panic("you can only call closures and builtins".to_string()); + } + }; + } + Instruction::Return => { + self.heap.drop(self.next_instruction.closure); + let caller = self.call_stack.pop().unwrap(); + self.next_instruction = caller; + } + Instruction::UseModule { current_module } => { + let relative_path = self.data_stack.pop().unwrap(); + match self.use_module(use_provider, current_module, relative_path) { + Ok(()) => {} + Err(reason) => { + self.panic(reason); + } + } + } + Instruction::Needs => { + let reason = self.data_stack.pop().unwrap(); + let condition = self.data_stack.pop().unwrap(); + + let reason = match self.heap.get(reason).data.clone() { + Data::Text(reason) => reason.value, + _ => { + self.panic("you can only use text as the reason of a `needs`".to_string()); + return; + } + }; + + match self.heap.get(condition).data.clone() { + Data::Symbol(symbol) => match symbol.value.as_str() { + "True" => { + self.data_stack.push(self.heap.create_nothing()); + } + "False" => self.status = Status::Panicked { reason }, + _ => { + self.panic("Needs expects True or False as a symbol.".to_string()); + } + }, + _ => { + self.panic("Needs expects a boolean symbol.".to_string()); + } + } + } + Instruction::RegisterFuzzableClosure(id) => { + let closure = *self.data_stack.last().unwrap(); + if !matches!(self.heap.get(closure).data, Data::Closure(_)) { + panic!("Instruction RegisterFuzzableClosure executed, but stack top is not a closure."); + } + self.heap.dup(closure); + self.fuzzable_closures.push((id, closure)); + } + Instruction::TraceValueEvaluated(id) => { + let value = *self.data_stack.last().unwrap(); + self.heap.dup(value); + self.tracer.push(TraceEntry::ValueEvaluated { id, value }); + } + Instruction::TraceCallStarts { id, num_args } => { + let closure = *self.data_stack.last().unwrap(); + self.heap.dup(closure); + + let mut args = vec![]; + let stack_size = self.data_stack.len(); + for i in 0..num_args { + let argument = self.data_stack[stack_size - i - 2]; + self.heap.dup(argument); + args.push(argument); + } + args.reverse(); + + self.tracer + .push(TraceEntry::CallStarted { id, closure, args }); + } + Instruction::TraceCallEnds => { + let return_value = *self.data_stack.last().unwrap(); + self.heap.dup(return_value); + self.tracer.push(TraceEntry::CallEnded { return_value }); + } + Instruction::TraceNeedsStarts { id } => { + let condition = self.data_stack[self.data_stack.len() - 1]; + let reason = self.data_stack[self.data_stack.len() - 2]; + self.heap.dup(condition); + self.heap.dup(reason); + self.tracer.push(TraceEntry::NeedsStarted { + id, + condition, + reason, + }); + } + Instruction::TraceNeedsEnds => self.tracer.push(TraceEntry::NeedsEnded), + Instruction::TraceModuleStarts { module } => { + if self.import_stack.contains(&module) { + self.panic(format!( + "there's an import cycle ({})", + self.import_stack + .iter() + .skip_while(|it| **it != module) + .chain([&module]) + .map(|module| format!("{module}")) + .join(" → "), + )); + } + self.import_stack.push(module.clone()); + self.tracer.push(TraceEntry::ModuleStarted { module }); + } + Instruction::TraceModuleEnds => { + self.import_stack.pop().unwrap(); + let export_map = *self.data_stack.last().unwrap(); + self.heap.dup(export_map); + self.tracer.push(TraceEntry::ModuleEnded { export_map }) + } + Instruction::Error { id, errors } => { + self.panic(format!( + "The fiber crashed because there {} at {id}: {errors:?}", + if errors.len() == 1 { + "was an error" + } else { + "were errors" + } + )); + } + } + } +} diff --git a/compiler/src/vm/heap/mod.rs b/compiler/src/vm/heap/mod.rs index e72983230..603d8730d 100644 --- a/compiler/src/vm/heap/mod.rs +++ b/compiler/src/vm/heap/mod.rs @@ -125,7 +125,6 @@ impl Heap { &self, other: &mut Heap, addresses: &[Pointer], - channel_map: &HashMap, ) -> Vec { let mut objects_to_refcounts = HashMap::new(); for address in addresses { @@ -147,7 +146,7 @@ impl Heap { address_map[&address], Object { reference_count: refcount, - data: Self::map_data(&address_map, channel_map, &self.get(address).data), + data: Self::map_data(&address_map, &self.get(address).data), }, ); } @@ -168,11 +167,7 @@ impl Heap { self.gather_objects_to_clone(objects_to_refcounts, child); } } - fn map_data( - address_map: &HashMap, - channel_map: &HashMap, - data: &Data, - ) -> Data { + fn map_data(address_map: &HashMap, data: &Data) -> Data { match data { Data::Int(int) => Data::Int(int.clone()), Data::Text(text) => Data::Text(text.clone()), @@ -194,19 +189,12 @@ impl Heap { body: closure.body.clone(), }), Data::Builtin(builtin) => Data::Builtin(builtin.clone()), - Data::SendPort(port) => Data::SendPort(SendPort::new(channel_map[&port.channel])), - Data::ReceivePort(port) => { - Data::ReceivePort(ReceivePort::new(channel_map[&port.channel])) - } + Data::SendPort(port) => Data::SendPort(SendPort::new(port.channel)), + Data::ReceivePort(port) => Data::ReceivePort(ReceivePort::new(port.channel)), } } - pub fn clone_single_to_other_heap( - &self, - other: &mut Heap, - address: Pointer, - channel_map: &HashMap, - ) -> Pointer { - self.clone_multiple_to_other_heap(other, &[address], channel_map) + pub fn clone_single_to_other_heap(&self, other: &mut Heap, address: Pointer) -> Pointer { + self.clone_multiple_to_other_heap(other, &[address]) .pop() .unwrap() } diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index af6ab459d..97c702515 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -1,9 +1,11 @@ mod builtin_functions; mod channel; +mod fiber; mod heap; pub mod tracer; pub mod use_provider; -mod vm; +pub mod vm; +pub use fiber::{Fiber, TearDownResult}; pub use heap::{Closure, Heap, Object, Pointer}; -pub use vm::{Status, TearDownResult, Vm}; +pub use vm::Vm; diff --git a/compiler/src/vm/use_provider.rs b/compiler/src/vm/use_provider.rs index 969164746..061675e74 100644 --- a/compiler/src/vm/use_provider.rs +++ b/compiler/src/vm/use_provider.rs @@ -1,6 +1,6 @@ use super::{ + fiber::Fiber, heap::{Closure, Data, Heap, Pointer}, - Vm, }; use crate::{ compiler::{ @@ -38,7 +38,7 @@ impl<'a> UseProvider for DbUseProvider<'a> { } } -impl Vm { +impl Fiber { pub fn use_module( &mut self, use_provider: &U, diff --git a/compiler/src/vm/vm.rs b/compiler/src/vm/vm.rs index 6ed2337e3..f553f19fc 100644 --- a/compiler/src/vm/vm.rs +++ b/compiler/src/vm/vm.rs @@ -1,33 +1,22 @@ use super::{ channel::{Channel, Packet}, - heap::{Builtin, ChannelId, Closure, Data, Heap, Pointer}, - tracer::{TraceEntry, Tracer}, + fiber::Fiber, + heap::ChannelId, + tracer::Tracer, use_provider::{DbUseProvider, UseProvider}, + Closure, Heap, Pointer, TearDownResult, }; -use crate::{ - compiler::{hir::Id, lir::Instruction}, - database::Database, - module::Module, -}; -use itertools::Itertools; +use crate::{database::Database, vm::fiber}; use std::collections::{HashMap, VecDeque}; -use tracing::{info, trace}; - -const TRACE: bool = false; +use tracing::{debug, info, trace}; /// A VM is a Candy program that thinks it's currently running. Because VMs are /// first-class Rust structs, they enable other code to store "freezed" programs /// and to remain in control about when and for how long they run. #[derive(Clone)] pub struct Vm { - // Core functionality to run code. VMs are stack-based machines that run - // instructions from a LIR. All values are stored on a heap. - pub status: Status, - next_instruction: InstructionPointer, - pub data_stack: Vec, - pub call_stack: Vec, - pub import_stack: Vec, - pub heap: Heap, + status: Status, + pub fiber: Fiber, // Channel functionality. VMs communicate with the outer world using // channels. Each channel is identified using an ID that is valid inside @@ -40,13 +29,6 @@ pub struct Vm { pub external_to_internal_channels: HashMap, pub internal_to_external_channels: HashMap, pub channel_operations: HashMap>, - - // Debug properties. These are not essential to a correct working of the VM, - // but they enable advanced functionality like stack traces or finding out - // who's fault a panic is. - pub tracer: Tracer, - pub fuzzable_closures: Vec<(Id, Pointer)>, - pub num_instructions_executed: usize, } #[derive(Clone, Debug)] @@ -56,59 +38,22 @@ pub enum Status { Panicked { reason: String }, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct InstructionPointer { - /// Pointer to the closure object that is currently running code. - closure: Pointer, - - /// Index of the next instruction to run. - instruction: usize, -} -impl InstructionPointer { - fn null_pointer() -> Self { - Self { - closure: Pointer::null(), - instruction: 0, - } - } - fn start_of_closure(closure: Pointer) -> Self { - Self { - closure, - instruction: 0, - } - } -} - #[derive(Clone)] pub enum ChannelOperation { Send { packet: Packet }, Receive, } -pub struct TearDownResult { - pub heap: Heap, - pub result: Result, - pub fuzzable_closures: Vec<(Id, Pointer)>, - pub tracer: Tracer, -} - impl Vm { - fn new_with_heap(heap: Heap) -> Self { + fn new_with_fiber(fiber: Fiber) -> Self { Self { - status: Status::Done, - next_instruction: InstructionPointer::null_pointer(), - data_stack: vec![], - call_stack: vec![], - import_stack: vec![], - heap, - internal_channels: HashMap::new(), + status: Status::Running, + fiber, + internal_channels: Default::default(), next_internal_channel_id: 0, - external_to_internal_channels: HashMap::new(), - internal_to_external_channels: HashMap::new(), + external_to_internal_channels: Default::default(), + internal_to_external_channels: Default::default(), channel_operations: Default::default(), - tracer: Tracer::default(), - fuzzable_closures: vec![], - num_instructions_executed: 0, } } pub fn new_for_running_closure( @@ -117,50 +62,31 @@ impl Vm { closure: Pointer, arguments: &[Pointer], ) -> Self { - let mut vm = Self::new_with_heap(heap); - - vm.data_stack.extend(arguments); - vm.data_stack.push(closure); - - vm.status = Status::Running; - vm.run_instruction( + Self::new_with_fiber(Fiber::new_for_running_closure( + heap, use_provider, - Instruction::Call { - num_args: arguments.len(), - }, - ); - vm + closure, + arguments, + )) } pub fn new_for_running_module_closure( use_provider: &U, closure: Closure, ) -> Self { - assert_eq!(closure.captured.len(), 0, "Called start_module_closure with a closure that is not a module closure (it captures stuff)."); - assert_eq!(closure.num_args, 0, "Called start_module_closure with a closure that is not a module closure (it has arguments)."); - let mut heap = Heap::default(); - let closure = heap.create_closure(closure); - Self::new_for_running_closure(heap, use_provider, closure, &[]) + Self::new_with_fiber(Fiber::new_for_running_module_closure(use_provider, closure)) } - pub fn tear_down(mut self) -> TearDownResult { - let result = match self.status { - Status::Running => panic!("Called `tear_down` on a VM that's still running."), - Status::Done => Ok(self.data_stack.pop().unwrap()), - Status::Panicked { reason } => Err(reason), - }; - TearDownResult { - heap: self.heap, - result, - fuzzable_closures: self.fuzzable_closures, - tracer: self.tracer, - } + pub fn tear_down(self) -> TearDownResult { + self.fiber.tear_down() } pub fn status(&self) -> Status { self.status.clone() } - - fn get_from_data_stack(&self, offset: usize) -> Pointer { - self.data_stack[self.data_stack.len() - 1 - offset as usize] + pub fn cloned_tracer(&self) -> Tracer { + self.fiber.tracer.clone() + } + pub fn num_instructions_executed(&self) -> usize { + self.fiber.num_instructions_executed } pub fn run(&mut self, use_provider: &U, mut num_instructions: usize) { @@ -168,277 +94,54 @@ impl Vm { matches!(self.status, Status::Running), "Called Vm::run on a vm that is not ready to run." ); - while matches!(self.status, Status::Running) && num_instructions > 0 { - num_instructions -= 1; - - let current_closure = self.heap.get(self.next_instruction.closure); - let current_body = if let Data::Closure(Closure { body, .. }) = ¤t_closure.data { - body - } else { - panic!("The instruction pointer points to a non-closure."); - }; - let instruction = current_body[self.next_instruction.instruction].clone(); - - if TRACE { - trace!( - "Data stack: {}", - self.data_stack - .iter() - .map(|it| it.format(&self.heap)) - .join(", ") - ); - trace!( - "Call stack: {}", - self.call_stack - .iter() - .map(|ip| format!("{}:{}", ip.closure, ip.instruction)) - .join(", ") - ); - trace!( - "Instruction pointer: {}:{}", - self.next_instruction.closure, - self.next_instruction.instruction - ); - trace!("Heap: {:?}", self.heap); - trace!("Running instruction: {instruction:?}"); - } - - self.next_instruction.instruction += 1; - self.run_instruction(use_provider, instruction); - self.num_instructions_executed += 1; - - if self.next_instruction == InstructionPointer::null_pointer() { - self.status = Status::Done; - } - } - } - pub fn run_instruction(&mut self, use_provider: &U, instruction: Instruction) { - match instruction { - Instruction::CreateInt(int) => { - let address = self.heap.create_int(int.into()); - self.data_stack.push(address); - } - Instruction::CreateText(text) => { - let address = self.heap.create_text(text); - self.data_stack.push(address); - } - Instruction::CreateSymbol(symbol) => { - let address = self.heap.create_symbol(symbol); - self.data_stack.push(address); - } - Instruction::CreateStruct { num_entries } => { - let mut key_value_addresses = vec![]; - for _ in 0..(2 * num_entries) { - key_value_addresses.push(self.data_stack.pop().unwrap()); - } - let mut entries = HashMap::new(); - for mut key_and_value in &key_value_addresses.into_iter().rev().chunks(2) { - let key = key_and_value.next().unwrap(); - let value = key_and_value.next().unwrap(); - assert_eq!(key_and_value.next(), None); - entries.insert(key, value); - } - let address = self.heap.create_struct(entries); - self.data_stack.push(address); - } - Instruction::CreateClosure { - num_args, - body, - captured, - } => { - let captured = captured - .iter() - .map(|offset| self.get_from_data_stack(*offset)) - .collect_vec(); - for address in &captured { - self.heap.dup(*address); - } - let address = self.heap.create_closure(Closure { - captured, - num_args, - body, - }); - self.data_stack.push(address); - } - Instruction::CreateBuiltin(builtin) => { - let address = self.heap.create_builtin(builtin); - self.data_stack.push(address); - } - Instruction::PushFromStack(offset) => { - let address = self.get_from_data_stack(offset); - self.heap.dup(address); - self.data_stack.push(address); - } - Instruction::PopMultipleBelowTop(n) => { - let top = self.data_stack.pop().unwrap(); - for _ in 0..n { - let address = self.data_stack.pop().unwrap(); - self.heap.drop(address); - } - self.data_stack.push(top); - } - Instruction::Call { num_args } => { - let closure_address = self.data_stack.pop().unwrap(); - let mut args = vec![]; - for _ in 0..num_args { - args.push(self.data_stack.pop().unwrap()); + loop { + debug!("Running fiber (status = {:?}).", self.fiber.status); + self.fiber.run(use_provider, num_instructions); + match self.fiber.status() { + fiber::Status::Running => {} + fiber::Status::CreatingChannel { capacity } => { + let id = self.next_internal_channel_id; + self.next_internal_channel_id += 1; + self.internal_channels.insert(id, Channel::new(capacity)); + self.fiber.complete_channel_create(id); } - args.reverse(); - - match self.heap.get(closure_address).data.clone() { - Data::Closure(Closure { - captured, - num_args: expected_num_args, - .. - }) => { - if num_args != expected_num_args { - self.panic(format!("Closure expects {expected_num_args} parameters, but you called it with {num_args} arguments.")); - return; - } - - self.call_stack.push(self.next_instruction); - self.data_stack.append(&mut captured.clone()); - for captured in captured { - self.heap.dup(captured); + fiber::Status::Sending { channel, packet } => { + if let Some(channel) = self.internal_channels.get_mut(&channel) { + // Internal channel. + if channel.send(packet) { + self.fiber.complete_send(); + } else { + panic!("Sent to internal full channel. Deadlock."); } - self.data_stack.append(&mut args); - self.next_instruction = - InstructionPointer::start_of_closure(closure_address); - } - Data::Builtin(Builtin { function: builtin }) => { - self.heap.drop(closure_address); - self.run_builtin_function(use_provider, &builtin, &args); - } - _ => { - self.panic("you can only call closures and builtins".to_string()); - } - }; - } - Instruction::Return => { - self.heap.drop(self.next_instruction.closure); - let caller = self.call_stack.pop().unwrap(); - self.next_instruction = caller; - } - Instruction::UseModule { current_module } => { - let relative_path = self.data_stack.pop().unwrap(); - match self.use_module(use_provider, current_module, relative_path) { - Ok(()) => {} - Err(reason) => { - self.panic(reason); + } else { + // External channel. + todo!() } } - } - Instruction::Needs => { - let reason = self.data_stack.pop().unwrap(); - let condition = self.data_stack.pop().unwrap(); - - let reason = match self.heap.get(reason).data.clone() { - Data::Text(reason) => reason.value, - _ => { - self.panic("you can only use text as the reason of a `needs`".to_string()); - return; - } - }; - - match self.heap.get(condition).data.clone() { - Data::Symbol(symbol) => match symbol.value.as_str() { - "True" => { - self.data_stack.push(self.heap.create_nothing()); - } - "False" => self.status = Status::Panicked { reason }, - _ => { - self.panic("Needs expects True or False as a symbol.".to_string()); + fiber::Status::Receiving { channel } => { + if let Some(channel) = self.internal_channels.get_mut(&channel) { + // Internal channel. + if let Some(packet) = channel.receive() { + self.fiber.complete_receive(packet); + } else { + panic!("Tried to receive from internal empty channel. Deadlock."); } - }, - _ => { - self.panic("Needs expects a boolean symbol.".to_string()); + } else { + // External channel. + todo!() } } - } - Instruction::RegisterFuzzableClosure(id) => { - let closure = *self.data_stack.last().unwrap(); - if !matches!(self.heap.get(closure).data, Data::Closure(_)) { - panic!("Instruction RegisterFuzzableClosure executed, but stack top is not a closure."); + fiber::Status::Done => { + self.status = Status::Done; + break; } - self.heap.dup(closure); - self.fuzzable_closures.push((id, closure)); - } - Instruction::TraceValueEvaluated(id) => { - let value = *self.data_stack.last().unwrap(); - self.heap.dup(value); - self.tracer.push(TraceEntry::ValueEvaluated { id, value }); - } - Instruction::TraceCallStarts { id, num_args } => { - let closure = *self.data_stack.last().unwrap(); - self.heap.dup(closure); - - let mut args = vec![]; - let stack_size = self.data_stack.len(); - for i in 0..num_args { - let argument = self.data_stack[stack_size - i - 2]; - self.heap.dup(argument); - args.push(argument); + fiber::Status::Panicked { reason } => { + self.status = Status::Panicked { reason }; + break; } - args.reverse(); - - self.tracer - .push(TraceEntry::CallStarted { id, closure, args }); - } - Instruction::TraceCallEnds => { - let return_value = *self.data_stack.last().unwrap(); - self.heap.dup(return_value); - self.tracer.push(TraceEntry::CallEnded { return_value }); - } - Instruction::TraceNeedsStarts { id } => { - let condition = self.data_stack[self.data_stack.len() - 1]; - let reason = self.data_stack[self.data_stack.len() - 2]; - self.heap.dup(condition); - self.heap.dup(reason); - self.tracer.push(TraceEntry::NeedsStarted { - id, - condition, - reason, - }); - } - Instruction::TraceNeedsEnds => self.tracer.push(TraceEntry::NeedsEnded), - Instruction::TraceModuleStarts { module } => { - if self.import_stack.contains(&module) { - self.panic(format!( - "there's an import cycle ({})", - self.import_stack - .iter() - .skip_while(|it| **it != module) - .chain([&module]) - .map(|module| format!("{module}")) - .join(" → "), - )); - } - self.import_stack.push(module.clone()); - self.tracer.push(TraceEntry::ModuleStarted { module }); - } - Instruction::TraceModuleEnds => { - self.import_stack.pop().unwrap(); - let export_map = *self.data_stack.last().unwrap(); - self.heap.dup(export_map); - self.tracer.push(TraceEntry::ModuleEnded { export_map }) - } - Instruction::Error { id, errors } => { - self.panic(format!( - "The VM crashed because there {} at {id}: {errors:?}", - if errors.len() == 1 { - "was an error" - } else { - "were errors" - } - )); } } } - - pub fn panic(&mut self, reason: String) { - self.status = Status::Panicked { reason }; - } - pub fn run_synchronously_until_completion(mut self, db: &Database) -> TearDownResult { let use_provider = DbUseProvider { db }; loop { diff --git a/packages/Core/.candy b/packages/Core/.candy index 7e529b92d..7995e406f 100644 --- a/packages/Core/.candy +++ b/packages/Core/.candy @@ -1,6 +1,10 @@ bool := use ".Bool" +channel := (use ".Channel") check := (use ".Check").check +concurrency = use ".Concurrency" +parallel := concurrency.parallel + conditionals = use ".Conditionals" if := conditionals.if ifElse := conditionals.ifElse diff --git a/packages/Core/Channel.candy b/packages/Core/Channel.candy new file mode 100644 index 000000000..3b4794558 --- /dev/null +++ b/packages/Core/Channel.candy @@ -0,0 +1,18 @@ +isInt = (use "..Int").is +equals = (use "..Equality").equals +isType = (use "..Type").is + +isSendPort value := isType value SendPort +isReceivePort value := isType value ReceivePort + +create capacity := + needs (isInt capacity) + ✨.channelCreate capacity + +send port packet := + needs (isSendPort port) "send needs a send port" + ✨.channelSend port packet + +receive port := + needs (isReceivePort port) "receive needs a receive port" + ✨.channelReceive port diff --git a/packages/Core/Concurrency.candy b/packages/Core/Concurrency.candy new file mode 100644 index 000000000..e2a39e845 --- /dev/null +++ b/packages/Core/Concurrency.candy @@ -0,0 +1,5 @@ +isFunction1 = (use "..Function").is1 + +parallel body := + needs (isFunction1 body) + ✨.parallel body From ea50a6e5e4f527a87791b9cbe664673ab6b949b7 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Fri, 19 Aug 2022 21:37:01 +0200 Subject: [PATCH 06/59] Start implementing fibers --- compiler/src/builtin_functions.rs | 1 + compiler/src/fuzzer/fuzzer.rs | 17 +- .../hints/constant_evaluator.rs | 32 +- compiler/src/vm/builtin_functions.rs | 11 + compiler/src/vm/fiber.rs | 10 +- compiler/src/vm/vm.rs | 301 +++++++++++++++--- packages/Core/.candy | 2 + packages/Core/Concurrency.candy | 15 +- 8 files changed, 317 insertions(+), 72 deletions(-) diff --git a/compiler/src/builtin_functions.rs b/compiler/src/builtin_functions.rs index f76261123..223b5d23f 100644 --- a/compiler/src/builtin_functions.rs +++ b/compiler/src/builtin_functions.rs @@ -25,6 +25,7 @@ pub enum BuiltinFunction { IntShiftLeft, // (value: int) (amount: int) -> (shifted: int) IntShiftRight, // (value: int) (amount: int) -> (shifted: int) IntSubtract, // (minuend: int) (subtrahend: int) -> (difference: int) + Parallel, // bodyClosureTakingNursery -> resultOfClosure Print, // message -> Nothing StructGet, // struct key -> value StructGetKeys, // struct -> listOfKeys diff --git a/compiler/src/fuzzer/fuzzer.rs b/compiler/src/fuzzer/fuzzer.rs index 44d7c86e3..49a5d6586 100644 --- a/compiler/src/fuzzer/fuzzer.rs +++ b/compiler/src/fuzzer/fuzzer.rs @@ -2,7 +2,7 @@ use super::{generator::generate_n_values, utils::did_need_in_closure_cause_panic use crate::{ compiler::hir, database::Database, - vm::{self, tracer::Tracer, use_provider::DbUseProvider, Closure, Heap, Pointer, Vm}, + vm::{tracer::Tracer, use_provider::DbUseProvider, vm, Closure, Heap, Pointer, Vm}, }; use std::mem; @@ -89,18 +89,19 @@ impl Fuzzer { num_instructions: usize, ) -> (Status, usize) { match status { - Status::StillFuzzing { mut vm, arguments } => match &vm.status { + Status::StillFuzzing { mut vm, arguments } => match vm.status() { vm::Status::Running => { let use_provider = DbUseProvider { db }; - let num_instructions_executed_before = vm.num_instructions_executed; + let num_instructions_executed_before = vm.num_instructions_executed(); vm.run(&use_provider, num_instructions); let num_instruction_executed = - vm.num_instructions_executed - num_instructions_executed_before; + vm.num_instructions_executed() - num_instructions_executed_before; ( Status::StillFuzzing { vm, arguments }, num_instruction_executed, ) } + vm::Status::WaitingForOperations => panic!("Fuzzing should not have to wait on channel operations because arguments were not channels."), // The VM finished running without panicking. vm::Status::Done => ( Status::new_fuzzing_attempt(db, &self.closure_heap, self.closure), @@ -111,15 +112,15 @@ impl Fuzzer { // satisfied, then the panic is not closure's fault, but our // fault. let is_our_fault = - did_need_in_closure_cause_panic(db, &self.closure_id, &vm.tracer); + did_need_in_closure_cause_panic(db, &self.closure_id, &vm.cloned_tracer()); let status = if is_our_fault { Status::new_fuzzing_attempt(db, &self.closure_heap, self.closure) } else { Status::PanickedForArguments { - heap: vm.heap, + heap: vm.fiber().heap.clone(), arguments, - reason: reason.clone(), - tracer: vm.tracer.clone(), + reason, + tracer: vm.fiber().tracer.clone(), } }; (status, 0) diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index f3be600b9..eaf1b0135 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -9,7 +9,7 @@ use crate::{ database::Database, language_server::hints::{utils::id_to_end_of_line, HintKind}, module::Module, - vm::{tracer::TraceEntry, use_provider::DbUseProvider, Closure, Heap, Pointer, Status, Vm}, + vm::{tracer::TraceEntry, use_provider::DbUseProvider, vm, Closure, Heap, Pointer, Vm}, }; use itertools::Itertools; use rand::{prelude::SliceRandom, thread_rng}; @@ -39,7 +39,7 @@ impl ConstantEvaluator { let mut running_vms = self .vms .iter_mut() - .filter(|(_, vm)| matches!(vm.status(), Status::Running)) + .filter(|(_, vm)| matches!(vm.status(), vm::Status::Running)) .collect_vec(); trace!( "Constant evaluator running. {} running VMs, {} in total.", @@ -59,8 +59,9 @@ impl ConstantEvaluator { pub fn get_fuzzable_closures(&self, module: &Module) -> (&Heap, Vec<(Id, Pointer)>) { let vm = &self.vms[module]; ( - &vm.heap, - vm.fuzzable_closures + &vm.fiber().heap, + vm.fiber() + .fuzzable_closures .iter() .filter(|(id, _)| &id.module == module) .cloned() @@ -75,16 +76,19 @@ impl ConstantEvaluator { let vm = &self.vms[module]; let mut hints = vec![]; - if let Status::Panicked { reason } = vm.status() { + if let vm::Status::Panicked { reason } = vm.status() { if let Some(hint) = panic_hint(db, module.clone(), vm, reason) { hints.push(hint); } }; if module.to_possible_paths().is_some() { - module.dump_associated_debug_file("trace", &vm.tracer.format_call_tree(&vm.heap)); + module.dump_associated_debug_file( + "trace", + &vm.fiber().tracer.format_call_tree(&vm.fiber().heap), + ); } - for entry in vm.tracer.log() { + for entry in vm.cloned_tracer().log() { let (id, value) = match entry { TraceEntry::ValueEvaluated { id, value } => { if &id.module != module { @@ -112,7 +116,7 @@ impl ConstantEvaluator { hints.push(Hint { kind: HintKind::Value, - text: value.format(&vm.heap), + text: value.format(&vm.fiber().heap), position: id_to_end_of_line(db, id).unwrap(), }); } @@ -125,7 +129,7 @@ fn panic_hint(db: &Database, module: Module, vm: &Vm, reason: String) -> Option< // We want to show the hint at the last call site still inside the current // module. If there is no call site in this module, then the panic results // from a compiler error in a previous stage which is already reported. - let stack = vm.tracer.stack(); + let stack = vm.fiber().tracer.stack(); if stack.len() == 1 { // The stack only contains a `ModuleStarted` entry. This indicates an // error during compilation resulting in a top-level error instruction. @@ -148,8 +152,10 @@ fn panic_hint(db: &Database, module: Module, vm: &Vm, reason: String) -> Option< id, format!( "{} {}", - closure.format(&vm.heap), - args.iter().map(|arg| arg.format(&vm.heap)).join(" ") + closure.format(&vm.fiber().heap), + args.iter() + .map(|arg| arg.format(&vm.fiber().heap)) + .join(" ") ), ), TraceEntry::NeedsStarted { @@ -160,8 +166,8 @@ fn panic_hint(db: &Database, module: Module, vm: &Vm, reason: String) -> Option< id, format!( "needs {} {}", - condition.format(&vm.heap), - reason.format(&vm.heap) + condition.format(&vm.fiber().heap), + reason.format(&vm.fiber().heap) ), ), _ => unreachable!(), diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index 62f8e4f70..2c5ddce53 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -69,6 +69,7 @@ impl Fiber { Ok(CreateChannel { capacity }) => self.status = Status::CreatingChannel { capacity }, Ok(Send { channel, packet }) => self.status = Status::Sending { channel, packet }, Ok(Receive { channel }) => self.status = Status::Receiving { channel }, + Ok(Parallel { body }) => self.status = Status::InParallelScope { body }, Err(reason) => self.panic(reason), } } @@ -81,6 +82,7 @@ enum SuccessfulBehavior { CreateChannel { capacity: Capacity }, Send { channel: ChannelId, packet: Packet }, Receive { channel: ChannelId }, + Parallel { body: Pointer }, } use SuccessfulBehavior::*; @@ -260,6 +262,15 @@ impl Heap { }) } + fn parallel(&mut self, args: &[Pointer]) -> BuiltinResult { + unpack_and_later_drop!(self, args, (body_taking_nursery: Closure), { + self.dup(body_taking_nursery.address); + Parallel { + body: body_taking_nursery.address, + } + }) + } + fn print(&mut self, args: &[Pointer]) -> BuiltinResult { unpack_and_later_drop!(self, args, (message: Text), { info!("{:?}", message.value); diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index 6f3f2f48f..086aacadb 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -41,6 +41,7 @@ pub enum Status { CreatingChannel { capacity: Capacity }, Sending { channel: ChannelId, packet: Packet }, Receiving { channel: ChannelId }, + InParallelScope { body: Pointer }, Done, Panicked { reason: String }, } @@ -151,10 +152,15 @@ impl Fiber { self.status = Status::Running; } pub fn complete_receive(&mut self, packet: Packet) { - let address_in_local_heap = packet + let address = packet .heap .clone_single_to_other_heap(&mut self.heap, packet.value); - self.data_stack.push(address_in_local_heap); + self.data_stack.push(address); + self.status = Status::Running; + } + pub fn complete_parallel_scope(&mut self, heap: &mut Heap, return_value: Pointer) { + let return_value = heap.clone_single_to_other_heap(&mut self.heap, return_value); + self.data_stack.push(return_value); self.status = Status::Running; } diff --git a/compiler/src/vm/vm.rs b/compiler/src/vm/vm.rs index f553f19fc..5ccd0731f 100644 --- a/compiler/src/vm/vm.rs +++ b/compiler/src/vm/vm.rs @@ -7,8 +7,13 @@ use super::{ Closure, Heap, Pointer, TearDownResult, }; use crate::{database::Database, vm::fiber}; -use std::collections::{HashMap, VecDeque}; -use tracing::{debug, info, trace}; +use itertools::Itertools; +use rand::{seq::IteratorRandom, thread_rng}; +use std::{ + collections::{HashMap, VecDeque}, + mem, +}; +use tracing::{debug, error, info, trace, warn}; /// A VM is a Candy program that thinks it's currently running. Because VMs are /// first-class Rust structs, they enable other code to store "freezed" programs @@ -16,7 +21,7 @@ use tracing::{debug, info, trace}; #[derive(Clone)] pub struct Vm { status: Status, - pub fiber: Fiber, + state: Option, // Only `None` temporarily during state transitions. // Channel functionality. VMs communicate with the outer world using // channels. Each channel is identified using an ID that is valid inside @@ -34,10 +39,33 @@ pub struct Vm { #[derive(Clone, Debug)] pub enum Status { Running, + WaitingForOperations, Done, Panicked { reason: String }, } +#[derive(Clone)] +enum State { + SingleFiber(Fiber), + ParallelSection { + /// The main fiber of this VM. Should have Status::InParallelSection. + paused_main_fiber: Fiber, + + /// The channel that you can send spawn commands to. + nursery: ChannelId, + + /// The VM for the closure with which `core.parallel` was called. + parallel_body: Box, + + /// Spawned child VMs. For each VM, there also exists a channel that + /// will contain the result of the VM (once it's done or panicked). This + /// channel is directly returned by the `core.async` function. + /// Here, we save the ID of the channel where the result of the VM will + /// be sent. + spawned_children: Vec<(ChannelId, Vm)>, + }, +} + #[derive(Clone)] pub enum ChannelOperation { Send { packet: Packet }, @@ -48,7 +76,7 @@ impl Vm { fn new_with_fiber(fiber: Fiber) -> Self { Self { status: Status::Running, - fiber, + state: Some(State::SingleFiber(fiber)), internal_channels: Default::default(), next_internal_channel_id: 0, external_to_internal_channels: Default::default(), @@ -76,17 +104,39 @@ impl Vm { Self::new_with_fiber(Fiber::new_for_running_module_closure(use_provider, closure)) } pub fn tear_down(self) -> TearDownResult { - self.fiber.tear_down() + match self.into_state() { + State::SingleFiber(fiber) => fiber.tear_down(), + State::ParallelSection { .. } => { + panic!("Called `Vm::tear_down` while in parallel scope") + } + } } pub fn status(&self) -> Status { self.status.clone() } + pub fn fiber(&self) -> &Fiber { + match self.state() { + State::SingleFiber(fiber) => fiber, + State::ParallelSection { + paused_main_fiber, .. + } => paused_main_fiber, + } + } pub fn cloned_tracer(&self) -> Tracer { - self.fiber.tracer.clone() + self.fiber().tracer.clone() } pub fn num_instructions_executed(&self) -> usize { - self.fiber.num_instructions_executed + self.fiber().num_instructions_executed + } + fn into_state(self) -> State { + self.state + .expect("Tried to get VM state during state transition") + } + fn state(&self) -> &State { + self.state + .as_ref() + .expect("Tried to get VM state during state transition") } pub fn run(&mut self, use_provider: &U, mut num_instructions: usize) { @@ -94,53 +144,204 @@ impl Vm { matches!(self.status, Status::Running), "Called Vm::run on a vm that is not ready to run." ); - loop { - debug!("Running fiber (status = {:?}).", self.fiber.status); - self.fiber.run(use_provider, num_instructions); - match self.fiber.status() { - fiber::Status::Running => {} - fiber::Status::CreatingChannel { capacity } => { - let id = self.next_internal_channel_id; - self.next_internal_channel_id += 1; - self.internal_channels.insert(id, Channel::new(capacity)); - self.fiber.complete_channel_create(id); - } - fiber::Status::Sending { channel, packet } => { - if let Some(channel) = self.internal_channels.get_mut(&channel) { - // Internal channel. - if channel.send(packet) { - self.fiber.complete_send(); - } else { - panic!("Sent to internal full channel. Deadlock."); + let mut state = mem::replace(&mut self.state, None).unwrap(); + while matches!(self.status, Status::Running) && num_instructions > 0 { + let (new_state, num_instructions_executed) = + self.run_and_map_state(state, use_provider, num_instructions); + state = new_state; + + if num_instructions_executed >= num_instructions { + break; + } else { + num_instructions -= num_instructions_executed; + } + } + self.state = Some(state); + } + fn run_and_map_state( + &mut self, + state: State, + use_provider: &U, + num_instructions: usize, + ) -> (State, usize) { + let mut num_instructions_executed = 0; + let new_state = 'new_state: { + match state { + State::SingleFiber(mut fiber) => { + debug!("Running fiber (status = {:?}).", fiber.status); + + fiber.num_instructions_executed = 0; + fiber.run(use_provider, num_instructions); + num_instructions_executed = fiber.num_instructions_executed; + + match fiber.status() { + fiber::Status::Running => {} + fiber::Status::CreatingChannel { capacity } => { + let id = self.generate_channel_id(); + self.internal_channels.insert(id, Channel::new(capacity)); + fiber.complete_channel_create(id); + } + fiber::Status::Sending { channel, packet } => { + if let Some(channel) = self.internal_channels.get_mut(&channel) { + // Internal channel. + if channel.send(packet) { + fiber.complete_send(); + } else { + warn!("Tried to send to a full channel that is local to a fiber. This will never complete."); + } + } else { + // External channel. + let channel = self.internal_to_external_channels[&channel]; + self.channel_operations + .entry(channel) + .or_default() + .push_back(ChannelOperation::Send { packet }); + self.status = Status::WaitingForOperations; + } + } + fiber::Status::Receiving { channel } => { + if let Some(channel) = self.internal_channels.get_mut(&channel) { + // Internal channel. + if let Some(packet) = channel.receive() { + fiber.complete_receive(packet); + } else { + warn!( + "Tried to receive from an empty channel that is local to a fiber. This will never complete." + ); + } + } else { + // External channel. + let channel = self.internal_to_external_channels[&channel]; + self.channel_operations + .entry(channel) + .or_default() + .push_back(ChannelOperation::Receive); + self.status = Status::WaitingForOperations; + } + } + fiber::Status::InParallelScope { body } => { + let mut heap = Heap::default(); + let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); + let nursery = self.generate_channel_id(); + let nursery_send_port = heap.create_send_port(nursery); + break 'new_state State::ParallelSection { + paused_main_fiber: fiber, + nursery, + parallel_body: Box::new(Vm::new_for_running_closure( + heap, + use_provider, + body, + &[nursery_send_port], + )), + spawned_children: vec![], + }; + } + fiber::Status::Done => { + self.status = Status::Done; + } + fiber::Status::Panicked { reason } => { + self.status = Status::Panicked { reason }; } - } else { - // External channel. - todo!() } + State::SingleFiber(fiber) } - fiber::Status::Receiving { channel } => { - if let Some(channel) = self.internal_channels.get_mut(&channel) { - // Internal channel. - if let Some(packet) = channel.receive() { - self.fiber.complete_receive(packet); - } else { - panic!("Tried to receive from internal empty channel. Deadlock."); + State::ParallelSection { + paused_main_fiber, + nursery, + mut parallel_body, + mut spawned_children, + } => { + let (index_and_result_channel, vm) = spawned_children + .iter_mut() + .enumerate() + .map(|(i, (channel, vm))| (Some((i, *channel)), vm)) + .chain([(None, &mut *parallel_body)].into_iter()) + .filter(|(_, vm)| matches!(vm.status, Status::Running)) + .choose(&mut rand::thread_rng()) + .expect("Tried to run Vm, but no child can run."); + + info!("Running child VM."); + vm.run(use_provider, num_instructions); + + for (channel, operations) in &vm.channel_operations { + // TODO + warn!("Handle operations on channel {channel}") + } + + // If this was a spawned channel and it ended execution, the result should be + // transmitted to the channel that's returned by the `core.async` call. + if let Some((index, result_channel)) = index_and_result_channel { + let packet = match vm.status() { + Status::Done => { + info!("Child done."); + let (_, vm) = spawned_children.remove(index); + let TearDownResult { + heap: vm_heap, + result, + .. + } = vm.tear_down(); + let return_value = result.unwrap(); + let mut heap = Heap::default(); + let return_value = + vm_heap.clone_single_to_other_heap(&mut heap, return_value); + let value = heap.create_result(Ok(return_value)); + Some(Packet { heap, value }) + } + Status::Panicked { reason } => { + warn!("Child panicked with reason {reason}"); + let mut heap = Heap::default(); + let reason = heap.create_text(reason); + let value = heap.create_result(Err(reason)); + Some(Packet { heap, value }) + } + _ => None, + }; + if let Some(packet) = packet { + self.channel_operations + .entry(result_channel) + .or_default() + .push_back(ChannelOperation::Send { packet }) } - } else { - // External channel. - todo!() } - } - fiber::Status::Done => { - self.status = Status::Done; - break; - } - fiber::Status::Panicked { reason } => { - self.status = Status::Panicked { reason }; - break; + + // Update status and state. + // let all_vms = spawned_children + // .iter() + // .map(|(_, vm)| vm) + // .chain([*parallel_body].iter()) + // .collect_vec(); + + // let can_something_run = all_vms + // .iter() + // .any(|vm| matches!(vm.status, Status::Running)); + // if can_something_run { + // self.status = Status::Running + // } + + // let all_finished = all_vms + // .iter() + // .all(|vm| matches!(vm.status, Status::Done | Status::Panicked { .. })); + // if all_finished { + // let TearDownResult { heap, result, .. } = parallel_body.tear_down(); + // match result { + // Ok(return_value) => { + // paused_main_fiber.complete_parallel_scope(&mut heap, return_value); + // break 'new_state State::SingleFiber(paused_main_fiber); + // } + // Err(_) => todo!(), + // } + // } + + State::ParallelSection { + paused_main_fiber, + nursery, + parallel_body, + spawned_children, + } } } - } + }; + (new_state, num_instructions_executed) } pub fn run_synchronously_until_completion(mut self, db: &Database) -> TearDownResult { let use_provider = DbUseProvider { db }; @@ -152,4 +353,10 @@ impl Vm { } } } + + fn generate_channel_id(&mut self) -> ChannelId { + let id = self.next_internal_channel_id; + self.next_internal_channel_id += 1; + id + } } diff --git a/packages/Core/.candy b/packages/Core/.candy index 7995e406f..749a2dc08 100644 --- a/packages/Core/.candy +++ b/packages/Core/.candy @@ -4,6 +4,8 @@ check := (use ".Check").check concurrency = use ".Concurrency" parallel := concurrency.parallel +async := concurrency.async +await := concurrency.await conditionals = use ".Conditionals" if := conditionals.if diff --git a/packages/Core/Concurrency.candy b/packages/Core/Concurrency.candy index e2a39e845..280c80402 100644 --- a/packages/Core/Concurrency.candy +++ b/packages/Core/Concurrency.candy @@ -1,5 +1,16 @@ -isFunction1 = (use "..Function").is1 +channel = use "..Channel" +function = use "..Function" parallel body := - needs (isFunction1 body) + needs (function.is1 body) ✨.parallel body + +async nursery body := + needs (channel.isSendPort nursery) + needs (function.is0 body) + channel.send nursery [Spawn, body] + +await childChannel := + needs (channel.isReceivePort childChannel) + result = channel.receive childChannel + needs False "Todo: Implement await" From b85327d262ad210e58e6eff9ee7e721b6cc5a55f Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sat, 20 Aug 2022 00:01:41 +0200 Subject: [PATCH 07/59] Continue implementing fibers --- compiler/src/fuzzer/fuzzer.rs | 14 +- compiler/src/fuzzer/mod.rs | 4 +- .../hints/constant_evaluator.rs | 14 +- compiler/src/main.rs | 4 +- compiler/src/vm/builtin_functions.rs | 48 +++- compiler/src/vm/fiber.rs | 30 ++- compiler/src/vm/heap/mod.rs | 7 + compiler/src/vm/mod.rs | 4 +- compiler/src/vm/{vm.rs => tree.rs} | 240 +++++++++++++----- packages/Core/Concurrency.candy | 20 +- 10 files changed, 271 insertions(+), 114 deletions(-) rename compiler/src/vm/{vm.rs => tree.rs} (60%) diff --git a/compiler/src/fuzzer/fuzzer.rs b/compiler/src/fuzzer/fuzzer.rs index 49a5d6586..0d1115edd 100644 --- a/compiler/src/fuzzer/fuzzer.rs +++ b/compiler/src/fuzzer/fuzzer.rs @@ -2,7 +2,7 @@ use super::{generator::generate_n_values, utils::did_need_in_closure_cause_panic use crate::{ compiler::hir, database::Database, - vm::{tracer::Tracer, use_provider::DbUseProvider, vm, Closure, Heap, Pointer, Vm}, + vm::{tracer::Tracer, tree, use_provider::DbUseProvider, Closure, FiberTree, Heap, Pointer}, }; use std::mem; @@ -18,7 +18,7 @@ pub enum Status { // stuff, we'll never find the errors if we accidentally first choose an // input that triggers the loop. StillFuzzing { - vm: Vm, + vm: FiberTree, arguments: Vec, }, // TODO: In the future, also add a state for trying to simplify the @@ -43,7 +43,7 @@ impl Status { let arguments = generate_n_values(&mut vm_heap, num_args); let use_provider = DbUseProvider { db }; - let vm = Vm::new_for_running_closure(vm_heap, &use_provider, closure, &arguments); + let vm = FiberTree::new_for_running_closure(vm_heap, &use_provider, closure, &arguments); Status::StillFuzzing { vm, arguments } } @@ -90,7 +90,7 @@ impl Fuzzer { ) -> (Status, usize) { match status { Status::StillFuzzing { mut vm, arguments } => match vm.status() { - vm::Status::Running => { + tree::Status::Running => { let use_provider = DbUseProvider { db }; let num_instructions_executed_before = vm.num_instructions_executed(); vm.run(&use_provider, num_instructions); @@ -101,13 +101,13 @@ impl Fuzzer { num_instruction_executed, ) } - vm::Status::WaitingForOperations => panic!("Fuzzing should not have to wait on channel operations because arguments were not channels."), + tree::Status::WaitingForOperations => panic!("Fuzzing should not have to wait on channel operations because arguments were not channels."), // The VM finished running without panicking. - vm::Status::Done => ( + tree::Status::Done => ( Status::new_fuzzing_attempt(db, &self.closure_heap, self.closure), 0, ), - vm::Status::Panicked { reason } => { + tree::Status::Panicked { reason } => { // If a `needs` directly inside the tested closure was not // satisfied, then the panic is not closure's fault, but our // fault. diff --git a/compiler/src/fuzzer/mod.rs b/compiler/src/fuzzer/mod.rs index 1a03cbea7..41426ea01 100644 --- a/compiler/src/fuzzer/mod.rs +++ b/compiler/src/fuzzer/mod.rs @@ -6,14 +6,14 @@ pub use self::fuzzer::{Fuzzer, Status}; use crate::{ database::Database, module::Module, - vm::{use_provider::DbUseProvider, Closure, Vm}, + vm::{use_provider::DbUseProvider, Closure, FiberTree}, }; use itertools::Itertools; use tracing::{error, info}; pub async fn fuzz(db: &Database, module: Module) { let (fuzzables_heap, fuzzables) = { - let result = Vm::new_for_running_module_closure( + let result = FiberTree::new_for_running_module_closure( &DbUseProvider { db }, Closure::of_module(db, module.clone()).unwrap(), ) diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index eaf1b0135..b65ce4930 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -9,7 +9,9 @@ use crate::{ database::Database, language_server::hints::{utils::id_to_end_of_line, HintKind}, module::Module, - vm::{tracer::TraceEntry, use_provider::DbUseProvider, vm, Closure, Heap, Pointer, Vm}, + vm::{ + tracer::TraceEntry, tree, use_provider::DbUseProvider, Closure, FiberTree, Heap, Pointer, + }, }; use itertools::Itertools; use rand::{prelude::SliceRandom, thread_rng}; @@ -18,12 +20,12 @@ use tracing::{span, trace, Level}; #[derive(Default)] pub struct ConstantEvaluator { - vms: HashMap, + vms: HashMap, } impl ConstantEvaluator { pub fn update_module(&mut self, db: &Database, module: Module) { - let vm = Vm::new_for_running_module_closure( + let vm = FiberTree::new_for_running_module_closure( &DbUseProvider { db }, Closure::of_module(db, module.clone()).unwrap(), ); @@ -39,7 +41,7 @@ impl ConstantEvaluator { let mut running_vms = self .vms .iter_mut() - .filter(|(_, vm)| matches!(vm.status(), vm::Status::Running)) + .filter(|(_, vm)| matches!(vm.status(), tree::Status::Running)) .collect_vec(); trace!( "Constant evaluator running. {} running VMs, {} in total.", @@ -76,7 +78,7 @@ impl ConstantEvaluator { let vm = &self.vms[module]; let mut hints = vec![]; - if let vm::Status::Panicked { reason } = vm.status() { + if let tree::Status::Panicked { reason } = vm.status() { if let Some(hint) = panic_hint(db, module.clone(), vm, reason) { hints.push(hint); } @@ -125,7 +127,7 @@ impl ConstantEvaluator { } } -fn panic_hint(db: &Database, module: Module, vm: &Vm, reason: String) -> Option { +fn panic_hint(db: &Database, module: Module, vm: &FiberTree, reason: String) -> Option { // We want to show the hint at the last call site still inside the current // module. If there is no call site in this module, then the panic results // from a compiler error in a previous stage which is already reported. diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 0304e68c7..d7f85eed0 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -26,7 +26,7 @@ use crate::{ database::Database, language_server::utils::LspPositionConversion, module::{Module, ModuleKind}, - vm::{use_provider::DbUseProvider, Closure, TearDownResult, Vm}, + vm::{use_provider::DbUseProvider, Closure, FiberTree, TearDownResult}, }; use compiler::lir::Lir; use itertools::Itertools; @@ -203,7 +203,7 @@ fn run(options: CandyRunOptions) { info!("Running `{path_string}`."); let use_provider = DbUseProvider { db: &db }; - let vm = Vm::new_for_running_module_closure(&use_provider, module_closure); + let vm = FiberTree::new_for_running_module_closure(&use_provider, module_closure); let TearDownResult { tracer, result, diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index 2c5ddce53..5bee4f0ad 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -69,7 +69,15 @@ impl Fiber { Ok(CreateChannel { capacity }) => self.status = Status::CreatingChannel { capacity }, Ok(Send { channel, packet }) => self.status = Status::Sending { channel, packet }, Ok(Receive { channel }) => self.status = Status::Receiving { channel }, - Ok(Parallel { body }) => self.status = Status::InParallelScope { body }, + Ok(Parallel { + body, + return_channel, + }) => { + self.status = Status::InParallelScope { + body, + return_channel, + } + } Err(reason) => self.panic(reason), } } @@ -78,11 +86,23 @@ impl Fiber { type BuiltinResult = Result; enum SuccessfulBehavior { Return(Pointer), - DivergeControlFlow { closure: Pointer }, - CreateChannel { capacity: Capacity }, - Send { channel: ChannelId, packet: Packet }, - Receive { channel: ChannelId }, - Parallel { body: Pointer }, + DivergeControlFlow { + closure: Pointer, + }, + CreateChannel { + capacity: Capacity, + }, + Send { + channel: ChannelId, + packet: Packet, + }, + Receive { + channel: ChannelId, + }, + Parallel { + body: Pointer, + return_channel: ChannelId, + }, } use SuccessfulBehavior::*; @@ -263,12 +283,18 @@ impl Heap { } fn parallel(&mut self, args: &[Pointer]) -> BuiltinResult { - unpack_and_later_drop!(self, args, (body_taking_nursery: Closure), { - self.dup(body_taking_nursery.address); - Parallel { - body: body_taking_nursery.address, + unpack_and_later_drop!( + self, + args, + (body_taking_nursery: Closure, return_channel: SendPort), + { + self.dup(body_taking_nursery.address); + Parallel { + body: body_taking_nursery.address, + return_channel: return_channel.channel, + } } - }) + ) } fn print(&mut self, args: &[Pointer]) -> BuiltinResult { diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index 086aacadb..26ac88229 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -38,12 +38,24 @@ pub struct Fiber { #[derive(Clone, Debug)] pub enum Status { Running, - CreatingChannel { capacity: Capacity }, - Sending { channel: ChannelId, packet: Packet }, - Receiving { channel: ChannelId }, - InParallelScope { body: Pointer }, + CreatingChannel { + capacity: Capacity, + }, + Sending { + channel: ChannelId, + packet: Packet, + }, + Receiving { + channel: ChannelId, + }, + InParallelScope { + body: Pointer, + return_channel: ChannelId, + }, Done, - Panicked { reason: String }, + Panicked { + reason: String, + }, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -90,6 +102,9 @@ impl Fiber { num_instructions_executed: 0, } } + pub fn new_in_done_state() -> Self { + Self::new_with_heap(Heap::default()) + } pub fn new_for_running_closure( heap: Heap, use_provider: &U, @@ -158,9 +173,8 @@ impl Fiber { self.data_stack.push(address); self.status = Status::Running; } - pub fn complete_parallel_scope(&mut self, heap: &mut Heap, return_value: Pointer) { - let return_value = heap.clone_single_to_other_heap(&mut self.heap, return_value); - self.data_stack.push(return_value); + pub fn complete_parallel_scope(&mut self) { + self.data_stack.push(self.heap.create_nothing()); self.status = Status::Running; } diff --git a/compiler/src/vm/heap/mod.rs b/compiler/src/vm/heap/mod.rs index 603d8730d..d50964ed5 100644 --- a/compiler/src/vm/heap/mod.rs +++ b/compiler/src/vm/heap/mod.rs @@ -199,6 +199,13 @@ impl Heap { .unwrap() } + pub fn all_objects(&self) -> &HashMap { + &self.objects + } + pub fn all_objects_mut(&mut self) -> &mut HashMap { + &mut self.objects + } + pub fn create_int(&mut self, int: BigInt) -> Pointer { self.create(Data::Int(Int { value: int })) } diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 97c702515..362b84608 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -3,9 +3,9 @@ mod channel; mod fiber; mod heap; pub mod tracer; +pub mod tree; pub mod use_provider; -pub mod vm; pub use fiber::{Fiber, TearDownResult}; pub use heap::{Closure, Heap, Object, Pointer}; -pub use vm::Vm; +pub use tree::FiberTree; diff --git a/compiler/src/vm/vm.rs b/compiler/src/vm/tree.rs similarity index 60% rename from compiler/src/vm/vm.rs rename to compiler/src/vm/tree.rs index 5ccd0731f..8d8b0be8f 100644 --- a/compiler/src/vm/vm.rs +++ b/compiler/src/vm/tree.rs @@ -1,7 +1,7 @@ use super::{ channel::{Channel, Packet}, fiber::Fiber, - heap::ChannelId, + heap::{ChannelId, Data, ReceivePort, SendPort}, tracer::Tracer, use_provider::{DbUseProvider, UseProvider}, Closure, Heap, Pointer, TearDownResult, @@ -15,20 +15,89 @@ use std::{ }; use tracing::{debug, error, info, trace, warn}; -/// A VM is a Candy program that thinks it's currently running. Because VMs are -/// first-class Rust structs, they enable other code to store "freezed" programs -/// and to remain in control about when and for how long they run. +/// A fiber tree is a part of or an entire Candy program that thinks it's +/// currently running. Because fiber trees are first-class Rust structs, they +/// enable other code to store "freezed" programs and to remain in control about +/// when and for how long code runs. +/// +/// While fibers are simple, pure virtual machines that manage a heap and stack, +/// fiber _trees_ encapsulate fibers and manage channels that are used by them. +/// As the name suggests, every Candy program can be represented by a tree at +/// any point in time. In particular, you can create new nodes in the tree by +/// entering a `core.parallel` scope. +/// +/// ```candy +/// core.parallel { nursery -> +/// banana = core.async { "Banana" } +/// peach = core.async { "Peach" } +/// } +/// ``` +/// +/// In this example, after `banana` and `peach` have been assigned, both those +/// closures run concurrently. Let's walk through what happens at each point in +/// time. Before entering the `core.parallel` scope, we only have a single fiber +/// managed by a `FiberTree`. +/// +/// FiberTree +/// | +/// Fiber +/// main +/// (running) +/// +/// As the program enters the `core.parallel` section, the fiber tree changes +/// its state. First, it generates a channel ID for a nursery; while a nursery +/// isn't a channel, it behaves just like one (you can send it closures). Then, +/// the fiber tree creates a send port for the nursery. Finally, it spawns a new +/// fiber with the body code of the parallel section and gives it a send port of +/// the nursery as an argument. +/// +/// When asked to run code, the fiber tree will not run the original main fiber, +/// but the body of the parallel section instead. +/// +/// FiberTree +/// | +/// +------+------+ +/// | | +/// Fiber Fiber +/// main body +/// (parallel scope) (running) +/// +/// Calls to `core.async` internally just send packets to the nursery containing +/// the closures to spawn. The fiber tree knows that the channel ID is that of +/// the nursery and instead of actually saving the packets spawns new fibers. +/// The packets also contain a send port of a channel that `core.async` creates +/// locally and that is expected to be sent the result of the running closure. +/// +/// After the two calls to `core.async` finished, the tree looks like this: +/// +/// FiberTree +/// | +/// +------+------+---------+-----------+ +/// | | | | +/// Fiber Fiber Fiber Fiber +/// main body banana peach +/// (parallel scope) (running) (running) (running) +/// +/// Now, when the tree is asked to run code, it will run a random running fiber. +/// +/// Once a spawned fiber is done, its return value is stored in the +/// corresponding channel and the fiber is deleted. +/// Once all fibers are done, the fiber tree exits the parallel section, using +/// the return value of the body fiber as the return value of the call to +/// `core.parallel`. +/// If any of the children panic, the parallel section itself will immediately +/// panic as well. #[derive(Clone)] -pub struct Vm { +pub struct FiberTree { status: Status, state: Option, // Only `None` temporarily during state transitions. - // Channel functionality. VMs communicate with the outer world using + // Channel functionality. Fiber trees communicate with the outer world using // channels. Each channel is identified using an ID that is valid inside - // this particular VM. Channels created by the program are managed ("owned") - // by the VM itself. For channels owned by the outside world (such as those - // referenced in the environment argument), the VM maintains a mapping - // between internal and external IDs. + // this particular tree node. Channels created by the current fiber are + // managed ("owned") by this node directly. For channels owned by the + // outside world (such as those referenced in the environment argument), + // this node maintains a mapping between internal and external IDs. pub internal_channels: HashMap, pub next_internal_channel_id: ChannelId, pub external_to_internal_channels: HashMap, @@ -46,23 +115,22 @@ pub enum Status { #[derive(Clone)] enum State { + /// This tree is currently focused on running a single fiber. SingleFiber(Fiber), - ParallelSection { - /// The main fiber of this VM. Should have Status::InParallelSection. - paused_main_fiber: Fiber, - /// The channel that you can send spawn commands to. + /// The fiber of this tree entered a `core.parallel` scope so that it's now + /// paused and waits for the parallel scope to end. Instead of the main + /// former single fiber, the tree now runs the closure passed to + /// `core.parallel` as well as any other spawned children. + ParallelSection { + paused_main_fiber: Fiber, // Should have Status::InParallelSection. nursery: ChannelId, - /// The VM for the closure with which `core.parallel` was called. - parallel_body: Box, - - /// Spawned child VMs. For each VM, there also exists a channel that - /// will contain the result of the VM (once it's done or panicked). This - /// channel is directly returned by the `core.async` function. + /// Children and a channels where to send the result of the child. The + /// channel's receive port is directly returned by the `core.async` function. /// Here, we save the ID of the channel where the result of the VM will /// be sent. - spawned_children: Vec<(ChannelId, Vm)>, + children: Vec<(ChannelId, FiberTree)>, }, } @@ -72,17 +140,27 @@ pub enum ChannelOperation { Receive, } -impl Vm { - fn new_with_fiber(fiber: Fiber) -> Self { - Self { - status: Status::Running, - state: Some(State::SingleFiber(fiber)), +impl FiberTree { + fn new_with_fiber(mut fiber: Fiber) -> Self { + let mut tree = Self { + status: match fiber.status { + fiber::Status::Done => Status::Done, + fiber::Status::Running => Status::Running, + _ => panic!("Tried to create fiber tree with invalid fiber."), + }, + state: None, internal_channels: Default::default(), next_internal_channel_id: 0, external_to_internal_channels: Default::default(), internal_to_external_channels: Default::default(), channel_operations: Default::default(), - } + }; + tree.create_channel_mappings_for(&fiber.heap); + fiber + .heap + .map_channel_ids(&tree.external_to_internal_channels); + tree.state = Some(State::SingleFiber(fiber)); + tree } pub fn new_for_running_closure( heap: Heap, @@ -115,6 +193,12 @@ impl Vm { pub fn status(&self) -> Status { self.status.clone() } + fn is_running(&self) -> bool { + matches!(self.status, Status::Running) + } + fn is_finished(&self) -> bool { + matches!(self.status, Status::Done | Status::Panicked { .. }) + } pub fn fiber(&self) -> &Fiber { match self.state() { State::SingleFiber(fiber) => fiber, @@ -139,6 +223,22 @@ impl Vm { .expect("Tried to get VM state during state transition") } + fn create_channel_mappings_for(&mut self, heap: &Heap) { + for object in heap.all_objects().values() { + if let Data::SendPort(SendPort { channel }) + | Data::ReceivePort(ReceivePort { channel }) = object.data + { + if !self.external_to_internal_channels.contains_key(&channel) { + let internal_id = self.generate_channel_id(); + self.external_to_internal_channels + .insert(channel, internal_id); + self.internal_to_external_channels + .insert(internal_id, channel); + } + } + } + } + pub fn run(&mut self, use_provider: &U, mut num_instructions: usize) { assert!( matches!(self.status, Status::Running), @@ -182,6 +282,7 @@ impl Vm { fiber.complete_channel_create(id); } fiber::Status::Sending { channel, packet } => { + info!("Sending packet to channel {channel}."); if let Some(channel) = self.internal_channels.get_mut(&channel) { // Internal channel. if channel.send(packet) { @@ -200,6 +301,7 @@ impl Vm { } } fiber::Status::Receiving { channel } => { + info!("Sending packet to channel {channel}."); if let Some(channel) = self.internal_channels.get_mut(&channel) { // Internal channel. if let Some(packet) = channel.receive() { @@ -219,43 +321,48 @@ impl Vm { self.status = Status::WaitingForOperations; } } - fiber::Status::InParallelScope { body } => { + fiber::Status::InParallelScope { + body, + return_channel, + } => { + info!("Entering parallel scope."); let mut heap = Heap::default(); let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); let nursery = self.generate_channel_id(); let nursery_send_port = heap.create_send_port(nursery); + let tree = FiberTree::new_for_running_closure( + heap, + use_provider, + body, + &[nursery_send_port], + ); + break 'new_state State::ParallelSection { paused_main_fiber: fiber, nursery, - parallel_body: Box::new(Vm::new_for_running_closure( - heap, - use_provider, - body, - &[nursery_send_port], - )), - spawned_children: vec![], + children: vec![(return_channel, tree)], }; } fiber::Status::Done => { + info!("Fiber done."); self.status = Status::Done; } fiber::Status::Panicked { reason } => { + info!("Fiber panicked because of {reason}."); self.status = Status::Panicked { reason }; } } State::SingleFiber(fiber) } State::ParallelSection { - paused_main_fiber, + mut paused_main_fiber, nursery, - mut parallel_body, - mut spawned_children, + mut children, } => { - let (index_and_result_channel, vm) = spawned_children + let (index_and_result_channel, vm) = children .iter_mut() .enumerate() .map(|(i, (channel, vm))| (Some((i, *channel)), vm)) - .chain([(None, &mut *parallel_body)].into_iter()) .filter(|(_, vm)| matches!(vm.status, Status::Running)) .choose(&mut rand::thread_rng()) .expect("Tried to run Vm, but no child can run."); @@ -265,7 +372,7 @@ impl Vm { for (channel, operations) in &vm.channel_operations { // TODO - warn!("Handle operations on channel {channel}") + warn!("Todo: Handle operations on channel {channel}") } // If this was a spawned channel and it ended execution, the result should be @@ -274,7 +381,7 @@ impl Vm { let packet = match vm.status() { Status::Done => { info!("Child done."); - let (_, vm) = spawned_children.remove(index); + let (_, vm) = children.remove(index); let TearDownResult { heap: vm_heap, result, @@ -305,38 +412,19 @@ impl Vm { } // Update status and state. - // let all_vms = spawned_children - // .iter() - // .map(|(_, vm)| vm) - // .chain([*parallel_body].iter()) - // .collect_vec(); - - // let can_something_run = all_vms - // .iter() - // .any(|vm| matches!(vm.status, Status::Running)); - // if can_something_run { - // self.status = Status::Running - // } - - // let all_finished = all_vms - // .iter() - // .all(|vm| matches!(vm.status, Status::Done | Status::Panicked { .. })); - // if all_finished { - // let TearDownResult { heap, result, .. } = parallel_body.tear_down(); - // match result { - // Ok(return_value) => { - // paused_main_fiber.complete_parallel_scope(&mut heap, return_value); - // break 'new_state State::SingleFiber(paused_main_fiber); - // } - // Err(_) => todo!(), - // } - // } + if children.iter().any(|(_, vm)| vm.is_running()) { + self.status = Status::Running + } + if children.iter().all(|(_, vm)| vm.is_finished()) { + paused_main_fiber.complete_parallel_scope(); + self.status = Status::Running; + break 'new_state State::SingleFiber(paused_main_fiber); + } State::ParallelSection { paused_main_fiber, nursery, - parallel_body, - spawned_children, + children, } } } @@ -360,3 +448,15 @@ impl Vm { id } } + +impl Heap { + fn map_channel_ids(&mut self, mapping: &HashMap) { + for object in self.all_objects_mut().values_mut() { + if let Data::SendPort(SendPort { channel }) + | Data::ReceivePort(ReceivePort { channel }) = &mut object.data + { + *channel = mapping[channel]; + } + } + } +} diff --git a/packages/Core/Concurrency.candy b/packages/Core/Concurrency.candy index 280c80402..65c8fc0ec 100644 --- a/packages/Core/Concurrency.candy +++ b/packages/Core/Concurrency.candy @@ -1,16 +1,24 @@ channel = use "..Channel" function = use "..Function" +structGet = (use "..Struct").getUnwrap parallel body := needs (function.is1 body) - ✨.parallel body + returnValueChannel = channel.create 1 + returnValueSendPort = structGet returnValueChannel 0 + returnValueReceivePort = structGet returnValueChannel 1 + ✨.parallel body returnValueSendPort + channel.receive returnValueReceivePort async nursery body := needs (channel.isSendPort nursery) needs (function.is0 body) - channel.send nursery [Spawn, body] + returnValueChannel = channel.create 1 + returnValueSendPort = structGet returnValueChannel 0 + returnValueReceivePort = structGet returnValueChannel 1 + channel.send nursery [Spawn, body, returnValueSendPort] + returnValueReceivePort -await childChannel := - needs (channel.isReceivePort childChannel) - result = channel.receive childChannel - needs False "Todo: Implement await" +await fiber := + needs (channel.isReceivePort fiber) + channel.receive fiber From 47b06b5c48ef8b4edb7e9b0c8b02532c3c7d3028 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sat, 20 Aug 2022 00:28:16 +0200 Subject: [PATCH 08/59] Update docs --- compiler/src/vm/fiber.rs | 3 -- compiler/src/vm/tree.rs | 85 +++++++++++++++++++++++++++++++--------- 2 files changed, 66 insertions(+), 22 deletions(-) diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index 26ac88229..863dd42df 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -102,9 +102,6 @@ impl Fiber { num_instructions_executed: 0, } } - pub fn new_in_done_state() -> Self { - Self::new_with_heap(Heap::default()) - } pub fn new_for_running_closure( heap: Heap, use_provider: &U, diff --git a/compiler/src/vm/tree.rs b/compiler/src/vm/tree.rs index 8d8b0be8f..c6a9b6e97 100644 --- a/compiler/src/vm/tree.rs +++ b/compiler/src/vm/tree.rs @@ -7,13 +7,12 @@ use super::{ Closure, Heap, Pointer, TearDownResult, }; use crate::{database::Database, vm::fiber}; -use itertools::Itertools; -use rand::{seq::IteratorRandom, thread_rng}; +use rand::seq::IteratorRandom; use std::{ collections::{HashMap, VecDeque}, mem, }; -use tracing::{debug, error, info, trace, warn}; +use tracing::{debug, info, warn}; /// A fiber tree is a part of or an entire Candy program that thinks it's /// currently running. Because fiber trees are first-class Rust structs, they @@ -22,6 +21,9 @@ use tracing::{debug, error, info, trace, warn}; /// /// While fibers are simple, pure virtual machines that manage a heap and stack, /// fiber _trees_ encapsulate fibers and manage channels that are used by them. +/// +/// ## Fibers +/// /// As the name suggests, every Candy program can be represented by a tree at /// any point in time. In particular, you can create new nodes in the tree by /// entering a `core.parallel` scope. @@ -48,8 +50,8 @@ use tracing::{debug, error, info, trace, warn}; /// its state. First, it generates a channel ID for a nursery; while a nursery /// isn't a channel, it behaves just like one (you can send it closures). Then, /// the fiber tree creates a send port for the nursery. Finally, it spawns a new -/// fiber with the body code of the parallel section and gives it a send port of -/// the nursery as an argument. +/// fiber tree with the body code of the parallel section, giving it a send port +/// of the nursery as an argument. /// /// When asked to run code, the fiber tree will not run the original main fiber, /// but the body of the parallel section instead. @@ -58,9 +60,11 @@ use tracing::{debug, error, info, trace, warn}; /// | /// +------+------+ /// | | -/// Fiber Fiber -/// main body -/// (parallel scope) (running) +/// Fiber FiberTree +/// main | +/// (parallel scope) Fiber +/// body +/// (running) /// /// Calls to `core.async` internally just send packets to the nursery containing /// the closures to spawn. The fiber tree knows that the channel ID is that of @@ -72,21 +76,64 @@ use tracing::{debug, error, info, trace, warn}; /// /// FiberTree /// | -/// +------+------+---------+-----------+ -/// | | | | -/// Fiber Fiber Fiber Fiber -/// main body banana peach -/// (parallel scope) (running) (running) (running) +/// +------+------+----------+----------+ +/// | | | | +/// Fiber FiberTree FiberTree FiberTree +/// main | | | +/// (parallel scope) Fiber Fiber Fiber +/// body banana peach +/// (running) (running) (running) /// /// Now, when the tree is asked to run code, it will run a random running fiber. /// /// Once a spawned fiber is done, its return value is stored in the -/// corresponding channel and the fiber is deleted. -/// Once all fibers are done, the fiber tree exits the parallel section, using -/// the return value of the body fiber as the return value of the call to -/// `core.parallel`. -/// If any of the children panic, the parallel section itself will immediately -/// panic as well. +/// corresponding channel and the fiber is deleted. Once all fibers finished +/// running, the fiber tree exits the parallel section. The `core.parallel` and +/// `core.await` calls take care of actually returning the values put into the +/// channels. If any of the children panic, the parallel section itself will +/// immediately panic as well. +/// +/// The internal fiber trees can of course also start their own parallel +/// sections, resulting in a nested tree. +/// +/// ## Channels +/// +/// In Candy code, channels only appear via their ends, the send and receive +/// ports. Those are unlike other values. In particular, they have an identity +/// and are mutable. Operations like `channel.receive` are not pure and may +/// return different values every time. Also, operations on channels are +/// blocking. +/// +/// Unlike fibers, channels don't form a tree – they can go all over the place! +/// Because you can transmit ports over channels, any two parts of a fiber tree +/// could theoretically be connected via channels. +/// +/// In most programs, we expect channels to stay "relatively" local. In +/// particular, most channels don't escape the fiber tree that they are created +/// in. In order to get the most benefit out of actual paralellism, it's +/// beneficial to store channels as local as possible. For example, if two +/// completely different parts of a program use channels locally to model +/// mutable variables or some other data flow, all channel operations should be +/// local only and not need to be propagated to a central location, avoiding +/// contention. This becomes even more important when (if?) we distribute +/// programs across multiple machines; local channels shouldn't require any +/// communication whatsoever. +/// +/// That's why channels are stored in the local-most subtree of the Candy +/// program that has access to corresponding ports. Fibers themselves don't +/// store channels though – the surrounding nodes of the fiber trees take care +/// of managing channels. +/// +/// The identity of channels is modelled using a channel ID, which is unique +/// within a node of the fiber tree. Whenever data with ports is transmitted to +/// child or parent nodes in the tree, the channel IDs of the ports need to be +/// translated. +/// +/// TODO: Example +/// +/// ## Catching panics +/// +/// TODO: Implement #[derive(Clone)] pub struct FiberTree { status: Status, From 37fb3cb3a99974ecba8dff6506601a334acd143c Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 22 Aug 2022 00:36:59 +0200 Subject: [PATCH 09/59] Fix Int --- packages/Core/Int.candy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/Core/Int.candy b/packages/Core/Int.candy index 3f0af342b..e68ca07e9 100644 --- a/packages/Core/Int.candy +++ b/packages/Core/Int.candy @@ -1,5 +1,5 @@ bool = use "..Bool" -check = use "..Check" +check = (use "..Check").check conditionals = use "..Conditionals" equals = (use "..Equality").equals type = use "..Type" From cb0c2119f3224cbd8a30a357d55bc38fc5e732e4 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 22 Aug 2022 00:40:00 +0200 Subject: [PATCH 10/59] Use Context instead of UseProvider when running code --- README.md | 1 - compiler/src/fuzzer/fuzzer.rs | 70 ++-- compiler/src/fuzzer/mod.rs | 26 +- .../hints/constant_evaluator.rs | 11 +- compiler/src/language_server/hints/fuzzer.rs | 17 +- compiler/src/language_server/hints/mod.rs | 4 +- compiler/src/main.rs | 14 +- compiler/src/vm/builtin_functions.rs | 8 +- compiler/src/vm/context.rs | 114 ++++++ compiler/src/vm/fiber.rs | 38 +- compiler/src/vm/mod.rs | 3 +- compiler/src/vm/tree.rs | 338 ++++++++---------- .../src/vm/{use_provider.rs => use_module.rs} | 47 +-- 13 files changed, 364 insertions(+), 327 deletions(-) create mode 100644 compiler/src/vm/context.rs rename compiler/src/vm/{use_provider.rs => use_module.rs} (68%) diff --git a/README.md b/README.md index 946814bba..b41d8928a 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,6 @@ Major milestones: - eliminate common subtrees - inline functions - minimize inputs found through fuzzing -- make condition whether to keep running more granular - fuzz parser - support recursion - tail call optimization diff --git a/compiler/src/fuzzer/fuzzer.rs b/compiler/src/fuzzer/fuzzer.rs index 0d1115edd..3288e765b 100644 --- a/compiler/src/fuzzer/fuzzer.rs +++ b/compiler/src/fuzzer/fuzzer.rs @@ -2,7 +2,7 @@ use super::{generator::generate_n_values, utils::did_need_in_closure_cause_panic use crate::{ compiler::hir, database::Database, - vm::{tracer::Tracer, tree, use_provider::DbUseProvider, Closure, FiberTree, Heap, Pointer}, + vm::{context::Context, tracer::Tracer, tree, Closure, FiberTree, Heap, Pointer}, }; use std::mem; @@ -32,7 +32,7 @@ pub enum Status { } impl Status { - fn new_fuzzing_attempt(db: &Database, closure_heap: &Heap, closure: Pointer) -> Status { + fn new_fuzzing_attempt(closure_heap: &Heap, closure: Pointer) -> Status { let num_args = { let closure: Closure = closure_heap.get(closure).data.clone().try_into().unwrap(); closure.num_args @@ -42,19 +42,18 @@ impl Status { let closure = closure_heap.clone_single_to_other_heap(&mut vm_heap, closure); let arguments = generate_n_values(&mut vm_heap, num_args); - let use_provider = DbUseProvider { db }; - let vm = FiberTree::new_for_running_closure(vm_heap, &use_provider, closure, &arguments); + let vm = FiberTree::new_for_running_closure(vm_heap, closure, &arguments); Status::StillFuzzing { vm, arguments } } } impl Fuzzer { - pub fn new(db: &Database, closure_heap: &Heap, closure: Pointer, closure_id: hir::Id) -> Self { + pub fn new(closure_heap: &Heap, closure: Pointer, closure_id: hir::Id) -> Self { // The given `closure_heap` may contain other fuzzable closures. let mut heap = Heap::default(); let closure = closure_heap.clone_single_to_other_heap(&mut heap, closure); - let status = Status::new_fuzzing_attempt(db, &heap, closure); + let status = Status::new_fuzzing_attempt(&heap, closure); Self { closure_heap: heap, closure, @@ -67,54 +66,31 @@ impl Fuzzer { self.status.as_ref().unwrap() } - pub fn run(&mut self, db: &Database, mut num_instructions: usize) { + pub fn run(&mut self, db: &Database, context: &mut C) { let mut status = mem::replace(&mut self.status, None).unwrap(); - while matches!(status, Status::StillFuzzing { .. }) { - let (new_status, num_instructions_executed) = - self.map_status(db, status, num_instructions); - status = new_status; - - if num_instructions_executed >= num_instructions { - break; - } else { - num_instructions -= num_instructions_executed; - } + while matches!(status, Status::StillFuzzing { .. }) && context.should_continue_running() { + status = self.map_status(status, db, context); } self.status = Some(status); } - fn map_status( - &self, - db: &Database, - status: Status, - num_instructions: usize, - ) -> (Status, usize) { + fn map_status(&self, status: Status, db: &Database, context: &mut C) -> Status { match status { Status::StillFuzzing { mut vm, arguments } => match vm.status() { tree::Status::Running => { - let use_provider = DbUseProvider { db }; - let num_instructions_executed_before = vm.num_instructions_executed(); - vm.run(&use_provider, num_instructions); - let num_instruction_executed = - vm.num_instructions_executed() - num_instructions_executed_before; - ( - Status::StillFuzzing { vm, arguments }, - num_instruction_executed, - ) + vm.run(context); + Status::StillFuzzing { vm, arguments } } tree::Status::WaitingForOperations => panic!("Fuzzing should not have to wait on channel operations because arguments were not channels."), // The VM finished running without panicking. - tree::Status::Done => ( - Status::new_fuzzing_attempt(db, &self.closure_heap, self.closure), - 0, - ), + tree::Status::Done => Status::new_fuzzing_attempt(&self.closure_heap, self.closure), tree::Status::Panicked { reason } => { // If a `needs` directly inside the tested closure was not // satisfied, then the panic is not closure's fault, but our // fault. let is_our_fault = did_need_in_closure_cause_panic(db, &self.closure_id, &vm.cloned_tracer()); - let status = if is_our_fault { - Status::new_fuzzing_attempt(db, &self.closure_heap, self.closure) + if is_our_fault { + Status::new_fuzzing_attempt(&self.closure_heap, self.closure) } else { Status::PanickedForArguments { heap: vm.fiber().heap.clone(), @@ -122,8 +98,7 @@ impl Fuzzer { reason, tracer: vm.fiber().tracer.clone(), } - }; - (status, 0) + } } }, // We already found some arguments that caused the closure to panic, @@ -133,15 +108,12 @@ impl Fuzzer { arguments, reason, tracer, - } => ( - Status::PanickedForArguments { - heap, - arguments, - reason, - tracer, - }, - 0, - ), + } => Status::PanickedForArguments { + heap, + arguments, + reason, + tracer, + }, } } } diff --git a/compiler/src/fuzzer/mod.rs b/compiler/src/fuzzer/mod.rs index 41426ea01..8a8e13f43 100644 --- a/compiler/src/fuzzer/mod.rs +++ b/compiler/src/fuzzer/mod.rs @@ -6,18 +6,24 @@ pub use self::fuzzer::{Fuzzer, Status}; use crate::{ database::Database, module::Module, - vm::{use_provider::DbUseProvider, Closure, FiberTree}, + vm::{ + context::{DbUseProvider, ModularContext, RunForever, RunLimitedNumberOfInstructions}, + Closure, FiberTree, + }, }; use itertools::Itertools; use tracing::{error, info}; pub async fn fuzz(db: &Database, module: Module) { let (fuzzables_heap, fuzzables) = { - let result = FiberTree::new_for_running_module_closure( - &DbUseProvider { db }, + let mut vm = FiberTree::new_for_running_module_closure( Closure::of_module(db, module.clone()).unwrap(), - ) - .run_synchronously_until_completion(db); + ); + vm.run(&mut ModularContext { + use_provider: DbUseProvider { db }, + execution_controller: RunForever, + }); + let result = vm.tear_down(); (result.heap, result.fuzzable_closures) }; @@ -27,8 +33,14 @@ pub async fn fuzz(db: &Database, module: Module) { ); for (id, closure) in fuzzables { - let mut fuzzer = Fuzzer::new(db, &fuzzables_heap, closure, id.clone()); - fuzzer.run(db, 1000); + let mut fuzzer = Fuzzer::new(&fuzzables_heap, closure, id.clone()); + fuzzer.run( + db, + &mut ModularContext { + use_provider: DbUseProvider { db }, + execution_controller: RunLimitedNumberOfInstructions::new(1000), + }, + ); match fuzzer.status() { Status::StillFuzzing { .. } => {} Status::PanickedForArguments { diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index b65ce4930..ad4ec76fa 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -10,7 +10,9 @@ use crate::{ language_server::hints::{utils::id_to_end_of_line, HintKind}, module::Module, vm::{ - tracer::TraceEntry, tree, use_provider::DbUseProvider, Closure, FiberTree, Heap, Pointer, + context::{DbUseProvider, ModularContext, RunLimitedNumberOfInstructions}, + tracer::TraceEntry, + tree, Closure, FiberTree, Heap, Pointer, }, }; use itertools::Itertools; @@ -26,7 +28,6 @@ pub struct ConstantEvaluator { impl ConstantEvaluator { pub fn update_module(&mut self, db: &Database, module: Module) { let vm = FiberTree::new_for_running_module_closure( - &DbUseProvider { db }, Closure::of_module(db, module.clone()).unwrap(), ); self.vms.insert(module, vm); @@ -50,8 +51,10 @@ impl ConstantEvaluator { ); if let Some((module, vm)) = running_vms.choose_mut(&mut thread_rng()) { - let use_provider = DbUseProvider { db }; - vm.run(&use_provider, 500); + vm.run(&mut ModularContext { + use_provider: DbUseProvider { db }, + execution_controller: RunLimitedNumberOfInstructions::new(500), + }); Some(module.clone()) } else { None diff --git a/compiler/src/language_server/hints/fuzzer.rs b/compiler/src/language_server/hints/fuzzer.rs index dd9866621..0c1b10e82 100644 --- a/compiler/src/language_server/hints/fuzzer.rs +++ b/compiler/src/language_server/hints/fuzzer.rs @@ -7,7 +7,11 @@ use crate::{ database::Database, fuzzer::{Fuzzer, Status}, module::Module, - vm::{tracer::TraceEntry, Heap, Pointer}, + vm::{ + context::{DbUseProvider, ModularContext, RunLimitedNumberOfInstructions}, + tracer::TraceEntry, + Heap, Pointer, + }, }; use itertools::Itertools; use rand::{prelude::SliceRandom, thread_rng}; @@ -22,14 +26,13 @@ pub struct FuzzerManager { impl FuzzerManager { pub fn update_module( &mut self, - db: &Database, module: Module, heap: &Heap, fuzzable_closures: &[(Id, Pointer)], ) { let fuzzers = fuzzable_closures .iter() - .map(|(id, closure)| (id.clone(), Fuzzer::new(db, heap, *closure, id.clone()))) + .map(|(id, closure)| (id.clone(), Fuzzer::new(heap, *closure, id.clone()))) .collect(); self.fuzzers.insert(module, fuzzers); } @@ -51,7 +54,13 @@ impl FuzzerManager { ); let fuzzer = running_fuzzers.choose_mut(&mut thread_rng())?; - fuzzer.run(db, 100); + fuzzer.run( + db, + &mut ModularContext { + use_provider: DbUseProvider { db }, + execution_controller: RunLimitedNumberOfInstructions::new(100), + }, + ); match &fuzzer.status() { Status::StillFuzzing { .. } => None, diff --git a/compiler/src/language_server/hints/mod.rs b/compiler/src/language_server/hints/mod.rs index 3d6a3ea10..dec9b3fd0 100644 --- a/compiler/src/language_server/hints/mod.rs +++ b/compiler/src/language_server/hints/mod.rs @@ -80,7 +80,7 @@ pub async fn run_server( db.did_change_module(&module, content); outgoing_hints.report_hints(module.clone(), vec![]).await; constant_evaluator.update_module(&db, module.clone()); - fuzzer.update_module(&db, module, &Heap::default(), &[]); + fuzzer.update_module(module, &Heap::default(), &[]); } Event::CloseModule(module) => { db.did_close_module(&module); @@ -99,7 +99,7 @@ pub async fn run_server( let module_with_new_insight = 'new_insight: { if let Some(module) = constant_evaluator.run(&db) { let (heap, closures) = constant_evaluator.get_fuzzable_closures(&module); - fuzzer.update_module(&db, module.clone(), heap, &closures); + fuzzer.update_module(module.clone(), heap, &closures); break 'new_insight Some(module); } if let Some(module) = fuzzer.run(&db) { diff --git a/compiler/src/main.rs b/compiler/src/main.rs index d7f85eed0..8142709d0 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -26,7 +26,10 @@ use crate::{ database::Database, language_server::utils::LspPositionConversion, module::{Module, ModuleKind}, - vm::{use_provider::DbUseProvider, Closure, FiberTree, TearDownResult}, + vm::{ + context::{DbUseProvider, ModularContext, RunForever}, + Closure, FiberTree, TearDownResult, + }, }; use compiler::lir::Lir; use itertools::Itertools; @@ -202,14 +205,17 @@ fn run(options: CandyRunOptions) { let path_string = options.file.to_string_lossy(); info!("Running `{path_string}`."); - let use_provider = DbUseProvider { db: &db }; - let vm = FiberTree::new_for_running_module_closure(&use_provider, module_closure); + let mut vm = FiberTree::new_for_running_module_closure(module_closure); + vm.run(&mut ModularContext { + use_provider: DbUseProvider { db: &db }, + execution_controller: RunForever, + }); let TearDownResult { tracer, result, heap, .. - } = vm.run_synchronously_until_completion(&db); + } = vm.tear_down(); match result { Ok(return_value) => info!( diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index 5bee4f0ad..8168a8544 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -1,8 +1,8 @@ use super::{ channel::{Capacity, Packet}, + context::Context, fiber::{Fiber, Status}, heap::{ChannelId, Closure, Data, Int, Pointer, ReceivePort, SendPort, Struct, Symbol, Text}, - use_provider::UseProvider, Heap, }; use crate::{builtin_functions::BuiltinFunction, compiler::lir::Instruction}; @@ -15,9 +15,9 @@ use tracing::{info, span, Level}; use unicode_segmentation::UnicodeSegmentation; impl Fiber { - pub(super) fn run_builtin_function( + pub(super) fn run_builtin_function( &mut self, - use_provider: &U, + context: &C, builtin_function: &BuiltinFunction, args: &[Pointer], ) { @@ -64,7 +64,7 @@ impl Fiber { Ok(Return(value)) => self.data_stack.push(value), Ok(DivergeControlFlow { closure }) => { self.data_stack.push(closure); - self.run_instruction(use_provider, Instruction::Call { num_args: 0 }); + self.run_instruction(context, Instruction::Call { num_args: 0 }); } Ok(CreateChannel { capacity }) => self.status = Status::CreatingChannel { capacity }, Ok(Send { channel, packet }) => self.status = Status::Sending { channel, packet }, diff --git a/compiler/src/vm/context.rs b/compiler/src/vm/context.rs new file mode 100644 index 000000000..0b412b311 --- /dev/null +++ b/compiler/src/vm/context.rs @@ -0,0 +1,114 @@ +use crate::{ + compiler::{hir_to_lir::HirToLir, lir::Lir}, + database::Database, + module::{Module, ModuleDb, ModuleKind}, +}; + +/// Fibers need a context whenever they want to run some expressions. It's used +/// to parameterize the running of the code over the outside world and effects +/// without bleeding implementation details (like salsa) into the code of the +/// VM itself. +pub trait Context { + fn use_module(&self, module: Module) -> Result; + fn should_continue_running(&self) -> bool; + fn instruction_executed(&mut self); +} +pub enum UseResult { + Asset(Vec), + Code(Lir), +} + +/// Context that can be used when you want to execute some known instructions +/// that are guaranteed not to import other modules. +pub struct DummyContext; +impl Context for DummyContext { + fn use_module(&self, _: Module) -> Result { + panic!("A dummy context was used for importing a module") + } + + fn should_continue_running(&self) -> bool { + true + } + + fn instruction_executed(&mut self) {} +} + +/// The modular context is a version of the `Context` where several sub-tasks +/// are handled in isolation. +pub struct ModularContext { + pub use_provider: U, + pub execution_controller: E, +} +pub trait UseProvider { + fn use_module(&self, module: Module) -> Result; +} +pub trait ExecutionController { + fn should_continue_running(&self) -> bool; + fn instruction_executed(&mut self); +} + +impl Context for ModularContext { + fn use_module(&self, module: Module) -> Result { + self.use_provider.use_module(module) + } + + fn should_continue_running(&self) -> bool { + self.execution_controller.should_continue_running() + } + + fn instruction_executed(&mut self) { + self.execution_controller.instruction_executed() + } +} + +/// Uses a salsa database to import modules. +pub struct DbUseProvider<'a> { + pub db: &'a Database, +} +impl<'a> UseProvider for DbUseProvider<'a> { + fn use_module(&self, module: Module) -> Result { + match module.kind { + ModuleKind::Asset => match self.db.get_module_content(module.clone()) { + Some(bytes) => Ok(UseResult::Asset((*bytes).clone())), + None => Err(format!("use couldn't import the asset module `{}`", module)), + }, + ModuleKind::Code => match self.db.lir(module.clone()) { + Some(lir) => Ok(UseResult::Code((*lir).clone())), + None => Err(format!("use couldn't import the code module `{}`", module)), + }, + } + } +} + +/// Limits the execution by the number of executed instructions. +pub struct RunLimitedNumberOfInstructions { + max_instructions: usize, + instructions_executed: usize, +} +impl RunLimitedNumberOfInstructions { + pub fn new(max_instructions: usize) -> Self { + Self { + max_instructions, + instructions_executed: 0, + } + } +} +impl ExecutionController for RunLimitedNumberOfInstructions { + fn should_continue_running(&self) -> bool { + self.instructions_executed < self.max_instructions + } + + fn instruction_executed(&mut self) { + self.instructions_executed += 1; + } +} + +/// Runs forever. +pub struct RunForever; +impl ExecutionController for RunForever { + fn should_continue_running(&self) -> bool { + true + } + + fn instruction_executed(&mut self) {} +} diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index 863dd42df..8188878f8 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -1,12 +1,13 @@ use super::{ channel::{Capacity, Packet}, + context::Context, heap::{Builtin, ChannelId, Closure, Data, Heap, Pointer}, tracer::{TraceEntry, Tracer}, - use_provider::UseProvider, }; use crate::{ compiler::{hir::Id, lir::Instruction}, module::Module, + vm::context::DummyContext, }; use itertools::Itertools; use std::collections::HashMap; @@ -32,7 +33,6 @@ pub struct Fiber { // fault a panic is. pub tracer: Tracer, pub fuzzable_closures: Vec<(Id, Pointer)>, - pub num_instructions_executed: usize, } #[derive(Clone, Debug)] @@ -99,15 +99,18 @@ impl Fiber { heap, tracer: Tracer::default(), fuzzable_closures: vec![], - num_instructions_executed: 0, } } - pub fn new_for_running_closure( + pub fn new_for_running_closure( heap: Heap, - use_provider: &U, closure: Pointer, arguments: &[Pointer], ) -> Self { + assert!( + !matches!(heap.get(closure).data, Data::Builtin(_),), + "can only use with closures, not builtins" + ); + let mut fiber = Self::new_with_heap(heap); fiber.data_stack.extend(arguments); @@ -115,22 +118,19 @@ impl Fiber { fiber.status = Status::Running; fiber.run_instruction( - use_provider, + &DummyContext, Instruction::Call { num_args: arguments.len(), }, ); fiber } - pub fn new_for_running_module_closure( - use_provider: &U, - closure: Closure, - ) -> Self { + pub fn new_for_running_module_closure(closure: Closure) -> Self { assert_eq!(closure.captured.len(), 0, "Called start_module_closure with a closure that is not a module closure (it captures stuff)."); assert_eq!(closure.num_args, 0, "Called start_module_closure with a closure that is not a module closure (it has arguments)."); let mut heap = Heap::default(); let closure = heap.create_closure(closure); - Self::new_for_running_closure(heap, use_provider, closure, &[]) + Self::new_for_running_closure(heap, closure, &[]) } pub fn tear_down(mut self) -> TearDownResult { let result = match self.status { @@ -179,14 +179,12 @@ impl Fiber { self.data_stack[self.data_stack.len() - 1 - offset as usize] } - pub fn run(&mut self, use_provider: &U, mut num_instructions: usize) { + pub fn run(&mut self, context: &mut C) { assert!( matches!(self.status, Status::Running), "Called Fiber::run on a fiber that is not ready to run." ); - while matches!(self.status, Status::Running) && num_instructions > 0 { - num_instructions -= 1; - + while matches!(self.status, Status::Running) && context.should_continue_running() { let current_closure = self.heap.get(self.next_instruction.closure); let current_body = if let Data::Closure(Closure { body, .. }) = ¤t_closure.data { body @@ -220,15 +218,15 @@ impl Fiber { } self.next_instruction.instruction += 1; - self.run_instruction(use_provider, instruction); - self.num_instructions_executed += 1; + self.run_instruction(context, instruction); + context.instruction_executed(); if self.next_instruction == InstructionPointer::null_pointer() { self.status = Status::Done; } } } - pub fn run_instruction(&mut self, use_provider: &U, instruction: Instruction) { + pub fn run_instruction(&mut self, context: &C, instruction: Instruction) { match instruction { Instruction::CreateInt(int) => { let address = self.heap.create_int(int.into()); @@ -323,7 +321,7 @@ impl Fiber { } Data::Builtin(Builtin { function: builtin }) => { self.heap.drop(closure_address); - self.run_builtin_function(use_provider, &builtin, &args); + self.run_builtin_function(context, &builtin, &args); } _ => { self.panic("you can only call closures and builtins".to_string()); @@ -337,7 +335,7 @@ impl Fiber { } Instruction::UseModule { current_module } => { let relative_path = self.data_stack.pop().unwrap(); - match self.use_module(use_provider, current_module, relative_path) { + match self.use_module(context, current_module, relative_path) { Ok(()) => {} Err(reason) => { self.panic(reason); diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 362b84608..d05eb9220 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -1,10 +1,11 @@ mod builtin_functions; mod channel; +pub mod context; mod fiber; mod heap; pub mod tracer; pub mod tree; -pub mod use_provider; +mod use_module; pub use fiber::{Fiber, TearDownResult}; pub use heap::{Closure, Heap, Object, Pointer}; diff --git a/compiler/src/vm/tree.rs b/compiler/src/vm/tree.rs index c6a9b6e97..77dfee04d 100644 --- a/compiler/src/vm/tree.rs +++ b/compiler/src/vm/tree.rs @@ -1,12 +1,12 @@ use super::{ channel::{Channel, Packet}, + context::Context, fiber::Fiber, heap::{ChannelId, Data, ReceivePort, SendPort}, tracer::Tracer, - use_provider::{DbUseProvider, UseProvider}, Closure, Heap, Pointer, TearDownResult, }; -use crate::{database::Database, vm::fiber}; +use crate::vm::fiber; use rand::seq::IteratorRandom; use std::{ collections::{HashMap, VecDeque}, @@ -209,24 +209,11 @@ impl FiberTree { tree.state = Some(State::SingleFiber(fiber)); tree } - pub fn new_for_running_closure( - heap: Heap, - use_provider: &U, - closure: Pointer, - arguments: &[Pointer], - ) -> Self { - Self::new_with_fiber(Fiber::new_for_running_closure( - heap, - use_provider, - closure, - arguments, - )) + pub fn new_for_running_closure(heap: Heap, closure: Pointer, arguments: &[Pointer]) -> Self { + Self::new_with_fiber(Fiber::new_for_running_closure(heap, closure, arguments)) } - pub fn new_for_running_module_closure( - use_provider: &U, - closure: Closure, - ) -> Self { - Self::new_with_fiber(Fiber::new_for_running_module_closure(use_provider, closure)) + pub fn new_for_running_module_closure(closure: Closure) -> Self { + Self::new_with_fiber(Fiber::new_for_running_module_closure(closure)) } pub fn tear_down(self) -> TearDownResult { match self.into_state() { @@ -257,9 +244,6 @@ impl FiberTree { pub fn cloned_tracer(&self) -> Tracer { self.fiber().tracer.clone() } - pub fn num_instructions_executed(&self) -> usize { - self.fiber().num_instructions_executed - } fn into_state(self) -> State { self.state .expect("Tried to get VM state during state transition") @@ -286,206 +270,174 @@ impl FiberTree { } } - pub fn run(&mut self, use_provider: &U, mut num_instructions: usize) { + pub fn run(&mut self, context: &mut C) { assert!( matches!(self.status, Status::Running), "Called Vm::run on a vm that is not ready to run." ); let mut state = mem::replace(&mut self.state, None).unwrap(); - while matches!(self.status, Status::Running) && num_instructions > 0 { - let (new_state, num_instructions_executed) = - self.run_and_map_state(state, use_provider, num_instructions); - state = new_state; - - if num_instructions_executed >= num_instructions { - break; - } else { - num_instructions -= num_instructions_executed; - } + while matches!(self.status, Status::Running) && context.should_continue_running() { + state = self.run_and_map_state(state, context); } self.state = Some(state); } - fn run_and_map_state( - &mut self, - state: State, - use_provider: &U, - num_instructions: usize, - ) -> (State, usize) { - let mut num_instructions_executed = 0; - let new_state = 'new_state: { - match state { - State::SingleFiber(mut fiber) => { - debug!("Running fiber (status = {:?}).", fiber.status); + fn run_and_map_state(&mut self, state: State, context: &mut C) -> State { + match state { + State::SingleFiber(mut fiber) => { + debug!("Running fiber (status = {:?}).", fiber.status); - fiber.num_instructions_executed = 0; - fiber.run(use_provider, num_instructions); - num_instructions_executed = fiber.num_instructions_executed; + fiber.run(context); - match fiber.status() { - fiber::Status::Running => {} - fiber::Status::CreatingChannel { capacity } => { - let id = self.generate_channel_id(); - self.internal_channels.insert(id, Channel::new(capacity)); - fiber.complete_channel_create(id); - } - fiber::Status::Sending { channel, packet } => { - info!("Sending packet to channel {channel}."); - if let Some(channel) = self.internal_channels.get_mut(&channel) { - // Internal channel. - if channel.send(packet) { - fiber.complete_send(); - } else { - warn!("Tried to send to a full channel that is local to a fiber. This will never complete."); - } + match fiber.status() { + fiber::Status::Running => {} + fiber::Status::CreatingChannel { capacity } => { + let id = self.generate_channel_id(); + self.internal_channels.insert(id, Channel::new(capacity)); + fiber.complete_channel_create(id); + } + fiber::Status::Sending { channel, packet } => { + info!("Sending packet to channel {channel}."); + if let Some(channel) = self.internal_channels.get_mut(&channel) { + // Internal channel. + if channel.send(packet) { + fiber.complete_send(); } else { - // External channel. - let channel = self.internal_to_external_channels[&channel]; - self.channel_operations - .entry(channel) - .or_default() - .push_back(ChannelOperation::Send { packet }); - self.status = Status::WaitingForOperations; + warn!("Tried to send to a full channel that is local to a fiber. This will never complete."); } + } else { + // External channel. + let channel = self.internal_to_external_channels[&channel]; + self.channel_operations + .entry(channel) + .or_default() + .push_back(ChannelOperation::Send { packet }); + self.status = Status::WaitingForOperations; } - fiber::Status::Receiving { channel } => { - info!("Sending packet to channel {channel}."); - if let Some(channel) = self.internal_channels.get_mut(&channel) { - // Internal channel. - if let Some(packet) = channel.receive() { - fiber.complete_receive(packet); - } else { - warn!( + } + fiber::Status::Receiving { channel } => { + info!("Sending packet to channel {channel}."); + if let Some(channel) = self.internal_channels.get_mut(&channel) { + // Internal channel. + if let Some(packet) = channel.receive() { + fiber.complete_receive(packet); + } else { + warn!( "Tried to receive from an empty channel that is local to a fiber. This will never complete." ); - } - } else { - // External channel. - let channel = self.internal_to_external_channels[&channel]; - self.channel_operations - .entry(channel) - .or_default() - .push_back(ChannelOperation::Receive); - self.status = Status::WaitingForOperations; } + } else { + // External channel. + let channel = self.internal_to_external_channels[&channel]; + self.channel_operations + .entry(channel) + .or_default() + .push_back(ChannelOperation::Receive); + self.status = Status::WaitingForOperations; } - fiber::Status::InParallelScope { - body, - return_channel, - } => { - info!("Entering parallel scope."); - let mut heap = Heap::default(); - let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); - let nursery = self.generate_channel_id(); - let nursery_send_port = heap.create_send_port(nursery); - let tree = FiberTree::new_for_running_closure( - heap, - use_provider, - body, - &[nursery_send_port], - ); + } + fiber::Status::InParallelScope { + body, + return_channel, + } => { + info!("Entering parallel scope."); + let mut heap = Heap::default(); + let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); + let nursery = self.generate_channel_id(); + let nursery_send_port = heap.create_send_port(nursery); + let tree = + FiberTree::new_for_running_closure(heap, body, &[nursery_send_port]); - break 'new_state State::ParallelSection { - paused_main_fiber: fiber, - nursery, - children: vec![(return_channel, tree)], - }; - } - fiber::Status::Done => { - info!("Fiber done."); - self.status = Status::Done; - } - fiber::Status::Panicked { reason } => { - info!("Fiber panicked because of {reason}."); - self.status = Status::Panicked { reason }; - } + return State::ParallelSection { + paused_main_fiber: fiber, + nursery, + children: vec![(return_channel, tree)], + }; + } + fiber::Status::Done => { + info!("Fiber done."); + self.status = Status::Done; + } + fiber::Status::Panicked { reason } => { + info!("Fiber panicked because of {reason}."); + self.status = Status::Panicked { reason }; } - State::SingleFiber(fiber) } - State::ParallelSection { - mut paused_main_fiber, - nursery, - mut children, - } => { - let (index_and_result_channel, vm) = children - .iter_mut() - .enumerate() - .map(|(i, (channel, vm))| (Some((i, *channel)), vm)) - .filter(|(_, vm)| matches!(vm.status, Status::Running)) - .choose(&mut rand::thread_rng()) - .expect("Tried to run Vm, but no child can run."); + State::SingleFiber(fiber) + } + State::ParallelSection { + mut paused_main_fiber, + nursery, + mut children, + } => { + let (index_and_result_channel, vm) = children + .iter_mut() + .enumerate() + .map(|(i, (channel, vm))| (Some((i, *channel)), vm)) + .filter(|(_, vm)| matches!(vm.status, Status::Running)) + .choose(&mut rand::thread_rng()) + .expect("Tried to run Vm, but no child can run."); - info!("Running child VM."); - vm.run(use_provider, num_instructions); + info!("Running child VM."); + vm.run(context); - for (channel, operations) in &vm.channel_operations { - // TODO - warn!("Todo: Handle operations on channel {channel}") - } + for (channel, operations) in &vm.channel_operations { + // TODO + warn!("Todo: Handle operations on channel {channel}"); + // for operation in operations {} + } - // If this was a spawned channel and it ended execution, the result should be - // transmitted to the channel that's returned by the `core.async` call. - if let Some((index, result_channel)) = index_and_result_channel { - let packet = match vm.status() { - Status::Done => { - info!("Child done."); - let (_, vm) = children.remove(index); - let TearDownResult { - heap: vm_heap, - result, - .. - } = vm.tear_down(); - let return_value = result.unwrap(); - let mut heap = Heap::default(); - let return_value = - vm_heap.clone_single_to_other_heap(&mut heap, return_value); - let value = heap.create_result(Ok(return_value)); - Some(Packet { heap, value }) - } - Status::Panicked { reason } => { - warn!("Child panicked with reason {reason}"); - let mut heap = Heap::default(); - let reason = heap.create_text(reason); - let value = heap.create_result(Err(reason)); - Some(Packet { heap, value }) - } - _ => None, - }; - if let Some(packet) = packet { - self.channel_operations - .entry(result_channel) - .or_default() - .push_back(ChannelOperation::Send { packet }) + // If this was a spawned channel and it ended execution, the result should be + // transmitted to the channel that's returned by the `core.async` call. + if let Some((index, result_channel)) = index_and_result_channel { + let packet = match vm.status() { + Status::Done => { + info!("Child done."); + let (_, vm) = children.remove(index); + let TearDownResult { + heap: vm_heap, + result, + .. + } = vm.tear_down(); + let return_value = result.unwrap(); + let mut heap = Heap::default(); + let return_value = + vm_heap.clone_single_to_other_heap(&mut heap, return_value); + let value = heap.create_result(Ok(return_value)); + Some(Packet { heap, value }) + } + Status::Panicked { reason } => { + warn!("Child panicked with reason {reason}"); + let mut heap = Heap::default(); + let reason = heap.create_text(reason); + let value = heap.create_result(Err(reason)); + Some(Packet { heap, value }) } + _ => None, + }; + if let Some(packet) = packet { + self.channel_operations + .entry(result_channel) + .or_default() + .push_back(ChannelOperation::Send { packet }) } + } - // Update status and state. - if children.iter().any(|(_, vm)| vm.is_running()) { - self.status = Status::Running - } - if children.iter().all(|(_, vm)| vm.is_finished()) { - paused_main_fiber.complete_parallel_scope(); - self.status = Status::Running; - break 'new_state State::SingleFiber(paused_main_fiber); - } + // Update status and state. + if children.iter().any(|(_, vm)| vm.is_running()) { + self.status = Status::Running + } + if children.iter().all(|(_, vm)| vm.is_finished()) { + paused_main_fiber.complete_parallel_scope(); + self.status = Status::Running; + return State::SingleFiber(paused_main_fiber); + } - State::ParallelSection { - paused_main_fiber, - nursery, - children, - } + State::ParallelSection { + paused_main_fiber, + nursery, + children, } } - }; - (new_state, num_instructions_executed) - } - pub fn run_synchronously_until_completion(mut self, db: &Database) -> TearDownResult { - let use_provider = DbUseProvider { db }; - loop { - self.run(&use_provider, 100000); - match self.status() { - Status::Running => info!("Code is still running."), - _ => return self.tear_down(), - } } } diff --git a/compiler/src/vm/use_provider.rs b/compiler/src/vm/use_module.rs similarity index 68% rename from compiler/src/vm/use_provider.rs rename to compiler/src/vm/use_module.rs index 061675e74..e0cc45f9d 100644 --- a/compiler/src/vm/use_provider.rs +++ b/compiler/src/vm/use_module.rs @@ -1,54 +1,25 @@ use super::{ - fiber::Fiber, - heap::{Closure, Data, Heap, Pointer}, + context::{Context, UseResult}, + heap::Data, + Closure, Fiber, Heap, Pointer, }; use crate::{ - compiler::{ - hir_to_lir::HirToLir, - lir::{Instruction, Lir}, - }, - database::Database, - module::{Module, ModuleDb, ModuleKind}, + compiler::lir::Instruction, + module::{Module, ModuleKind}, }; use itertools::Itertools; -pub trait UseProvider { - fn use_module(&self, module: Module) -> Result; -} -pub enum UseResult { - Asset(Vec), - Code(Lir), -} - -pub struct DbUseProvider<'a> { - pub db: &'a Database, -} -impl<'a> UseProvider for DbUseProvider<'a> { - fn use_module(&self, module: Module) -> Result { - match module.kind { - ModuleKind::Asset => match self.db.get_module_content(module.clone()) { - Some(bytes) => Ok(UseResult::Asset((*bytes).clone())), - None => Err(format!("use couldn't import the asset module `{}`", module)), - }, - ModuleKind::Code => match self.db.lir(module.clone()) { - Some(lir) => Ok(UseResult::Code((*lir).clone())), - None => Err(format!("use couldn't import the code module `{}`", module)), - }, - } - } -} - impl Fiber { - pub fn use_module( + pub fn use_module( &mut self, - use_provider: &U, + context: &C, current_module: Module, relative_path: Pointer, ) -> Result<(), String> { let target = UsePath::parse(&self.heap, relative_path)?; let module = target.resolve_relative_to(current_module)?; - match use_provider.use_module(module.clone())? { + match context.use_module(module.clone())? { UseResult::Asset(bytes) => { let bytes = bytes .iter() @@ -61,7 +32,7 @@ impl Fiber { let module_closure = Closure::of_lir(module, lir); let address = self.heap.create_closure(module_closure); self.data_stack.push(address); - self.run_instruction(use_provider, Instruction::Call { num_args: 0 }); + self.run_instruction(context, Instruction::Call { num_args: 0 }); } } From 7925556531c8a7e0d859a5e7a32274b104e715a9 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Fri, 26 Aug 2022 13:05:44 +0200 Subject: [PATCH 11/59] Update todos --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b41d8928a..573a8acb0 100644 --- a/README.md +++ b/README.md @@ -122,16 +122,18 @@ Major milestones: - add caching while compile-time evaluating code - pattern matching - pipe operator +- text interpolation - eliminate common subtrees - inline functions - minimize inputs found through fuzzing - fuzz parser -- support recursion - tail call optimization - new name? - parse function declaration with doc comment but no code - complain about comment lines with too much indentation - develop guidelines about how to format reasons +- module files: `.candy` vs `_.candy`? +- casing of module names? ## How to use Candy From 4ab75e027d67deab230a96f0f00bc5b82e048e1a Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Fri, 26 Aug 2022 13:05:51 +0200 Subject: [PATCH 12/59] Create better benchmark --- packages/Benchmark.candy | 133 ++++++++++----------------------------- 1 file changed, 34 insertions(+), 99 deletions(-) diff --git a/packages/Benchmark.candy b/packages/Benchmark.candy index 7d5560f0f..0dd16d125 100644 --- a/packages/Benchmark.candy +++ b/packages/Benchmark.candy @@ -1,100 +1,35 @@ core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" -core = use "..Core" +✨.print "Imported Core" + +#fibRec fibRec n = +# core.ifElse (core.int.isLessThan n 2) { n } { +# core.int.add +# fibRec fibRec (core.int.subtract n 1) +# fibRec fibRec (core.int.subtract n 2) +# } +#fib n = fibRec fibRec n +# +#✨.print "Defined fib" +#✨.print (fib 8) + + +channel = core.channel.create 3 +sendPort = core.struct.getUnwrap channel 0 +receivePort = core.struct.getUnwrap channel 1 +core.channel.send sendPort "One" +core.channel.send sendPort "Two" +✨.print (core.channel.receive receivePort) +core.channel.send sendPort "Three" +✨.print (core.channel.receive receivePort) +✨.print (core.channel.receive receivePort) + +core.parallel { nursery -> + ✨.print (core.await (core.async nursery { + ✨.print "Hello from fiber!" + "Hello, async await!" + })) + + core.async nursery { ✨.print "Kiwi" } + core.async nursery { ✨.print "Banana" } + ✨.print "Hi" +} From 832f7ae31edcc8c0b95259335fd2842e4cba8222 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Fri, 26 Aug 2022 13:06:51 +0200 Subject: [PATCH 13/59] Continue implementing the pushing of operations --- compiler/src/vm/channel.rs | 3 + compiler/src/vm/tree.rs | 289 ++++++++++++++++++++++++------------- 2 files changed, 190 insertions(+), 102 deletions(-) diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index e0b844da2..04c5e04a6 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -31,6 +31,9 @@ impl Channel { } } + pub fn is_empty(&self) -> bool { + self.packets.is_empty() + } pub fn is_full(&self) -> bool { self.packets.len() == self.capacity } diff --git a/compiler/src/vm/tree.rs b/compiler/src/vm/tree.rs index 77dfee04d..bde850ba7 100644 --- a/compiler/src/vm/tree.rs +++ b/compiler/src/vm/tree.rs @@ -10,6 +10,7 @@ use crate::vm::fiber; use rand::seq::IteratorRandom; use std::{ collections::{HashMap, VecDeque}, + marker::PhantomData, mem, }; use tracing::{debug, info, warn}; @@ -145,11 +146,11 @@ pub struct FiberTree { // managed ("owned") by this node directly. For channels owned by the // outside world (such as those referenced in the environment argument), // this node maintains a mapping between internal and external IDs. - pub internal_channels: HashMap, - pub next_internal_channel_id: ChannelId, + pub internal_channels: HashMap, + external_operations: HashMap, + internal_channel_id_generator: IdGenerator, pub external_to_internal_channels: HashMap, pub internal_to_external_channels: HashMap, - pub channel_operations: HashMap>, } #[derive(Clone, Debug)] @@ -172,17 +173,29 @@ enum State { ParallelSection { paused_main_fiber: Fiber, // Should have Status::InParallelSection. nursery: ChannelId, + child_id_generator: IdGenerator, /// Children and a channels where to send the result of the child. The /// channel's receive port is directly returned by the `core.async` function. /// Here, we save the ID of the channel where the result of the VM will /// be sent. - children: Vec<(ChannelId, FiberTree)>, + children: HashMap, }, } +type ChildId = usize; + +/// A list of pending operations. All operations should be of the same kind +/// – if a send and a receive operation meet, they should cancel each other out. +type Operations = VecDeque; + #[derive(Clone)] -pub enum ChannelOperation { +pub struct Operation { + child: ChildId, + kind: OperationKind, +} +#[derive(Clone)] +pub enum OperationKind { Send { packet: Packet }, Receive, } @@ -197,10 +210,10 @@ impl FiberTree { }, state: None, internal_channels: Default::default(), - next_internal_channel_id: 0, + internal_channel_id_generator: IdGenerator::new(), external_to_internal_channels: Default::default(), internal_to_external_channels: Default::default(), - channel_operations: Default::default(), + external_operations: Default::default(), }; tree.create_channel_mappings_for(&fiber.heap); fiber @@ -260,7 +273,7 @@ impl FiberTree { | Data::ReceivePort(ReceivePort { channel }) = object.data { if !self.external_to_internal_channels.contains_key(&channel) { - let internal_id = self.generate_channel_id(); + let internal_id = self.internal_channel_id_generator.generate(); self.external_to_internal_channels .insert(channel, internal_id); self.internal_to_external_channels @@ -270,6 +283,61 @@ impl FiberTree { } } + fn push_operation(&mut self, channel_id: ChannelId, new_operation: Operation) { + if let Some((channel, operations)) = self.internal_channels.get_mut(&channel_id) { + // Internal channel. + if let Some(operation) = operations.front() && new_operation.cancels_out(operation) { + let operation = operations.pop_front().unwrap(); + self.complete_canceling_operations(channel_id, operation, new_operation); + return; + } + match &new_operation.kind { + OperationKind::Send { packet } => { + if channel.send(packet.clone()) { + self.complete_send(channel_id, new_operation.child); + return; + } + } + OperationKind::Receive => { + if let Some(packet) = channel.receive() { + self.complete_receive(channel_id, new_operation.child, packet); + return; + } + } + } + operations.push_back(new_operation); + } else { + // External channel. + let channel_id = self.internal_to_external_channels[&channel_id]; + let operations = self.external_operations.entry(channel_id).or_default(); + if let Some(operation) = operations.front() && new_operation.cancels_out(operation) { + let operation = operations.pop_front().unwrap(); + self.complete_canceling_operations(channel_id, operation, new_operation); + } else { + operations.push_back(new_operation); + } + } + } + fn complete_canceling_operations(&mut self, channel: ChannelId, a: Operation, b: Operation) { + match (a.kind, b.kind) { + (OperationKind::Send { packet }, OperationKind::Receive) => { + self.complete_send(channel, a.child); + self.complete_receive(channel, b.child, packet); + } + (OperationKind::Receive, OperationKind::Send { packet }) => { + self.complete_send(channel, b.child); + self.complete_receive(channel, a.child, packet); + } + _ => panic!("operations do not cancel each other out"), + } + } + fn complete_send(&mut self, channel: ChannelId, child: ChildId) { + todo!() + } + fn complete_receive(&mut self, channel: ChannelId, child: ChildId, packet: Packet) { + todo!() + } + pub fn run(&mut self, context: &mut C) { assert!( matches!(self.status, Status::Running), @@ -291,49 +359,30 @@ impl FiberTree { match fiber.status() { fiber::Status::Running => {} fiber::Status::CreatingChannel { capacity } => { - let id = self.generate_channel_id(); - self.internal_channels.insert(id, Channel::new(capacity)); + let id = self.internal_channel_id_generator.generate(); + self.internal_channels + .insert(id, (Channel::new(capacity), VecDeque::new())); fiber.complete_channel_create(id); } fiber::Status::Sending { channel, packet } => { info!("Sending packet to channel {channel}."); - if let Some(channel) = self.internal_channels.get_mut(&channel) { - // Internal channel. - if channel.send(packet) { - fiber.complete_send(); - } else { - warn!("Tried to send to a full channel that is local to a fiber. This will never complete."); - } - } else { - // External channel. - let channel = self.internal_to_external_channels[&channel]; - self.channel_operations - .entry(channel) - .or_default() - .push_back(ChannelOperation::Send { packet }); - self.status = Status::WaitingForOperations; - } + self.push_operation( + channel, + Operation { + child: 0, + kind: OperationKind::Send { packet }, + }, + ); } fiber::Status::Receiving { channel } => { - info!("Sending packet to channel {channel}."); - if let Some(channel) = self.internal_channels.get_mut(&channel) { - // Internal channel. - if let Some(packet) = channel.receive() { - fiber.complete_receive(packet); - } else { - warn!( - "Tried to receive from an empty channel that is local to a fiber. This will never complete." - ); - } - } else { - // External channel. - let channel = self.internal_to_external_channels[&channel]; - self.channel_operations - .entry(channel) - .or_default() - .push_back(ChannelOperation::Receive); - self.status = Status::WaitingForOperations; - } + info!("Receiving packet from channel {channel}."); + self.push_operation( + channel, + Operation { + child: 0, + kind: OperationKind::Receive, + }, + ); } fiber::Status::InParallelScope { body, @@ -342,91 +391,97 @@ impl FiberTree { info!("Entering parallel scope."); let mut heap = Heap::default(); let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); - let nursery = self.generate_channel_id(); + let nursery = self.internal_channel_id_generator.generate(); let nursery_send_port = heap.create_send_port(nursery); let tree = FiberTree::new_for_running_closure(heap, body, &[nursery_send_port]); + let mut fiber_id_generator = IdGenerator::start_at(1); + let mut children = HashMap::new(); + children.insert(fiber_id_generator.generate(), (return_channel, tree)); + return State::ParallelSection { paused_main_fiber: fiber, nursery, - children: vec![(return_channel, tree)], + child_id_generator: fiber_id_generator, + children, }; } - fiber::Status::Done => { - info!("Fiber done."); - self.status = Status::Done; - } + fiber::Status::Done => info!("Fiber done."), fiber::Status::Panicked { reason } => { - info!("Fiber panicked because of {reason}."); - self.status = Status::Panicked { reason }; + info!("Fiber panicked because of {reason}.") } } + self.status = match fiber.status() { + fiber::Status::Running => Status::Running, + fiber::Status::Sending { .. } | fiber::Status::Receiving { .. } => { + Status::WaitingForOperations + } + fiber::Status::Done => Status::Done, + fiber::Status::Panicked { reason } => Status::Panicked { reason }, + _ => unreachable!(), + }; State::SingleFiber(fiber) } State::ParallelSection { mut paused_main_fiber, nursery, + child_id_generator: fiber_id_generator, mut children, } => { - let (index_and_result_channel, vm) = children + let (child_id, result_channel, vm) = children .iter_mut() - .enumerate() - .map(|(i, (channel, vm))| (Some((i, *channel)), vm)) - .filter(|(_, vm)| matches!(vm.status, Status::Running)) + .map(|(id, (channel, vm))| (*id, *channel, vm)) + .filter(|(_, _, vm)| matches!(vm.status, Status::Running)) .choose(&mut rand::thread_rng()) .expect("Tried to run Vm, but no child can run."); info!("Running child VM."); vm.run(context); - for (channel, operations) in &vm.channel_operations { - // TODO - warn!("Todo: Handle operations on channel {channel}"); - // for operation in operations {} - } - - // If this was a spawned channel and it ended execution, the result should be - // transmitted to the channel that's returned by the `core.async` call. - if let Some((index, result_channel)) = index_and_result_channel { - let packet = match vm.status() { - Status::Done => { - info!("Child done."); - let (_, vm) = children.remove(index); - let TearDownResult { - heap: vm_heap, - result, - .. - } = vm.tear_down(); - let return_value = result.unwrap(); - let mut heap = Heap::default(); - let return_value = - vm_heap.clone_single_to_other_heap(&mut heap, return_value); - let value = heap.create_result(Ok(return_value)); - Some(Packet { heap, value }) - } - Status::Panicked { reason } => { - warn!("Child panicked with reason {reason}"); - let mut heap = Heap::default(); - let reason = heap.create_text(reason); - let value = heap.create_result(Err(reason)); - Some(Packet { heap, value }) - } - _ => None, - }; - if let Some(packet) = packet { - self.channel_operations - .entry(result_channel) - .or_default() - .push_back(ChannelOperation::Send { packet }) + // If the child finished executing, the result should be + // transmitted to the channel that's returned by the + // `core.async` call. + let packet = match vm.status() { + Status::Done => { + info!("Child done."); + let (_, vm) = children.remove(&child_id).unwrap(); + let TearDownResult { + heap: vm_heap, + result, + .. + } = vm.tear_down(); + let return_value = result.unwrap(); + let mut heap = Heap::default(); + let return_value = + vm_heap.clone_single_to_other_heap(&mut heap, return_value); + let value = heap.create_result(Ok(return_value)); + Some(Packet { heap, value }) } + Status::Panicked { reason } => { + warn!("Child panicked with reason {reason}"); + let mut heap = Heap::default(); + let reason = heap.create_text(reason); + let value = heap.create_result(Err(reason)); + Some(Packet { heap, value }) + } + _ => None, + }; + if let Some(packet) = packet { + self.push_operation( + result_channel, + Operation { + child: child_id, + kind: OperationKind::Send { packet }, + }, + ); } // Update status and state. - if children.iter().any(|(_, vm)| vm.is_running()) { + if children.values().any(|(_, tree)| tree.is_running()) { self.status = Status::Running } - if children.iter().all(|(_, vm)| vm.is_finished()) { + if children.values().all(|(_, tree)| tree.is_finished()) { paused_main_fiber.complete_parallel_scope(); self.status = Status::Running; return State::SingleFiber(paused_main_fiber); @@ -435,16 +490,21 @@ impl FiberTree { State::ParallelSection { paused_main_fiber, nursery, + child_id_generator: fiber_id_generator, children, } } } } +} - fn generate_channel_id(&mut self) -> ChannelId { - let id = self.next_internal_channel_id; - self.next_internal_channel_id += 1; - id +impl Operation { + fn cancels_out(&self, other: &Self) -> bool { + matches!( + (&self.kind, &other.kind), + (OperationKind::Send { .. }, OperationKind::Receive) + | (OperationKind::Receive, OperationKind::Send { .. }) + ) } } @@ -459,3 +519,28 @@ impl Heap { } } } + +#[derive(Clone)] +struct IdGenerator> { + next_id: usize, + _data: PhantomData, +} +impl> IdGenerator { + fn new() -> Self { + Self { + next_id: 0, + _data: Default::default(), + } + } + fn start_at(id: usize) -> Self { + Self { + next_id: id, + _data: Default::default(), + } + } + fn generate(&mut self) -> T { + let id = self.next_id; + self.next_id += 1; + id.into() + } +} From 12f83810a88848682a8dcdfb3e91633bfef55890 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sat, 27 Aug 2022 23:19:57 +0200 Subject: [PATCH 14/59] Continue implementing channel operations --- compiler/src/vm/tree.rs | 187 +++++++++++++++++++++++++++++----------- 1 file changed, 139 insertions(+), 48 deletions(-) diff --git a/compiler/src/vm/tree.rs b/compiler/src/vm/tree.rs index bde850ba7..0bccf3968 100644 --- a/compiler/src/vm/tree.rs +++ b/compiler/src/vm/tree.rs @@ -146,8 +146,7 @@ pub struct FiberTree { // managed ("owned") by this node directly. For channels owned by the // outside world (such as those referenced in the environment argument), // this node maintains a mapping between internal and external IDs. - pub internal_channels: HashMap, - external_operations: HashMap, + pub internal_channels: HashMap)>, internal_channel_id_generator: IdGenerator, pub external_to_internal_channels: HashMap, pub internal_to_external_channels: HashMap, @@ -180,18 +179,24 @@ enum State { /// Here, we save the ID of the channel where the result of the VM will /// be sent. children: HashMap, + + /// Channel operations may be long-running and complete in any order. + /// That's why we expose operations to the parent node in the fiber + /// tree. The parent can complete operations by calling the `complete_*` + /// methods. + pending_operations: HashMap, + operation_id_generator: IdGenerator, + operation_id_to_child_and_its_operation_id: HashMap, }, } type ChildId = usize; - -/// A list of pending operations. All operations should be of the same kind -/// – if a send and a receive operation meet, they should cancel each other out. -type Operations = VecDeque; +type OperationId = usize; #[derive(Clone)] pub struct Operation { - child: ChildId, + id: OperationId, + channel: ChannelId, kind: OperationKind, } #[derive(Clone)] @@ -213,7 +218,6 @@ impl FiberTree { internal_channel_id_generator: IdGenerator::new(), external_to_internal_channels: Default::default(), internal_to_external_channels: Default::default(), - external_operations: Default::default(), }; tree.create_channel_mappings_for(&fiber.heap); fiber @@ -246,6 +250,42 @@ impl FiberTree { fn is_finished(&self) -> bool { matches!(self.status, Status::Done | Status::Panicked { .. }) } + + fn operations(&self) -> HashMap { + match self.state.unwrap() { + State::SingleFiber(fiber) => { + let mut operations = HashMap::new(); + match fiber.status() { + fiber::Status::Sending { channel, packet } => { + operations.insert( + 0, + Operation { + id: 0, + channel, + kind: OperationKind::Send { packet }, + }, + ); + } + fiber::Status::Receiving { channel } => { + operations.insert( + 0, + Operation { + id: 0, + channel, + kind: OperationKind::Receive, + }, + ); + } + _ => {} + } + operations + } + State::ParallelSection { + pending_operations, .. + } => pending_operations, + } + } + pub fn fiber(&self) -> &Fiber { match self.state() { State::SingleFiber(fiber) => fiber, @@ -283,24 +323,25 @@ impl FiberTree { } } - fn push_operation(&mut self, channel_id: ChannelId, new_operation: Operation) { - if let Some((channel, operations)) = self.internal_channels.get_mut(&channel_id) { + fn push_operation(&mut self, new_operation: Operation) { + if let Some((channel, operations)) = self.internal_channels.get_mut(&new_operation.channel) + { // Internal channel. if let Some(operation) = operations.front() && new_operation.cancels_out(operation) { let operation = operations.pop_front().unwrap(); - self.complete_canceling_operations(channel_id, operation, new_operation); + self.complete_canceling_operations(operation, new_operation); return; } match &new_operation.kind { OperationKind::Send { packet } => { if channel.send(packet.clone()) { - self.complete_send(channel_id, new_operation.child); + self.complete_send(0); return; } } OperationKind::Receive => { if let Some(packet) = channel.receive() { - self.complete_receive(channel_id, new_operation.child, packet); + self.complete_receive(0, packet); return; } } @@ -308,34 +349,80 @@ impl FiberTree { operations.push_back(new_operation); } else { // External channel. - let channel_id = self.internal_to_external_channels[&channel_id]; - let operations = self.external_operations.entry(channel_id).or_default(); - if let Some(operation) = operations.front() && new_operation.cancels_out(operation) { - let operation = operations.pop_front().unwrap(); - self.complete_canceling_operations(channel_id, operation, new_operation); - } else { - operations.push_back(new_operation); + match self.state.unwrap() { + State::SingleFiber(_) => { + // Nothing to do. When the parent node asks for this fiber's + // `operations()`, the single pending operation will be + // created and communicated on-the-fly based on the status + // of the fiber. + } + State::ParallelSection { + paused_main_fiber, + nursery, + child_id_generator, + children, + pending_operations, + operation_id_generator, + operation_id_to_child_and_its_operation_id, + } => { + let id = operation_id_generator.generate(); + let channel = self.internal_to_external_channels[&new_operation.channel]; + pending_operations.insert( + id, + Operation { + channel, + ..new_operation + }, + ); + } } } } - fn complete_canceling_operations(&mut self, channel: ChannelId, a: Operation, b: Operation) { + fn complete_canceling_operations(&mut self, a: Operation, b: Operation) { + assert_eq!(a.channel, b.channel); match (a.kind, b.kind) { (OperationKind::Send { packet }, OperationKind::Receive) => { - self.complete_send(channel, a.child); - self.complete_receive(channel, b.child, packet); + self.complete_send(a.id); + self.complete_receive(b.id, packet); } (OperationKind::Receive, OperationKind::Send { packet }) => { - self.complete_send(channel, b.child); - self.complete_receive(channel, a.child, packet); + self.complete_send(b.id); + self.complete_receive(a.id, packet); } _ => panic!("operations do not cancel each other out"), } } - fn complete_send(&mut self, channel: ChannelId, child: ChildId) { - todo!() + fn complete_send(&mut self, id: OperationId) { + match self.state.unwrap() { + State::SingleFiber(fiber) => { + assert_eq!(id, 0); + fiber.complete_send(); + } + State::ParallelSection { + children, + operation_id_to_child_and_its_operation_id, + .. + } => { + let (operation_id, child_id) = operation_id_to_child_and_its_operation_id[&id]; + children[&child_id].1.complete_send(operation_id); + } + } } - fn complete_receive(&mut self, channel: ChannelId, child: ChildId, packet: Packet) { - todo!() + fn complete_receive(&mut self, id: OperationId, packet: Packet) { + match self.state.unwrap() { + State::SingleFiber(fiber) => { + assert_eq!(id, 0); + fiber.complete_receive(packet); + } + State::ParallelSection { + children, + operation_id_to_child_and_its_operation_id, + .. + } => { + let (operation_id, child_id) = operation_id_to_child_and_its_operation_id[&id]; + children[&child_id].1.complete_receive(operation_id, packet); + } + } } pub fn run(&mut self, context: &mut C) { @@ -366,23 +453,19 @@ impl FiberTree { } fiber::Status::Sending { channel, packet } => { info!("Sending packet to channel {channel}."); - self.push_operation( + self.push_operation(Operation { + id: 0, channel, - Operation { - child: 0, - kind: OperationKind::Send { packet }, - }, - ); + kind: OperationKind::Send { packet }, + }); } fiber::Status::Receiving { channel } => { info!("Receiving packet from channel {channel}."); - self.push_operation( + self.push_operation(Operation { + id: 0, channel, - Operation { - child: 0, - kind: OperationKind::Receive, - }, - ); + kind: OperationKind::Receive, + }); } fiber::Status::InParallelScope { body, @@ -405,6 +488,9 @@ impl FiberTree { nursery, child_id_generator: fiber_id_generator, children, + pending_operations: Default::default(), + operation_id_generator: IdGenerator::start_at(0), + operation_id_to_child_and_its_operation_id: Default::default(), }; } fiber::Status::Done => info!("Fiber done."), @@ -428,6 +514,10 @@ impl FiberTree { nursery, child_id_generator: fiber_id_generator, mut children, + pending_operations, + mut operation_id_generator, + operation_id_to_child_and_its_operation_id, + .. } => { let (child_id, result_channel, vm) = children .iter_mut() @@ -468,13 +558,11 @@ impl FiberTree { _ => None, }; if let Some(packet) = packet { - self.push_operation( - result_channel, - Operation { - child: child_id, - kind: OperationKind::Send { packet }, - }, - ); + self.push_operation(Operation { + id: operation_id_generator.generate(), + channel: result_channel, + kind: OperationKind::Send { packet }, + }); } // Update status and state. @@ -492,6 +580,9 @@ impl FiberTree { nursery, child_id_generator: fiber_id_generator, children, + pending_operations, + operation_id_generator, + operation_id_to_child_and_its_operation_id, } } } From 01b5ca3cc6c68158b3d0094878724b253b83c5a5 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sat, 3 Sep 2022 18:20:50 +0200 Subject: [PATCH 15/59] Make channel more efficient --- compiler/src/vm/channel.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index 04c5e04a6..575330746 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -27,7 +27,7 @@ impl Channel { pub fn new(capacity: Capacity) -> Self { Self { capacity, - packets: Default::default(), + packets: VecDeque::with_capacity(capacity), } } From 6b41d82bc41666462385be695573bd3d434b7cf3 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sun, 4 Sep 2022 17:16:09 +0200 Subject: [PATCH 16/59] Specify Rust version --- compiler/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index 3a7512340..24476a700 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -2,6 +2,7 @@ name = "candy" version = "0.1.0" edition = "2021" +rust-version = "1.56" [profile.release] debug = true From 6bc1bae577a59f9b737540169de8f3a7811f3e75 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sun, 4 Sep 2022 17:16:32 +0200 Subject: [PATCH 17/59] Improve Packet's Debug implementation --- compiler/src/vm/channel.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index 575330746..7acd99d78 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -1,5 +1,5 @@ use super::{Heap, Pointer}; -use std::collections::VecDeque; +use std::{collections::VecDeque, fmt}; /// A conveyer belt or pipe that flows between send and receive ports in the /// program. Using send ports, you can put packets into a channel. Using receive @@ -8,7 +8,7 @@ use std::collections::VecDeque; /// Channels always have a maximum capacity of packets that they can hold /// simultaneously – you can set it to something large, but having no capacity /// enables buggy code that leaks memory. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Channel { pub capacity: Capacity, packets: VecDeque, @@ -17,7 +17,7 @@ pub struct Channel { pub type Capacity = usize; /// A self-contained value that is sent over a channel. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct Packet { pub heap: Heap, pub value: Pointer, @@ -50,3 +50,9 @@ impl Channel { self.packets.pop_front() } } + +impl fmt::Debug for Packet { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.value.format(&self.heap)) + } +} From fa02344b914137c25f67fbb5e82c3020984c6019 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sun, 4 Sep 2022 17:16:46 +0200 Subject: [PATCH 18/59] Continue implementing channel (WIP) --- compiler/src/main.rs | 24 +++-- compiler/src/vm/tree.rs | 230 ++++++++++++++++++---------------------- 2 files changed, 120 insertions(+), 134 deletions(-) diff --git a/compiler/src/main.rs b/compiler/src/main.rs index c633fda56..3578b8dc3 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -1,5 +1,6 @@ #![feature(async_closure)] #![feature(box_patterns)] +#![feature(let_chains)] #![feature(never_type)] #![feature(try_trait_v2)] #![allow(clippy::module_inception)] @@ -27,7 +28,7 @@ use crate::{ module::{Module, ModuleKind}, vm::{ context::{DbUseProvider, ModularContext, RunForever}, - Closure, FiberTree, TearDownResult, + Closure, FiberTree, TearDownResult, tree::Status, }, }; use compiler::lir::Lir; @@ -203,12 +204,23 @@ fn run(options: CandyRunOptions) { let path_string = options.file.to_string_lossy(); info!("Running `{path_string}`."); - let mut vm = FiberTree::new_for_running_module_closure(module_closure); - vm.run(&mut ModularContext { - use_provider: DbUseProvider { db: &db }, - execution_controller: RunForever, - }); + loop { + match vm.status() { + Status::Running => { + info!("VM still running."); + let operations = vm.run(&mut ModularContext { + use_provider: DbUseProvider { db: &db }, + execution_controller: RunForever, + }); + info!("Operations: {operations:?}"); + }, + Status::WaitingForOperations => { + todo!("VM can't proceed until some operations complete."); + }, + _ => break, + } + } let TearDownResult { tracer, result, diff --git a/compiler/src/vm/tree.rs b/compiler/src/vm/tree.rs index 0bccf3968..486edd642 100644 --- a/compiler/src/vm/tree.rs +++ b/compiler/src/vm/tree.rs @@ -140,16 +140,23 @@ pub struct FiberTree { status: Status, state: Option, // Only `None` temporarily during state transitions. - // Channel functionality. Fiber trees communicate with the outer world using - // channels. Each channel is identified using an ID that is valid inside - // this particular tree node. Channels created by the current fiber are - // managed ("owned") by this node directly. For channels owned by the - // outside world (such as those referenced in the environment argument), - // this node maintains a mapping between internal and external IDs. + // Fiber trees communicate with the outer world using channels. Each channel + // is identified using an ID that is valid inside this particular tree node. + // Channels that are only used in this subtree are stored here directly. pub internal_channels: HashMap)>, internal_channel_id_generator: IdGenerator, pub external_to_internal_channels: HashMap, pub internal_to_external_channels: HashMap, + + // Channel operations may be long-running and complete in any order. That's + // why we expose operations to the parent node in this map. The parent can + // call our `complete_` methods with an operation ID to indicate that the + // operation is finished. + // The parent can remove entries from this map at will. We should be able + // to calling the `complete_*` methods. This makes it efficient to + // propagate operations up the tree. For example, a send operation + // containing a large value can just propagate upwards. + // pending_operations: HashMap, } #[derive(Clone, Debug)] @@ -163,6 +170,9 @@ pub enum Status { #[derive(Clone)] enum State { /// This tree is currently focused on running a single fiber. + /// + /// Since we only have at most one channel operation running at any given + /// time, the only valid operation ID is 0. SingleFiber(Fiber), /// The fiber of this tree entered a `core.parallel` scope so that it's now @@ -171,6 +181,7 @@ enum State { /// `core.parallel` as well as any other spawned children. ParallelSection { paused_main_fiber: Fiber, // Should have Status::InParallelSection. + nursery: ChannelId, child_id_generator: IdGenerator, @@ -180,11 +191,6 @@ enum State { /// be sent. children: HashMap, - /// Channel operations may be long-running and complete in any order. - /// That's why we expose operations to the parent node in the fiber - /// tree. The parent can complete operations by calling the `complete_*` - /// methods. - pending_operations: HashMap, operation_id_generator: IdGenerator, operation_id_to_child_and_its_operation_id: HashMap, }, @@ -193,13 +199,13 @@ enum State { type ChildId = usize; type OperationId = usize; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Operation { id: OperationId, channel: ChannelId, kind: OperationKind, } -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum OperationKind { Send { packet: Packet }, Receive, @@ -218,6 +224,7 @@ impl FiberTree { internal_channel_id_generator: IdGenerator::new(), external_to_internal_channels: Default::default(), internal_to_external_channels: Default::default(), + // pending_operations: Default::default(), }; tree.create_channel_mappings_for(&fiber.heap); fiber @@ -233,6 +240,7 @@ impl FiberTree { Self::new_with_fiber(Fiber::new_for_running_module_closure(closure)) } pub fn tear_down(self) -> TearDownResult { + debug!("FiberTree::tear_down called (our status = {:?}).", self.status); match self.into_state() { State::SingleFiber(fiber) => fiber.tear_down(), State::ParallelSection { .. } => { @@ -251,42 +259,7 @@ impl FiberTree { matches!(self.status, Status::Done | Status::Panicked { .. }) } - fn operations(&self) -> HashMap { - match self.state.unwrap() { - State::SingleFiber(fiber) => { - let mut operations = HashMap::new(); - match fiber.status() { - fiber::Status::Sending { channel, packet } => { - operations.insert( - 0, - Operation { - id: 0, - channel, - kind: OperationKind::Send { packet }, - }, - ); - } - fiber::Status::Receiving { channel } => { - operations.insert( - 0, - Operation { - id: 0, - channel, - kind: OperationKind::Receive, - }, - ); - } - _ => {} - } - operations - } - State::ParallelSection { - pending_operations, .. - } => pending_operations, - } - } - - pub fn fiber(&self) -> &Fiber { + pub fn fiber(&self) -> &Fiber { // TODO: Remove before merging the PR match self.state() { State::SingleFiber(fiber) => fiber, State::ParallelSection { @@ -299,12 +272,12 @@ impl FiberTree { } fn into_state(self) -> State { self.state - .expect("Tried to get VM state during state transition") + .expect("Tried to get tree state during state transition") } fn state(&self) -> &State { self.state .as_ref() - .expect("Tried to get VM state during state transition") + .expect("Tried to get tree state during state transition") } fn create_channel_mappings_for(&mut self, heap: &Heap) { @@ -323,61 +296,6 @@ impl FiberTree { } } - fn push_operation(&mut self, new_operation: Operation) { - if let Some((channel, operations)) = self.internal_channels.get_mut(&new_operation.channel) - { - // Internal channel. - if let Some(operation) = operations.front() && new_operation.cancels_out(operation) { - let operation = operations.pop_front().unwrap(); - self.complete_canceling_operations(operation, new_operation); - return; - } - match &new_operation.kind { - OperationKind::Send { packet } => { - if channel.send(packet.clone()) { - self.complete_send(0); - return; - } - } - OperationKind::Receive => { - if let Some(packet) = channel.receive() { - self.complete_receive(0, packet); - return; - } - } - } - operations.push_back(new_operation); - } else { - // External channel. - match self.state.unwrap() { - State::SingleFiber(_) => { - // Nothing to do. When the parent node asks for this fiber's - // `operations()`, the single pending operation will be - // created and communicated on-the-fly based on the status - // of the fiber. - } - State::ParallelSection { - paused_main_fiber, - nursery, - child_id_generator, - children, - pending_operations, - operation_id_generator, - operation_id_to_child_and_its_operation_id, - } => { - let id = operation_id_generator.generate(); - let channel = self.internal_to_external_channels[&new_operation.channel]; - pending_operations.insert( - id, - Operation { - channel, - ..new_operation - }, - ); - } - } - } - } fn complete_canceling_operations(&mut self, a: Operation, b: Operation) { assert_eq!(a.channel, b.channel); match (a.kind, b.kind) { @@ -393,7 +311,8 @@ impl FiberTree { } } fn complete_send(&mut self, id: OperationId) { - match self.state.unwrap() { + debug!("Completing send {id}."); + match self.state.as_mut().unwrap() { State::SingleFiber(fiber) => { assert_eq!(id, 0); fiber.complete_send(); @@ -403,14 +322,15 @@ impl FiberTree { operation_id_to_child_and_its_operation_id, .. } => { - let (operation_id, child_id) = operation_id_to_child_and_its_operation_id[&id]; - children[&child_id].1.complete_send(operation_id); + let (child_id, operation_id) = operation_id_to_child_and_its_operation_id[&id]; + children.get_mut(&child_id).unwrap().1.complete_send(operation_id); } } } fn complete_receive(&mut self, id: OperationId, packet: Packet) { - match self.state.unwrap() { - State::SingleFiber(fiber) => { + debug!("Completing receive {id}."); + match self.state.as_mut().unwrap() { + State::SingleFiber (fiber) => { assert_eq!(id, 0); fiber.complete_receive(packet); } @@ -419,26 +339,73 @@ impl FiberTree { operation_id_to_child_and_its_operation_id, .. } => { - let (operation_id, child_id) = operation_id_to_child_and_its_operation_id[&id]; - children[&child_id].1.complete_receive(operation_id, packet); + let (child_id, operation_id) = operation_id_to_child_and_its_operation_id[&id]; + children.get_mut(&child_id).unwrap().1.complete_receive(operation_id, packet); } } } - pub fn run(&mut self, context: &mut C) { + pub fn run(&mut self, context: &mut C) -> Vec { assert!( - matches!(self.status, Status::Running), - "Called Vm::run on a vm that is not ready to run." + self.is_running(), + "Called FiberTree::run on a tree that is not ready to run." ); let mut state = mem::replace(&mut self.state, None).unwrap(); - while matches!(self.status, Status::Running) && context.should_continue_running() { - state = self.run_and_map_state(state, context); + let mut operations = vec![]; + while self.is_running() && context.should_continue_running() { + state = self.run_and_map_state(state, &mut operations, context); } self.state = Some(state); + debug!("Finished running tree (status = {:?}).", self.status); + + let mut external_operations = vec![]; + for operation in operations { + let (channel, channel_operations) = match self.internal_channels.get_mut(&operation.channel) { + None => { + external_operations.push(operation); + continue; + }, + Some(it) => it, + }; + + let was_completed = match &operation.kind { + OperationKind::Send { packet } => { + if channel.send(packet.clone()) { // TODO: Don't clone + self.complete_send(operation.id); + true + } else { + false + } + } + OperationKind::Receive => { + if let Some(packet) = channel.receive() { + self.complete_receive(operation.id, packet); + true + } else { + false + } + } + }; + + // TODO: Try canceling out with first operation. + // if let Some(operation) = operations.front() && new_operation.cancels_out(operation) { + // let operation = operations.pop_front().unwrap(); + // self.complete_canceling_operations(operation, new_operation); + // return; + // } + + if was_completed { + // TODO: Try completing more operations if that succeeded. + } else { + channel_operations.push_back(operation); + } + } + + operations } - fn run_and_map_state(&mut self, state: State, context: &mut C) -> State { + fn run_and_map_state(&mut self, state: State, operations: &mut Vec, context: &mut C) -> State { match state { - State::SingleFiber(mut fiber) => { + State::SingleFiber (mut fiber) => { debug!("Running fiber (status = {:?}).", fiber.status); fiber.run(context); @@ -453,7 +420,7 @@ impl FiberTree { } fiber::Status::Sending { channel, packet } => { info!("Sending packet to channel {channel}."); - self.push_operation(Operation { + operations.push(Operation { id: 0, channel, kind: OperationKind::Send { packet }, @@ -461,7 +428,7 @@ impl FiberTree { } fiber::Status::Receiving { channel } => { info!("Receiving packet from channel {channel}."); - self.push_operation(Operation { + operations.push(Operation { id: 0, channel, kind: OperationKind::Receive, @@ -488,7 +455,6 @@ impl FiberTree { nursery, child_id_generator: fiber_id_generator, children, - pending_operations: Default::default(), operation_id_generator: IdGenerator::start_at(0), operation_id_to_child_and_its_operation_id: Default::default(), }; @@ -514,9 +480,8 @@ impl FiberTree { nursery, child_id_generator: fiber_id_generator, mut children, - pending_operations, mut operation_id_generator, - operation_id_to_child_and_its_operation_id, + mut operation_id_to_child_and_its_operation_id, .. } => { let (child_id, result_channel, vm) = children @@ -527,7 +492,17 @@ impl FiberTree { .expect("Tried to run Vm, but no child can run."); info!("Running child VM."); - vm.run(context); + let new_operations = vm.run(context); + + for operation in new_operations { + let id = operation_id_generator.generate(); + operation_id_to_child_and_its_operation_id.insert(id, (child_id, operation.id)); + operations.push(Operation { + id, + ..operation + }); + } + // If the child finished executing, the result should be // transmitted to the channel that's returned by the @@ -558,7 +533,7 @@ impl FiberTree { _ => None, }; if let Some(packet) = packet { - self.push_operation(Operation { + operations.push(Operation { id: operation_id_generator.generate(), channel: result_channel, kind: OperationKind::Send { packet }, @@ -580,7 +555,6 @@ impl FiberTree { nursery, child_id_generator: fiber_id_generator, children, - pending_operations, operation_id_generator, operation_id_to_child_and_its_operation_id, } From 2814e69c975a6cc369656c59b56a6c174ba50b20 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sun, 4 Sep 2022 17:16:54 +0200 Subject: [PATCH 19/59] Comment out Fibonacci code --- packages/Benchmark.candy | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/Benchmark.candy b/packages/Benchmark.candy index 535371959..18078db2b 100644 --- a/packages/Benchmark.candy +++ b/packages/Benchmark.candy @@ -3,15 +3,15 @@ core = use "..Core" -fibRec fibRec n = - core.ifElse (core.int.isLessThan n 2) { n } { - core.int.add - fibRec fibRec (core.int.subtract n 1) - fibRec fibRec (core.int.subtract n 2) - } -fib n = fibRec fibRec n - -twentyOne := fib 8 +# fibRec fibRec n = +# core.ifElse (core.int.isLessThan n 2) { n } { +# core.int.add +# fibRec fibRec (core.int.subtract n 1) +# fibRec fibRec (core.int.subtract n 2) +# } +# fib n = fibRec fibRec n +# +# twentyOne := fib 8 channel = core.channel.create 3 sendPort = core.struct.getUnwrap channel 0 From 33d7a28d3c6eda9709e923c2f0a6baa8e1234603 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 5 Sep 2022 00:25:17 +0200 Subject: [PATCH 20/59] Improve debug output --- compiler/src/main.rs | 5 +- compiler/src/vm/channel.rs | 12 +- compiler/src/vm/tree.rs | 236 ++++++++++++++++++++++++++----------- 3 files changed, 180 insertions(+), 73 deletions(-) diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 3578b8dc3..d3d41948d 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -206,14 +206,15 @@ fn run(options: CandyRunOptions) { info!("Running `{path_string}`."); let mut vm = FiberTree::new_for_running_module_closure(module_closure); loop { + info!("Tree: {:#?}", vm); match vm.status() { Status::Running => { - info!("VM still running."); + debug!("VM still running."); let operations = vm.run(&mut ModularContext { use_provider: DbUseProvider { db: &db }, execution_controller: RunForever, }); - info!("Operations: {operations:?}"); + debug!("Operations: {operations:?}"); }, Status::WaitingForOperations => { todo!("VM can't proceed until some operations complete."); diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index 7acd99d78..f3027423c 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -1,3 +1,4 @@ +use itertools::Itertools; use super::{Heap, Pointer}; use std::{collections::VecDeque, fmt}; @@ -8,7 +9,7 @@ use std::{collections::VecDeque, fmt}; /// Channels always have a maximum capacity of packets that they can hold /// simultaneously – you can set it to something large, but having no capacity /// enables buggy code that leaks memory. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct Channel { pub capacity: Capacity, packets: VecDeque, @@ -51,6 +52,15 @@ impl Channel { } } +impl fmt::Debug for Channel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.is_empty() { + write!(f, "") + } else { + write!(f, "{}", self.packets.iter().map(|packet| format!("{:?}", packet)).join(", ")) + } + } +} impl fmt::Debug for Packet { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.value.format(&self.heap)) diff --git a/compiler/src/vm/tree.rs b/compiler/src/vm/tree.rs index 486edd642..9773d63c5 100644 --- a/compiler/src/vm/tree.rs +++ b/compiler/src/vm/tree.rs @@ -7,7 +7,9 @@ use super::{ Closure, Heap, Pointer, TearDownResult, }; use crate::vm::fiber; +use itertools::Itertools; use rand::seq::IteratorRandom; +use core::fmt; use std::{ collections::{HashMap, VecDeque}, marker::PhantomData, @@ -137,7 +139,7 @@ use tracing::{debug, info, warn}; /// TODO: Implement #[derive(Clone)] pub struct FiberTree { - status: Status, + // status: Status, state: Option, // Only `None` temporarily during state transitions. // Fiber trees communicate with the outer world using channels. Each channel @@ -199,7 +201,7 @@ enum State { type ChildId = usize; type OperationId = usize; -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct Operation { id: OperationId, channel: ChannelId, @@ -214,11 +216,11 @@ pub enum OperationKind { impl FiberTree { fn new_with_fiber(mut fiber: Fiber) -> Self { let mut tree = Self { - status: match fiber.status { - fiber::Status::Done => Status::Done, - fiber::Status::Running => Status::Running, - _ => panic!("Tried to create fiber tree with invalid fiber."), - }, + // status: match fiber.status { + // fiber::Status::Done => Status::Done, + // fiber::Status::Running => Status::Running, + // _ => panic!("Tried to create fiber tree with invalid fiber."), + // }, state: None, internal_channels: Default::default(), internal_channel_id_generator: IdGenerator::new(), @@ -240,7 +242,7 @@ impl FiberTree { Self::new_with_fiber(Fiber::new_for_running_module_closure(closure)) } pub fn tear_down(self) -> TearDownResult { - debug!("FiberTree::tear_down called (our status = {:?}).", self.status); + debug!("FiberTree::tear_down called (our status = {:?}).", self.status()); match self.into_state() { State::SingleFiber(fiber) => fiber.tear_down(), State::ParallelSection { .. } => { @@ -250,13 +252,13 @@ impl FiberTree { } pub fn status(&self) -> Status { - self.status.clone() + self.state.as_ref().unwrap().status() } fn is_running(&self) -> bool { - matches!(self.status, Status::Running) + matches!(self.status(), Status::Running) } fn is_finished(&self) -> bool { - matches!(self.status, Status::Done | Status::Panicked { .. }) + matches!(self.status(), Status::Done | Status::Panicked { .. }) } pub fn fiber(&self) -> &Fiber { // TODO: Remove before merging the PR @@ -311,38 +313,10 @@ impl FiberTree { } } fn complete_send(&mut self, id: OperationId) { - debug!("Completing send {id}."); - match self.state.as_mut().unwrap() { - State::SingleFiber(fiber) => { - assert_eq!(id, 0); - fiber.complete_send(); - } - State::ParallelSection { - children, - operation_id_to_child_and_its_operation_id, - .. - } => { - let (child_id, operation_id) = operation_id_to_child_and_its_operation_id[&id]; - children.get_mut(&child_id).unwrap().1.complete_send(operation_id); - } - } + self.state.as_mut().unwrap().complete_send(id); } fn complete_receive(&mut self, id: OperationId, packet: Packet) { - debug!("Completing receive {id}."); - match self.state.as_mut().unwrap() { - State::SingleFiber (fiber) => { - assert_eq!(id, 0); - fiber.complete_receive(packet); - } - State::ParallelSection { - children, - operation_id_to_child_and_its_operation_id, - .. - } => { - let (child_id, operation_id) = operation_id_to_child_and_its_operation_id[&id]; - children.get_mut(&child_id).unwrap().1.complete_receive(operation_id, packet); - } - } + self.state.as_mut().unwrap().complete_receive(id, packet); } pub fn run(&mut self, context: &mut C) -> Vec { @@ -352,17 +326,27 @@ impl FiberTree { ); let mut state = mem::replace(&mut self.state, None).unwrap(); let mut operations = vec![]; - while self.is_running() && context.should_continue_running() { + // while state.is_running() && context.should_continue_running() { state = self.run_and_map_state(state, &mut operations, context); - } + // } self.state = Some(state); - debug!("Finished running tree (status = {:?}).", self.status); + debug!("Finished running tree (status = {:?}).", self.status()); let mut external_operations = vec![]; for operation in operations { let (channel, channel_operations) = match self.internal_channels.get_mut(&operation.channel) { None => { - external_operations.push(operation); + if let State::ParallelSection { nursery, children , ..} = self.state.as_mut().unwrap() && operation.channel == *nursery { + info!("Operation is for nursery."); + todo!(); + continue; + } + info!("Operation is for channel ch#{}, which is an external channel: {operation:?}", operation.channel); + todo!(); // Migrate channels if necessary. + external_operations.push(Operation { + channel: self.internal_to_external_channels[&operation.channel], + ..operation}); + info!("In particular, it corresponds to channel ch#{} in the outer node.", self.internal_to_external_channels[&operation.channel]); continue; }, Some(it) => it, @@ -371,7 +355,7 @@ impl FiberTree { let was_completed = match &operation.kind { OperationKind::Send { packet } => { if channel.send(packet.clone()) { // TODO: Don't clone - self.complete_send(operation.id); + self.state.as_mut().unwrap().complete_send(operation.id); true } else { false @@ -379,7 +363,7 @@ impl FiberTree { } OperationKind::Receive => { if let Some(packet) = channel.receive() { - self.complete_receive(operation.id, packet); + self.state.as_mut().unwrap().complete_receive(operation.id, packet); true } else { false @@ -400,8 +384,8 @@ impl FiberTree { channel_operations.push_back(operation); } } - - operations + + external_operations } fn run_and_map_state(&mut self, state: State, operations: &mut Vec, context: &mut C) -> State { match state { @@ -419,7 +403,7 @@ impl FiberTree { fiber.complete_channel_create(id); } fiber::Status::Sending { channel, packet } => { - info!("Sending packet to channel {channel}."); + debug!("Sending packet to channel {channel}."); operations.push(Operation { id: 0, channel, @@ -427,7 +411,7 @@ impl FiberTree { }); } fiber::Status::Receiving { channel } => { - info!("Receiving packet from channel {channel}."); + debug!("Receiving packet from channel {channel}."); operations.push(Operation { id: 0, channel, @@ -438,7 +422,7 @@ impl FiberTree { body, return_channel, } => { - info!("Entering parallel scope."); + debug!("Entering parallel scope."); let mut heap = Heap::default(); let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); let nursery = self.internal_channel_id_generator.generate(); @@ -461,18 +445,9 @@ impl FiberTree { } fiber::Status::Done => info!("Fiber done."), fiber::Status::Panicked { reason } => { - info!("Fiber panicked because of {reason}.") + debug!("Fiber panicked because of {reason}.") } } - self.status = match fiber.status() { - fiber::Status::Running => Status::Running, - fiber::Status::Sending { .. } | fiber::Status::Receiving { .. } => { - Status::WaitingForOperations - } - fiber::Status::Done => Status::Done, - fiber::Status::Panicked { reason } => Status::Panicked { reason }, - _ => unreachable!(), - }; State::SingleFiber(fiber) } State::ParallelSection { @@ -487,11 +462,11 @@ impl FiberTree { let (child_id, result_channel, vm) = children .iter_mut() .map(|(id, (channel, vm))| (*id, *channel, vm)) - .filter(|(_, _, vm)| matches!(vm.status, Status::Running)) + .filter(|(_, _, vm)| matches!(vm.status(), Status::Running)) .choose(&mut rand::thread_rng()) .expect("Tried to run Vm, but no child can run."); - info!("Running child VM."); + debug!("Running child VM."); let new_operations = vm.run(context); for operation in new_operations { @@ -502,14 +477,13 @@ impl FiberTree { ..operation }); } - // If the child finished executing, the result should be // transmitted to the channel that's returned by the // `core.async` call. let packet = match vm.status() { Status::Done => { - info!("Child done."); + debug!("Child done."); let (_, vm) = children.remove(&child_id).unwrap(); let TearDownResult { heap: vm_heap, @@ -541,12 +515,8 @@ impl FiberTree { } // Update status and state. - if children.values().any(|(_, tree)| tree.is_running()) { - self.status = Status::Running - } if children.values().all(|(_, tree)| tree.is_finished()) { paused_main_fiber.complete_parallel_scope(); - self.status = Status::Running; return State::SingleFiber(paused_main_fiber); } @@ -563,6 +533,132 @@ impl FiberTree { } } +impl State { + pub fn status(&self) -> Status { + match self { + State::SingleFiber(fiber) => match &fiber.status { + fiber::Status::Running => Status::Running, + fiber::Status::Sending { .. } | + fiber::Status::Receiving { .. } => Status::WaitingForOperations, + fiber::Status::CreatingChannel { .. } | + fiber::Status::InParallelScope { .. } => unreachable!(), + fiber::Status::Done => Status::Done, + fiber::Status::Panicked { reason } => Status::Panicked { reason: reason.clone() }, + }, + State::ParallelSection { children, .. } => { + for (_, child) in children.values() { + return match child.status() { + Status::Running => Status::Running, + Status::WaitingForOperations => Status::WaitingForOperations, + Status::Done => continue, + Status::Panicked { reason } => Status::Panicked { reason }, + }; + } + unreachable!("We should have exited the parallel section") + }, + } + } + fn is_running(&self) -> bool { + matches!(self.status(), Status::Running) + } + + fn complete_send(&mut self, id: OperationId) { + debug!("Completing send {id}."); + match self { + State::SingleFiber(fiber) => { + assert_eq!(id, 0); + fiber.complete_send(); + } + State::ParallelSection { + children, + operation_id_to_child_and_its_operation_id, + .. + } => { + let (child_id, operation_id) = operation_id_to_child_and_its_operation_id[&id]; + children.get_mut(&child_id).unwrap().1.complete_send(operation_id); + } + } + } + fn complete_receive(&mut self, id: OperationId, packet: Packet) { + debug!("Completing receive {id}."); + match self { + State::SingleFiber (fiber) => { + assert_eq!(id, 0); + fiber.complete_receive(packet); + } + State::ParallelSection { + children, + operation_id_to_child_and_its_operation_id, + .. + } => { + let (child_id, operation_id) = operation_id_to_child_and_its_operation_id[&id]; + children.get_mut(&child_id).unwrap().1.complete_receive(operation_id, packet); + } + } + } +} + +impl fmt::Debug for FiberTree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let formatted_channels = { + let mut out = String::from(""); + if !self.internal_to_external_channels.is_empty() { + out.push_str(&format!(" internal to external: {}\n", self.internal_to_external_channels.iter().map(|(internal, external)| format!("{internal}->{external}")).join(", "))); + } + for (id, (channel, pending_operations)) in &self.internal_channels { + out.push_str(&format!(" ch#{id}: {channel:?}")); + if !pending_operations.is_empty() { + out.push_str(&format!(" pending: ")); + for operation in pending_operations { + out.push_str(&format!("{operation:?}")); + } + } + out.push('\n'); + } + out + }; + + match self.state.as_ref().unwrap() { + State::SingleFiber(fiber) => { + writeln!(f, "SingleFiber {{")?; + write!(f, "{formatted_channels}")?; + writeln!(f, " status: {:?}", fiber.status)?; + write!(f, "}}")?; + }, + State::ParallelSection { nursery, children, .. } => { + writeln!(f, "ParallelSection {{")?; + write!(f, "{formatted_channels}")?; + writeln!(f, " nursery: ch#{nursery}")?; + for (id, (result_channel, child)) in children { + write!(f, " child#{id} completing to ch#{result_channel}: ")?; + let child = format!("{child:?}"); + writeln!(f, "{}\n{}", child.lines().nth(0).unwrap(), child.lines().skip(1).map(|line| format!(" {line}")).join("\n"))?; + } + write!(f, "}}")?; + }, + } + Ok(()) + } +} +impl fmt::Debug for State { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SingleFiber(fiber) => write!(f, "SingleFiber ({:?})", fiber.status), + Self::ParallelSection { children, .. } => f.debug_struct("ParallelSection") + .field("children", children).finish(), + } + } +} +impl fmt::Debug for Operation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "op#{} {} ch#{}", self.id, match &self.kind { + OperationKind::Send { packet } => format!("sending {packet:?} to"), + OperationKind::Receive => format!("receiving from"), + }, self.channel) + } +} + + impl Operation { fn cancels_out(&self, other: &Self) -> bool { matches!( From 3ac34ec25370b7fad83b1d990ecd634094f54803 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 5 Sep 2022 09:25:22 +0200 Subject: [PATCH 21/59] Better todos --- compiler/src/vm/tree.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/compiler/src/vm/tree.rs b/compiler/src/vm/tree.rs index 9773d63c5..227f9007a 100644 --- a/compiler/src/vm/tree.rs +++ b/compiler/src/vm/tree.rs @@ -17,15 +17,16 @@ use std::{ }; use tracing::{debug, info, warn}; -/// A fiber tree is a part of or an entire Candy program that thinks it's -/// currently running. Because fiber trees are first-class Rust structs, they -/// enable other code to store "freezed" programs and to remain in control about -/// when and for how long code runs. +/// A fiber tree a Candy program that thinks it's currently running. Everything +/// from a single fiber to a whole program spanning multiple nested parallel +/// scopes is represented as a fiber tree. Because fiber trees are first-class +/// Rust structs, they enable other code to store "freezed" programs and to +/// remain in control about when and for how long code runs. /// -/// While fibers are simple, pure virtual machines that manage a heap and stack, -/// fiber _trees_ encapsulate fibers and manage channels that are used by them. +/// While fibers are "pure" virtual machines that manage a heap and stack, fiber +/// _trees_ encapsulate fibers and manage channels that are used by them. /// -/// ## Fibers +/// ## A Tree of Fibers /// /// As the name suggests, every Candy program can be represented by a tree at /// any point in time. In particular, you can create new nodes in the tree by @@ -326,6 +327,7 @@ impl FiberTree { ); let mut state = mem::replace(&mut self.state, None).unwrap(); let mut operations = vec![]; + // FIXME: Comment in before merging PR. // while state.is_running() && context.should_continue_running() { state = self.run_and_map_state(state, &mut operations, context); // } @@ -335,21 +337,21 @@ impl FiberTree { let mut external_operations = vec![]; for operation in operations { let (channel, channel_operations) = match self.internal_channels.get_mut(&operation.channel) { + Some(it) => it, None => { if let State::ParallelSection { nursery, children , ..} = self.state.as_mut().unwrap() && operation.channel == *nursery { info!("Operation is for nursery."); - todo!(); + todo!("Handle message for nursery."); continue; } info!("Operation is for channel ch#{}, which is an external channel: {operation:?}", operation.channel); - todo!(); // Migrate channels if necessary. + todo!("Migrate channels if necessary."); external_operations.push(Operation { channel: self.internal_to_external_channels[&operation.channel], ..operation}); info!("In particular, it corresponds to channel ch#{} in the outer node.", self.internal_to_external_channels[&operation.channel]); continue; }, - Some(it) => it, }; let was_completed = match &operation.kind { From e2f0a30b446f427aeecc40a380af159b2181e638 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 5 Sep 2022 13:37:46 +0200 Subject: [PATCH 22/59] Restructure code --- compiler/src/fuzzer/fuzzer.rs | 2 +- compiler/src/fuzzer/mod.rs | 2 +- .../hints/constant_evaluator.rs | 2 +- compiler/src/main.rs | 15 +- compiler/src/vm/channel.rs | 6 +- compiler/src/vm/mod.rs | 1 + compiler/src/vm/tree.rs | 462 ++++++++---------- compiler/src/vm/utils.rs | 20 + 8 files changed, 232 insertions(+), 278 deletions(-) create mode 100644 compiler/src/vm/utils.rs diff --git a/compiler/src/fuzzer/fuzzer.rs b/compiler/src/fuzzer/fuzzer.rs index 3288e765b..110adff80 100644 --- a/compiler/src/fuzzer/fuzzer.rs +++ b/compiler/src/fuzzer/fuzzer.rs @@ -77,7 +77,7 @@ impl Fuzzer { match status { Status::StillFuzzing { mut vm, arguments } => match vm.status() { tree::Status::Running => { - vm.run(context); + vm.run(context, todo!()); Status::StillFuzzing { vm, arguments } } tree::Status::WaitingForOperations => panic!("Fuzzing should not have to wait on channel operations because arguments were not channels."), diff --git a/compiler/src/fuzzer/mod.rs b/compiler/src/fuzzer/mod.rs index aed9027d0..d4ac2db00 100644 --- a/compiler/src/fuzzer/mod.rs +++ b/compiler/src/fuzzer/mod.rs @@ -22,7 +22,7 @@ pub async fn fuzz(db: &Database, module: Module) { vm.run(&mut ModularContext { use_provider: DbUseProvider { db }, execution_controller: RunForever, - }); + }, todo!()); let result = vm.tear_down(); (result.heap, result.fuzzable_closures) }; diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index ad4ec76fa..812d76484 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -54,7 +54,7 @@ impl ConstantEvaluator { vm.run(&mut ModularContext { use_provider: DbUseProvider { db }, execution_controller: RunLimitedNumberOfInstructions::new(500), - }); + }, todo!()); Some(module.clone()) } else { None diff --git a/compiler/src/main.rs b/compiler/src/main.rs index d3d41948d..cf83bc872 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -28,7 +28,7 @@ use crate::{ module::{Module, ModuleKind}, vm::{ context::{DbUseProvider, ModularContext, RunForever}, - Closure, FiberTree, TearDownResult, tree::Status, + Closure, FiberTree, TearDownResult, tree::Status, utils::IdGenerator, }, }; use compiler::lir::Lir; @@ -204,16 +204,17 @@ fn run(options: CandyRunOptions) { let path_string = options.file.to_string_lossy(); info!("Running `{path_string}`."); - let mut vm = FiberTree::new_for_running_module_closure(module_closure); + let mut tree = FiberTree::new_for_running_module_closure(module_closure); + let mut id_generator = IdGenerator::start_at(0); loop { - info!("Tree: {:#?}", vm); - match vm.status() { + // info!("Tree: {:#?}", tree); + match tree.status() { Status::Running => { debug!("VM still running."); - let operations = vm.run(&mut ModularContext { + let operations = tree.run(&mut ModularContext { use_provider: DbUseProvider { db: &db }, execution_controller: RunForever, - }); + }, &mut id_generator); debug!("Operations: {operations:?}"); }, Status::WaitingForOperations => { @@ -227,7 +228,7 @@ fn run(options: CandyRunOptions) { result, heap, .. - } = vm.tear_down(); + } = tree.tear_down(); match result { Ok(return_value) => info!( diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index f3027423c..fddba8641 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -10,7 +10,7 @@ use std::{collections::VecDeque, fmt}; /// simultaneously – you can set it to something large, but having no capacity /// enables buggy code that leaks memory. #[derive(Clone)] -pub struct Channel { +pub struct ChannelBuf { pub capacity: Capacity, packets: VecDeque, } @@ -24,7 +24,7 @@ pub struct Packet { pub value: Pointer, } -impl Channel { +impl ChannelBuf { pub fn new(capacity: Capacity) -> Self { Self { capacity, @@ -52,7 +52,7 @@ impl Channel { } } -impl fmt::Debug for Channel { +impl fmt::Debug for ChannelBuf { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if self.is_empty() { write!(f, "") diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index d05eb9220..8681986c3 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -6,6 +6,7 @@ mod heap; pub mod tracer; pub mod tree; mod use_module; +pub mod utils; pub use fiber::{Fiber, TearDownResult}; pub use heap::{Closure, Heap, Object, Pointer}; diff --git a/compiler/src/vm/tree.rs b/compiler/src/vm/tree.rs index 227f9007a..2c35739b7 100644 --- a/compiler/src/vm/tree.rs +++ b/compiler/src/vm/tree.rs @@ -1,10 +1,10 @@ use super::{ - channel::{Channel, Packet}, + channel::{ChannelBuf, Packet}, context::Context, fiber::Fiber, heap::{ChannelId, Data, ReceivePort, SendPort}, tracer::Tracer, - Closure, Heap, Pointer, TearDownResult, + Closure, Heap, Pointer, TearDownResult, utils::IdGenerator, }; use crate::vm::fiber; use itertools::Itertools; @@ -140,34 +140,17 @@ use tracing::{debug, info, warn}; /// TODO: Implement #[derive(Clone)] pub struct FiberTree { - // status: Status, state: Option, // Only `None` temporarily during state transitions. - - // Fiber trees communicate with the outer world using channels. Each channel - // is identified using an ID that is valid inside this particular tree node. - // Channels that are only used in this subtree are stored here directly. - pub internal_channels: HashMap)>, - internal_channel_id_generator: IdGenerator, - pub external_to_internal_channels: HashMap, - pub internal_to_external_channels: HashMap, - - // Channel operations may be long-running and complete in any order. That's - // why we expose operations to the parent node in this map. The parent can - // call our `complete_` methods with an operation ID to indicate that the - // operation is finished. - // The parent can remove entries from this map at will. We should be able - // to calling the `complete_*` methods. This makes it efficient to - // propagate operations up the tree. For example, a send operation - // containing a large value can just propagate upwards. - // pending_operations: HashMap, + internal_channels: HashMap, + operation_id_generator: IdGenerator, + ongoing_migrations: HashMap, + operations_dependent_on_migration: HashMap>, } -#[derive(Clone, Debug)] -pub enum Status { - Running, - WaitingForOperations, - Done, - Panicked { reason: String }, +#[derive(Clone)] +struct InternalChannel { + buffer: ChannelBuf, + pending_operations: VecDeque, } #[derive(Clone)] @@ -178,61 +161,68 @@ enum State { /// time, the only valid operation ID is 0. SingleFiber(Fiber), - /// The fiber of this tree entered a `core.parallel` scope so that it's now - /// paused and waits for the parallel scope to end. Instead of the main - /// former single fiber, the tree now runs the closure passed to + /// The original single fiber of this tree entered a `core.parallel` scope + /// so that it's now paused and waits for the parallel scope to end. Instead + /// of the original fiber, the tree now runs the closure passed to /// `core.parallel` as well as any other spawned children. ParallelSection { - paused_main_fiber: Fiber, // Should have Status::InParallelSection. + paused_main_fiber: Fiber, // Should have `Status::InParallelSection`. nursery: ChannelId, child_id_generator: IdGenerator, + children: HashMap, - /// Children and a channels where to send the result of the child. The - /// channel's receive port is directly returned by the `core.async` function. - /// Here, we save the ID of the channel where the result of the VM will - /// be sent. - children: HashMap, - - operation_id_generator: IdGenerator, + // We forward operations that our children want to perform and we can't + // handle ourselves to our parent. This map allows us to dispatch + // completions of those operations to the correct child. operation_id_to_child_and_its_operation_id: HashMap, }, } -type ChildId = usize; +#[derive(Clone)] +struct Child { + tree: FiberTree, + + /// When the `tree` finishes running without panicking, its return value + /// will be sent to this channel. The channel's send port is not exposed in + /// Candy code, but its receive port is returned by the `core.async` + /// function. + channel_for_completion: ChannelId, +} + +pub type ChildId = usize; type OperationId = usize; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Operation { id: OperationId, - channel: ChannelId, kind: OperationKind, } #[derive(Clone, Debug)] pub enum OperationKind { - Send { packet: Packet }, - Receive, + Send { channel: ChannelId, packet: Packet }, + Receive { channel: ChannelId }, + MigrateOut, + // Drop { channel: Channel }, +} + +#[derive(Clone, Debug)] +pub enum Status { + Running, + WaitingForOperations, + Done, + Panicked { reason: String }, } impl FiberTree { fn new_with_fiber(mut fiber: Fiber) -> Self { let mut tree = Self { - // status: match fiber.status { - // fiber::Status::Done => Status::Done, - // fiber::Status::Running => Status::Running, - // _ => panic!("Tried to create fiber tree with invalid fiber."), - // }, state: None, internal_channels: Default::default(), - internal_channel_id_generator: IdGenerator::new(), - external_to_internal_channels: Default::default(), - internal_to_external_channels: Default::default(), - // pending_operations: Default::default(), + operation_id_generator: IdGenerator::start_at(0), + ongoing_migrations: Default::default(), + operations_dependent_on_migration: Default::default(), }; - tree.create_channel_mappings_for(&fiber.heap); - fiber - .heap - .map_channel_ids(&tree.external_to_internal_channels); tree.state = Some(State::SingleFiber(fiber)); tree } @@ -252,16 +242,6 @@ impl FiberTree { } } - pub fn status(&self) -> Status { - self.state.as_ref().unwrap().status() - } - fn is_running(&self) -> bool { - matches!(self.status(), Status::Running) - } - fn is_finished(&self) -> bool { - matches!(self.status(), Status::Done | Status::Panicked { .. }) - } - pub fn fiber(&self) -> &Fiber { // TODO: Remove before merging the PR match self.state() { State::SingleFiber(fiber) => fiber, @@ -273,6 +253,7 @@ impl FiberTree { pub fn cloned_tracer(&self) -> Tracer { self.fiber().tracer.clone() } + fn into_state(self) -> State { self.state .expect("Tried to get tree state during state transition") @@ -282,45 +263,67 @@ impl FiberTree { .as_ref() .expect("Tried to get tree state during state transition") } + fn state_mut(&mut self) -> &mut State { + self.state + .as_mut() + .expect("Tried to get tree state during state transition") + } - fn create_channel_mappings_for(&mut self, heap: &Heap) { - for object in heap.all_objects().values() { - if let Data::SendPort(SendPort { channel }) - | Data::ReceivePort(ReceivePort { channel }) = object.data - { - if !self.external_to_internal_channels.contains_key(&channel) { - let internal_id = self.internal_channel_id_generator.generate(); - self.external_to_internal_channels - .insert(channel, internal_id); - self.internal_to_external_channels - .insert(internal_id, channel); - } - } + fn is_internal(&self, channel: ChannelId) -> bool { + if self.internal_channels.contains_key(&channel) { + true + } else if let State::ParallelSection { nursery, .. } = self.state() { + *nursery == channel + } else { + false } } - fn complete_canceling_operations(&mut self, a: Operation, b: Operation) { - assert_eq!(a.channel, b.channel); - match (a.kind, b.kind) { - (OperationKind::Send { packet }, OperationKind::Receive) => { - self.complete_send(a.id); - self.complete_receive(b.id, packet); - } - (OperationKind::Receive, OperationKind::Send { packet }) => { - self.complete_send(b.id); - self.complete_receive(a.id, packet); - } - _ => panic!("operations do not cancel each other out"), + pub fn status(&self) -> Status { + self.state().status() + } + fn is_running(&self) -> bool { + self.state().is_running() + } + fn is_finished(&self) -> bool { + self.state().is_finished() + } +} +impl State { + pub fn status(&self) -> Status { + match self { + State::SingleFiber(fiber) => match &fiber.status { + fiber::Status::Running => Status::Running, + fiber::Status::Sending { .. } | + fiber::Status::Receiving { .. } => Status::WaitingForOperations, + fiber::Status::CreatingChannel { .. } | + fiber::Status::InParallelScope { .. } => unreachable!(), + fiber::Status::Done => Status::Done, + fiber::Status::Panicked { reason } => Status::Panicked { reason: reason.clone() }, + }, + State::ParallelSection { children, .. } => { + for child in children.values() { + return match child.tree.status() { + Status::Running => Status::Running, + Status::WaitingForOperations => Status::WaitingForOperations, + Status::Done => continue, + Status::Panicked { reason } => Status::Panicked { reason }, + }; + } + unreachable!("We should have exited the parallel section") + }, } } - fn complete_send(&mut self, id: OperationId) { - self.state.as_mut().unwrap().complete_send(id); + fn is_running(&self) -> bool { + matches!(self.status(), Status::Running) } - fn complete_receive(&mut self, id: OperationId, packet: Packet) { - self.state.as_mut().unwrap().complete_receive(id, packet); + fn is_finished(&self) -> bool { + matches!(self.status(), Status::Done | Status::Panicked { .. }) } +} - pub fn run(&mut self, context: &mut C) -> Vec { +impl FiberTree { + pub fn run(&mut self, context: &mut C, channel_id_generator: &mut IdGenerator) -> Vec { assert!( self.is_running(), "Called FiberTree::run on a tree that is not ready to run." @@ -329,33 +332,72 @@ impl FiberTree { let mut operations = vec![]; // FIXME: Comment in before merging PR. // while state.is_running() && context.should_continue_running() { - state = self.run_and_map_state(state, &mut operations, context); + state = self.run_and_map_state(state, &mut operations, context, channel_id_generator); // } self.state = Some(state); debug!("Finished running tree (status = {:?}).", self.status()); let mut external_operations = vec![]; + + fn push_external_operation(this: &mut FiberTree, channel_id_generator: &mut IdGenerator, external_operations: &mut Vec, operation: Operation) { + let used_channels = match &operation.kind { + OperationKind::Send { channel, packet } => { + let mut out = vec![*channel]; + packet.collect_channel_ids(&mut out); + out + }, + OperationKind::Receive { channel } => vec![*channel], + OperationKind::MigrateOut => vec![], + }; + let internal_channels = used_channels.into_iter().filter(|channel| this.is_internal(*channel)).collect_vec(); + for channel in &internal_channels { + let operation_id = channel_id_generator.generate(); + this.ongoing_migrations.insert(operation_id, *channel); + external_operations.push(Operation { + id: operation_id, + kind: OperationKind::MigrateOut, + }); + } + if !internal_channels.is_empty() { + this.operations_dependent_on_migration.entry(internal_channels[0]).or_default().push(operation); + } + } + for operation in operations { - let (channel, channel_operations) = match self.internal_channels.get_mut(&operation.channel) { - Some(it) => it, + let channel = match operation.kind { + OperationKind::MigrateOut => { + let id = channel_id_generator.generate(); + match self.state_mut() { + State::SingleFiber(_) => unreachable!("Single fibers should never have to migrate out channels."), + State::ParallelSection { children, operation_id_to_child_and_its_operation_id, .. } => { + let (child_id, operation_id) = operation_id_to_child_and_its_operation_id[&id]; + let channel = children.get_mut(&child_id).unwrap().tree.migrate_out(operation_id, id); + self.internal_channels.insert(id, InternalChannel { buffer: channel, pending_operations: Default::default() }); + }, + } + continue; + } + OperationKind::Send { channel, .. } => channel, + OperationKind::Receive { channel } => channel, + }; + + let (channel, pending_operations) = match self.internal_channels.get_mut(&channel) { + Some(InternalChannel { buffer, pending_operations }) => (buffer, pending_operations), None => { - if let State::ParallelSection { nursery, children , ..} = self.state.as_mut().unwrap() && operation.channel == *nursery { + if let State::ParallelSection { nursery, children , ..} = self.state.as_mut().unwrap() && channel == *nursery { info!("Operation is for nursery."); todo!("Handle message for nursery."); continue; } - info!("Operation is for channel ch#{}, which is an external channel: {operation:?}", operation.channel); - todo!("Migrate channels if necessary."); - external_operations.push(Operation { - channel: self.internal_to_external_channels[&operation.channel], - ..operation}); - info!("In particular, it corresponds to channel ch#{} in the outer node.", self.internal_to_external_channels[&operation.channel]); + info!("Operation is for channel ch#{}, which is an external channel: {operation:?}", channel); + push_external_operation(self, channel_id_generator, &mut external_operations, operation); continue; }, }; let was_completed = match &operation.kind { - OperationKind::Send { packet } => { + OperationKind::MigrateOut => unreachable!("handled above"), + OperationKind::Send { packet, .. } => { if channel.send(packet.clone()) { // TODO: Don't clone self.state.as_mut().unwrap().complete_send(operation.id); true @@ -363,7 +405,7 @@ impl FiberTree { false } } - OperationKind::Receive => { + OperationKind::Receive {..}=> { if let Some(packet) = channel.receive() { self.state.as_mut().unwrap().complete_receive(operation.id, packet); true @@ -383,13 +425,13 @@ impl FiberTree { if was_completed { // TODO: Try completing more operations if that succeeded. } else { - channel_operations.push_back(operation); + pending_operations.push_back(operation); } } external_operations } - fn run_and_map_state(&mut self, state: State, operations: &mut Vec, context: &mut C) -> State { + fn run_and_map_state(&mut self, state: State, operations: &mut Vec, context: &mut C, channel_id_generator: &mut IdGenerator) -> State { match state { State::SingleFiber (mut fiber) => { debug!("Running fiber (status = {:?}).", fiber.status); @@ -399,25 +441,26 @@ impl FiberTree { match fiber.status() { fiber::Status::Running => {} fiber::Status::CreatingChannel { capacity } => { - let id = self.internal_channel_id_generator.generate(); + let id = channel_id_generator.generate(); self.internal_channels - .insert(id, (Channel::new(capacity), VecDeque::new())); + .insert(id, InternalChannel { + buffer: ChannelBuf::new(capacity), + pending_operations: VecDeque::new(), + }); fiber.complete_channel_create(id); } fiber::Status::Sending { channel, packet } => { debug!("Sending packet to channel {channel}."); operations.push(Operation { id: 0, - channel, - kind: OperationKind::Send { packet }, + kind: OperationKind::Send { channel, packet }, }); } fiber::Status::Receiving { channel } => { debug!("Receiving packet from channel {channel}."); operations.push(Operation { id: 0, - channel, - kind: OperationKind::Receive, + kind: OperationKind::Receive { channel }, }); } fiber::Status::InParallelScope { @@ -427,21 +470,21 @@ impl FiberTree { debug!("Entering parallel scope."); let mut heap = Heap::default(); let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); - let nursery = self.internal_channel_id_generator.generate(); + let nursery = channel_id_generator.generate(); let nursery_send_port = heap.create_send_port(nursery); let tree = FiberTree::new_for_running_closure(heap, body, &[nursery_send_port]); let mut fiber_id_generator = IdGenerator::start_at(1); let mut children = HashMap::new(); - children.insert(fiber_id_generator.generate(), (return_channel, tree)); + children.insert(fiber_id_generator.generate(), Child { + tree,channel_for_completion: return_channel, }); return State::ParallelSection { paused_main_fiber: fiber, nursery, child_id_generator: fiber_id_generator, children, - operation_id_generator: IdGenerator::start_at(0), operation_id_to_child_and_its_operation_id: Default::default(), }; } @@ -457,22 +500,23 @@ impl FiberTree { nursery, child_id_generator: fiber_id_generator, mut children, - mut operation_id_generator, mut operation_id_to_child_and_its_operation_id, .. } => { - let (child_id, result_channel, vm) = children + + let (child_id, child) = children .iter_mut() - .map(|(id, (channel, vm))| (*id, *channel, vm)) - .filter(|(_, _, vm)| matches!(vm.status(), Status::Running)) + .map(|(id, child)| (*id, child)) + .filter(|(_, child)| matches!(child.tree.status(), Status::Running)) .choose(&mut rand::thread_rng()) .expect("Tried to run Vm, but no child can run."); + let channel_for_completion = child.channel_for_completion; debug!("Running child VM."); - let new_operations = vm.run(context); + let new_operations = child.tree.run(context, channel_id_generator); for operation in new_operations { - let id = operation_id_generator.generate(); + let id = self.operation_id_generator.generate(); operation_id_to_child_and_its_operation_id.insert(id, (child_id, operation.id)); operations.push(Operation { id, @@ -483,15 +527,15 @@ impl FiberTree { // If the child finished executing, the result should be // transmitted to the channel that's returned by the // `core.async` call. - let packet = match vm.status() { + let packet = match child.tree.status() { Status::Done => { debug!("Child done."); - let (_, vm) = children.remove(&child_id).unwrap(); + let child = children.remove(&child_id).unwrap(); let TearDownResult { heap: vm_heap, result, .. - } = vm.tear_down(); + } = child.tree.tear_down(); let return_value = result.unwrap(); let mut heap = Heap::default(); let return_value = @@ -510,14 +554,12 @@ impl FiberTree { }; if let Some(packet) = packet { operations.push(Operation { - id: operation_id_generator.generate(), - channel: result_channel, - kind: OperationKind::Send { packet }, + id: self.operation_id_generator.generate(), + kind: OperationKind::Send { channel: channel_for_completion, packet }, }); } - // Update status and state. - if children.values().all(|(_, tree)| tree.is_finished()) { + if children.values().all(|child| child.tree.is_finished()) { paused_main_fiber.complete_parallel_scope(); return State::SingleFiber(paused_main_fiber); } @@ -527,43 +569,24 @@ impl FiberTree { nursery, child_id_generator: fiber_id_generator, children, - operation_id_generator, operation_id_to_child_and_its_operation_id, } } } } -} -impl State { - pub fn status(&self) -> Status { - match self { - State::SingleFiber(fiber) => match &fiber.status { - fiber::Status::Running => Status::Running, - fiber::Status::Sending { .. } | - fiber::Status::Receiving { .. } => Status::WaitingForOperations, - fiber::Status::CreatingChannel { .. } | - fiber::Status::InParallelScope { .. } => unreachable!(), - fiber::Status::Done => Status::Done, - fiber::Status::Panicked { reason } => Status::Panicked { reason: reason.clone() }, - }, - State::ParallelSection { children, .. } => { - for (_, child) in children.values() { - return match child.status() { - Status::Running => Status::Running, - Status::WaitingForOperations => Status::WaitingForOperations, - Status::Done => continue, - Status::Panicked { reason } => Status::Panicked { reason }, - }; - } - unreachable!("We should have exited the parallel section") - }, - } + fn migrate_out(&mut self, id: OperationId, external_id: ChannelId) -> ChannelBuf { + todo!() } - fn is_running(&self) -> bool { - matches!(self.status(), Status::Running) + fn complete_send(&mut self, id: OperationId) { + self.state_mut().complete_send(id); + } + fn complete_receive(&mut self, id: OperationId, packet: Packet) { + self.state_mut().complete_receive(id, packet); } - +} + +impl State { fn complete_send(&mut self, id: OperationId) { debug!("Completing send {id}."); match self { @@ -577,7 +600,7 @@ impl State { .. } => { let (child_id, operation_id) = operation_id_to_child_and_its_operation_id[&id]; - children.get_mut(&child_id).unwrap().1.complete_send(operation_id); + children.get_mut(&child_id).unwrap().tree.complete_send(operation_id); } } } @@ -594,116 +617,25 @@ impl State { .. } => { let (child_id, operation_id) = operation_id_to_child_and_its_operation_id[&id]; - children.get_mut(&child_id).unwrap().1.complete_receive(operation_id, packet); + children.get_mut(&child_id).unwrap().tree.complete_receive(operation_id, packet); } } } } -impl fmt::Debug for FiberTree { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let formatted_channels = { - let mut out = String::from(""); - if !self.internal_to_external_channels.is_empty() { - out.push_str(&format!(" internal to external: {}\n", self.internal_to_external_channels.iter().map(|(internal, external)| format!("{internal}->{external}")).join(", "))); - } - for (id, (channel, pending_operations)) in &self.internal_channels { - out.push_str(&format!(" ch#{id}: {channel:?}")); - if !pending_operations.is_empty() { - out.push_str(&format!(" pending: ")); - for operation in pending_operations { - out.push_str(&format!("{operation:?}")); - } - } - out.push('\n'); - } - out - }; - - match self.state.as_ref().unwrap() { - State::SingleFiber(fiber) => { - writeln!(f, "SingleFiber {{")?; - write!(f, "{formatted_channels}")?; - writeln!(f, " status: {:?}", fiber.status)?; - write!(f, "}}")?; - }, - State::ParallelSection { nursery, children, .. } => { - writeln!(f, "ParallelSection {{")?; - write!(f, "{formatted_channels}")?; - writeln!(f, " nursery: ch#{nursery}")?; - for (id, (result_channel, child)) in children { - write!(f, " child#{id} completing to ch#{result_channel}: ")?; - let child = format!("{child:?}"); - writeln!(f, "{}\n{}", child.lines().nth(0).unwrap(), child.lines().skip(1).map(|line| format!(" {line}")).join("\n"))?; - } - write!(f, "}}")?; - }, - } - Ok(()) +impl Packet { + fn collect_channel_ids(&self, out: &mut Vec) { + self.value.collect_channel_ids(&self.heap, out) } } -impl fmt::Debug for State { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::SingleFiber(fiber) => write!(f, "SingleFiber ({:?})", fiber.status), - Self::ParallelSection { children, .. } => f.debug_struct("ParallelSection") - .field("children", children).finish(), +impl Pointer { + fn collect_channel_ids(&self, heap: &Heap, out: &mut Vec) { + let object = heap.get(*self); + if let Data::SendPort(SendPort {channel }) | Data::ReceivePort(ReceivePort { channel }) = object.data { + out.push(channel); } - } -} -impl fmt::Debug for Operation { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "op#{} {} ch#{}", self.id, match &self.kind { - OperationKind::Send { packet } => format!("sending {packet:?} to"), - OperationKind::Receive => format!("receiving from"), - }, self.channel) - } -} - - -impl Operation { - fn cancels_out(&self, other: &Self) -> bool { - matches!( - (&self.kind, &other.kind), - (OperationKind::Send { .. }, OperationKind::Receive) - | (OperationKind::Receive, OperationKind::Send { .. }) - ) - } -} - -impl Heap { - fn map_channel_ids(&mut self, mapping: &HashMap) { - for object in self.all_objects_mut().values_mut() { - if let Data::SendPort(SendPort { channel }) - | Data::ReceivePort(ReceivePort { channel }) = &mut object.data - { - *channel = mapping[channel]; - } + for address in object.children() { + address.collect_channel_ids(heap, out); } } } - -#[derive(Clone)] -struct IdGenerator> { - next_id: usize, - _data: PhantomData, -} -impl> IdGenerator { - fn new() -> Self { - Self { - next_id: 0, - _data: Default::default(), - } - } - fn start_at(id: usize) -> Self { - Self { - next_id: id, - _data: Default::default(), - } - } - fn generate(&mut self) -> T { - let id = self.next_id; - self.next_id += 1; - id.into() - } -} diff --git a/compiler/src/vm/utils.rs b/compiler/src/vm/utils.rs new file mode 100644 index 000000000..ba4c1a6d2 --- /dev/null +++ b/compiler/src/vm/utils.rs @@ -0,0 +1,20 @@ +use std::marker::PhantomData; + +#[derive(Clone)] +pub struct IdGenerator> { + next_id: usize, + _data: PhantomData, +} +impl> IdGenerator { + pub fn start_at(id: usize) -> Self { + Self { + next_id: id, + _data: Default::default(), + } + } + pub fn generate(&mut self) -> T { + let id = self.next_id; + self.next_id += 1; + id.into() + } +} From af9ff38bf30c90ef909ce176dd158f98852312a6 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Wed, 7 Sep 2022 00:31:57 +0200 Subject: [PATCH 23/59] Completely refactor VM architecture --- compiler/src/fuzzer/fuzzer.rs | 14 +- compiler/src/fuzzer/mod.rs | 4 +- .../hints/constant_evaluator.rs | 12 +- compiler/src/main.rs | 8 +- compiler/src/vm/channel.rs | 6 +- compiler/src/vm/mod.rs | 484 +++++++++++- compiler/src/vm/tree.rs | 709 ------------------ 7 files changed, 504 insertions(+), 733 deletions(-) delete mode 100644 compiler/src/vm/tree.rs diff --git a/compiler/src/fuzzer/fuzzer.rs b/compiler/src/fuzzer/fuzzer.rs index 3288e765b..e07c251cb 100644 --- a/compiler/src/fuzzer/fuzzer.rs +++ b/compiler/src/fuzzer/fuzzer.rs @@ -2,7 +2,7 @@ use super::{generator::generate_n_values, utils::did_need_in_closure_cause_panic use crate::{ compiler::hir, database::Database, - vm::{context::Context, tracer::Tracer, tree, Closure, FiberTree, Heap, Pointer}, + vm::{context::Context, tracer::Tracer, Closure, Vm, Heap, Pointer, self}, }; use std::mem; @@ -18,7 +18,7 @@ pub enum Status { // stuff, we'll never find the errors if we accidentally first choose an // input that triggers the loop. StillFuzzing { - vm: FiberTree, + vm: Vm, arguments: Vec, }, // TODO: In the future, also add a state for trying to simplify the @@ -42,7 +42,7 @@ impl Status { let closure = closure_heap.clone_single_to_other_heap(&mut vm_heap, closure); let arguments = generate_n_values(&mut vm_heap, num_args); - let vm = FiberTree::new_for_running_closure(vm_heap, closure, &arguments); + let vm = Vm::new_for_running_closure(vm_heap, closure, &arguments); Status::StillFuzzing { vm, arguments } } @@ -76,14 +76,14 @@ impl Fuzzer { fn map_status(&self, status: Status, db: &Database, context: &mut C) -> Status { match status { Status::StillFuzzing { mut vm, arguments } => match vm.status() { - tree::Status::Running => { + vm::Status::Running => { vm.run(context); Status::StillFuzzing { vm, arguments } } - tree::Status::WaitingForOperations => panic!("Fuzzing should not have to wait on channel operations because arguments were not channels."), + vm::Status::WaitingForOperations => panic!("Fuzzing should not have to wait on channel operations because arguments were not channels."), // The VM finished running without panicking. - tree::Status::Done => Status::new_fuzzing_attempt(&self.closure_heap, self.closure), - tree::Status::Panicked { reason } => { + vm::Status::Done => Status::new_fuzzing_attempt(&self.closure_heap, self.closure), + vm::Status::Panicked { reason } => { // If a `needs` directly inside the tested closure was not // satisfied, then the panic is not closure's fault, but our // fault. diff --git a/compiler/src/fuzzer/mod.rs b/compiler/src/fuzzer/mod.rs index aed9027d0..7419ce6d9 100644 --- a/compiler/src/fuzzer/mod.rs +++ b/compiler/src/fuzzer/mod.rs @@ -8,7 +8,7 @@ use crate::{ module::Module, vm::{ context::{DbUseProvider, ModularContext, RunForever, RunLimitedNumberOfInstructions}, - Closure, FiberTree, + Closure, Vm, }, }; use itertools::Itertools; @@ -16,7 +16,7 @@ use tracing::info; pub async fn fuzz(db: &Database, module: Module) { let (fuzzables_heap, fuzzables) = { - let mut vm = FiberTree::new_for_running_module_closure( + let mut vm = Vm::new_for_running_module_closure( Closure::of_module(db, module.clone()).unwrap(), ); vm.run(&mut ModularContext { diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index ad4ec76fa..679dacf1c 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -12,7 +12,7 @@ use crate::{ vm::{ context::{DbUseProvider, ModularContext, RunLimitedNumberOfInstructions}, tracer::TraceEntry, - tree, Closure, FiberTree, Heap, Pointer, + Closure, Vm, Heap, Pointer, self, }, }; use itertools::Itertools; @@ -22,12 +22,12 @@ use tracing::{span, trace, Level}; #[derive(Default)] pub struct ConstantEvaluator { - vms: HashMap, + vms: HashMap, } impl ConstantEvaluator { pub fn update_module(&mut self, db: &Database, module: Module) { - let vm = FiberTree::new_for_running_module_closure( + let vm = Vm::new_for_running_module_closure( Closure::of_module(db, module.clone()).unwrap(), ); self.vms.insert(module, vm); @@ -42,7 +42,7 @@ impl ConstantEvaluator { let mut running_vms = self .vms .iter_mut() - .filter(|(_, vm)| matches!(vm.status(), tree::Status::Running)) + .filter(|(_, vm)| matches!(vm.status(), vm::Status::Running)) .collect_vec(); trace!( "Constant evaluator running. {} running VMs, {} in total.", @@ -81,7 +81,7 @@ impl ConstantEvaluator { let vm = &self.vms[module]; let mut hints = vec![]; - if let tree::Status::Panicked { reason } = vm.status() { + if let vm::Status::Panicked { reason } = vm.status() { if let Some(hint) = panic_hint(db, module.clone(), vm, reason) { hints.push(hint); } @@ -130,7 +130,7 @@ impl ConstantEvaluator { } } -fn panic_hint(db: &Database, module: Module, vm: &FiberTree, reason: String) -> Option { +fn panic_hint(db: &Database, module: Module, vm: &Vm, reason: String) -> Option { // We want to show the hint at the last call site still inside the current // module. If there is no call site in this module, then the panic results // from a compiler error in a previous stage which is already reported. diff --git a/compiler/src/main.rs b/compiler/src/main.rs index d3d41948d..23ec7934b 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -28,7 +28,7 @@ use crate::{ module::{Module, ModuleKind}, vm::{ context::{DbUseProvider, ModularContext, RunForever}, - Closure, FiberTree, TearDownResult, tree::Status, + Closure, Vm, TearDownResult, Status, }, }; use compiler::lir::Lir; @@ -204,17 +204,17 @@ fn run(options: CandyRunOptions) { let path_string = options.file.to_string_lossy(); info!("Running `{path_string}`."); - let mut vm = FiberTree::new_for_running_module_closure(module_closure); + let mut vm = Vm::new_for_running_module_closure(module_closure); loop { info!("Tree: {:#?}", vm); match vm.status() { Status::Running => { debug!("VM still running."); - let operations = vm.run(&mut ModularContext { + vm.run(&mut ModularContext { use_provider: DbUseProvider { db: &db }, execution_controller: RunForever, }); - debug!("Operations: {operations:?}"); + // TODO: handle operations }, Status::WaitingForOperations => { todo!("VM can't proceed until some operations complete."); diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index f3027423c..fddba8641 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -10,7 +10,7 @@ use std::{collections::VecDeque, fmt}; /// simultaneously – you can set it to something large, but having no capacity /// enables buggy code that leaks memory. #[derive(Clone)] -pub struct Channel { +pub struct ChannelBuf { pub capacity: Capacity, packets: VecDeque, } @@ -24,7 +24,7 @@ pub struct Packet { pub value: Pointer, } -impl Channel { +impl ChannelBuf { pub fn new(capacity: Capacity) -> Self { Self { capacity, @@ -52,7 +52,7 @@ impl Channel { } } -impl fmt::Debug for Channel { +impl fmt::Debug for ChannelBuf { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if self.is_empty() { write!(f, "") diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index d05eb9220..6f5952051 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -4,9 +4,489 @@ pub mod context; mod fiber; mod heap; pub mod tracer; -pub mod tree; mod use_module; +use std::{marker::PhantomData, collections::{HashMap, VecDeque}, fmt}; pub use fiber::{Fiber, TearDownResult}; pub use heap::{Closure, Heap, Object, Pointer}; -pub use tree::FiberTree; +use rand::seq::SliceRandom; +use tracing::{info, warn}; +use self::{heap::ChannelId, channel::{ChannelBuf, Packet}, context::Context, tracer::Tracer}; + +/// A fiber tree a Candy program that thinks it's currently running. Everything +/// from a single fiber to a whole program spanning multiple nested parallel +/// scopes is represented as a fiber tree. Because fiber trees are first-class +/// Rust structs, they enable other code to store "freezed" programs and to +/// remain in control about when and for how long code runs. +/// +/// While fibers are "pure" virtual machines that manage a heap and stack, fiber +/// _trees_ encapsulate fibers and manage channels that are used by them. +/// +/// ## A Tree of Fibers +/// +/// As the name suggests, every Candy program can be represented by a tree at +/// any point in time. In particular, you can create new nodes in the tree by +/// entering a `core.parallel` scope. +/// +/// ```candy +/// core.parallel { nursery -> +/// banana = core.async { "Banana" } +/// peach = core.async { "Peach" } +/// } +/// ``` +/// +/// In this example, after `banana` and `peach` have been assigned, both those +/// closures run concurrently. Let's walk through what happens at each point in +/// time. Before entering the `core.parallel` scope, we only have a single fiber +/// managed by a `FiberTree`. +/// +/// FiberTree +/// | +/// Fiber +/// main +/// (running) +/// +/// As the program enters the `core.parallel` section, the fiber tree changes +/// its state. First, it generates a channel ID for a nursery; while a nursery +/// isn't a channel, it behaves just like one (you can send it closures). Then, +/// the fiber tree creates a send port for the nursery. Finally, it spawns a new +/// fiber tree with the body code of the parallel section, giving it a send port +/// of the nursery as an argument. +/// +/// When asked to run code, the fiber tree will not run the original main fiber, +/// but the body of the parallel section instead. +/// +/// FiberTree +/// | +/// +------+------+ +/// | | +/// Fiber FiberTree +/// main | +/// (parallel scope) Fiber +/// body +/// (running) +/// +/// Calls to `core.async` internally just send packets to the nursery containing +/// the closures to spawn. The fiber tree knows that the channel ID is that of +/// the nursery and instead of actually saving the packets spawns new fibers. +/// The packets also contain a send port of a channel that `core.async` creates +/// locally and that is expected to be sent the result of the running closure. +/// +/// After the two calls to `core.async` finished, the tree looks like this: +/// +/// FiberTree +/// | +/// +------+------+----------+----------+ +/// | | | | +/// Fiber FiberTree FiberTree FiberTree +/// main | | | +/// (parallel scope) Fiber Fiber Fiber +/// body banana peach +/// (running) (running) (running) +/// +/// Now, when the tree is asked to run code, it will run a random running fiber. +/// +/// Once a spawned fiber is done, its return value is stored in the +/// corresponding channel and the fiber is deleted. Once all fibers finished +/// running, the fiber tree exits the parallel section. The `core.parallel` and +/// `core.await` calls take care of actually returning the values put into the +/// channels. If any of the children panic, the parallel section itself will +/// immediately panic as well. +/// +/// The internal fiber trees can of course also start their own parallel +/// sections, resulting in a nested tree. +/// +/// ## Channels +/// +/// In Candy code, channels only appear via their ends, the send and receive +/// ports. Those are unlike other values. In particular, they have an identity +/// and are mutable. Operations like `channel.receive` are not pure and may +/// return different values every time. Also, operations on channels are +/// blocking. +/// +/// Unlike fibers, channels don't form a tree – they can go all over the place! +/// Because you can transmit ports over channels, any two parts of a fiber tree +/// could theoretically be connected via channels. +/// +/// In most programs, we expect channels to stay "relatively" local. In +/// particular, most channels don't escape the fiber tree that they are created +/// in. In order to get the most benefit out of actual paralellism, it's +/// beneficial to store channels as local as possible. For example, if two +/// completely different parts of a program use channels locally to model +/// mutable variables or some other data flow, all channel operations should be +/// local only and not need to be propagated to a central location, avoiding +/// contention. This becomes even more important when (if?) we distribute +/// programs across multiple machines; local channels shouldn't require any +/// communication whatsoever. +/// +/// That's why channels are stored in the local-most subtree of the Candy +/// program that has access to corresponding ports. Fibers themselves don't +/// store channels though – the surrounding nodes of the fiber trees take care +/// of managing channels. +/// +/// The identity of channels is modelled using a channel ID, which is unique +/// within a node of the fiber tree. Whenever data with ports is transmitted to +/// child or parent nodes in the tree, the channel IDs of the ports need to be +/// translated. +/// +/// TODO: Example +/// +/// ## Catching panics +/// +/// TODO: Implement +/// + + + + + + +#[derive(Clone)] +pub struct Vm { + fibers: HashMap, + root_fiber: FiberId, + + channels: HashMap, + pub external_operations: HashMap>, + + fiber_id_generator: IdGenerator, + channel_id_generator: IdGenerator, +} + +type FiberId = usize; +type OperationId = usize; + +#[derive(Clone, Debug)] +enum Channel { + Internal { + buffer: ChannelBuf, + pending_operations: VecDeque, + }, + External(ChannelId), + Nursery { + children: Vec, + } +} +#[derive(Clone, Debug)] +struct Child { + fiber: FiberId, + return_value_channel: ChannelId, +} + +#[derive(Clone)] +pub struct Operation { + performing_fiber: FiberId, + kind: OperationKind, +} +#[derive(Clone, Debug)] +pub enum OperationKind { + Send { packet: Packet }, + Receive, +} + +#[derive(Clone)] +enum FiberTree { + /// This tree is currently focused on running a single fiber. + SingleFiber(Fiber), + + /// The fiber of this tree entered a `core.parallel` scope so that it's now + /// paused and waits for the parallel scope to end. Instead of the main + /// former single fiber, the tree now runs the closure passed to + /// `core.parallel` as well as any other spawned children. + ParallelSection { + paused_main_fiber: Fiber, // Should have Status::InParallelSection. + nursery: ChannelId, + }, +} + +#[derive(Clone, Debug)] +pub enum Status { + Running, + WaitingForOperations, + Done, + Panicked { reason: String }, +} + + + + +impl Vm { + fn new_with_fiber(mut fiber: Fiber) -> Self { + let fiber = FiberTree::SingleFiber(fiber); + let mut fiber_id_generator = IdGenerator::start_at(0); + let root_fiber_id = fiber_id_generator.generate(); + Self { + channels: Default::default(), + fibers: [(root_fiber_id, fiber)].into_iter().collect(), + root_fiber: root_fiber_id, + external_operations: Default::default(), + channel_id_generator: IdGenerator::start_at(0), + fiber_id_generator, + } + } + pub fn new_for_running_closure(heap: Heap, closure: Pointer, arguments: &[Pointer]) -> Self { + Self::new_with_fiber(Fiber::new_for_running_closure(heap, closure, arguments)) + } + pub fn new_for_running_module_closure(closure: Closure) -> Self { + Self::new_with_fiber(Fiber::new_for_running_module_closure(closure)) + } + pub fn tear_down(mut self) -> TearDownResult { + let fiber = self.fibers.remove(&self.root_fiber).unwrap(); + let fiber = match fiber { + FiberTree::SingleFiber(fiber) => fiber, + FiberTree::ParallelSection { .. } => unreachable!(), + }; + fiber.tear_down() + } + + pub fn status(&self) -> Status { + self.status_of(self.root_fiber) + } + fn status_of(&self, fiber: FiberId) -> Status { + match &self.fibers[&fiber] { + FiberTree::SingleFiber(fiber) => match &fiber.status { + fiber::Status::Running => Status::Running, + fiber::Status::Sending { .. } | + fiber::Status::Receiving { .. } => Status::WaitingForOperations, + fiber::Status::CreatingChannel { .. } | + fiber::Status::InParallelScope { .. } => unreachable!(), + fiber::Status::Done => Status::Done, + fiber::Status::Panicked { reason } => Status::Panicked { reason: reason.clone() }, + }, + FiberTree::ParallelSection { nursery, .. } => { + let children = match &self.channels[nursery] { + Channel::Nursery { children } => children, + _ => unreachable!(), + }; + for child in children { + return match self.status_of(child.fiber) { + Status::Running => Status::Running, + Status::WaitingForOperations => Status::WaitingForOperations, + Status::Done => continue, + Status::Panicked { reason } => Status::Panicked { reason }, + }; + } + unreachable!("We should have exited the parallel section") + }, + } + } + fn is_running(&self) -> bool { + matches!(self.status(), Status::Running) + } + fn is_finished(&self) -> bool { + matches!(self.status(), Status::Done | Status::Panicked { .. }) + } + + pub fn fiber(&self) -> &Fiber { // TODO: Remove before merging the PR + todo!() + } + pub fn cloned_tracer(&self) -> Tracer { + self.fiber().tracer.clone() + } + + fn complete_send(&mut self, fiber: FiberId) { + let fiber = self.fibers.get_mut(&fiber).unwrap(); + let fiber = match fiber { + FiberTree::SingleFiber(fiber) => fiber, + FiberTree::ParallelSection { .. } => unreachable!(), + }; + fiber.complete_send(); + } + fn complete_receive(&mut self, fiber: FiberId, packet: Packet) { + let fiber = self.fibers.get_mut(&fiber).unwrap(); + let fiber = match fiber { + FiberTree::SingleFiber(fiber) => fiber, + FiberTree::ParallelSection { .. } => unreachable!(), + }; + fiber.complete_receive(packet); + } + + pub fn run(&mut self, context: &mut C) { + assert!( + self.is_running(), + "Called Vm::run on a VM that is not ready to run." + ); + + let mut fiber_id = self.root_fiber; + let fiber = loop { + match self.fibers.get_mut(&fiber_id).unwrap() { + FiberTree::SingleFiber(fiber) => break fiber, + FiberTree::ParallelSection { nursery, .. } => { + let children = match &self.channels[&nursery] { + Channel::Nursery { children } => children, + _ => unreachable!(), + }; + fiber_id = children + .choose(&mut rand::thread_rng()).unwrap().fiber; + }, + } + }; + + fiber.run(context); + + match fiber.status() { + fiber::Status::Running => {}, + fiber::Status::CreatingChannel { capacity } => { + let channel_id = self.channel_id_generator.generate(); + self.channels.insert(channel_id, Channel::Internal { buffer: ChannelBuf::new(capacity), pending_operations: Default::default() }); + fiber.complete_channel_create(channel_id); + }, + fiber::Status::Sending { channel, packet } => + self.send_to_channel(fiber_id, channel, packet), + fiber::Status::Receiving { channel } => + self.receive_from_channel(fiber_id, channel), + fiber::Status::InParallelScope { body, return_channel } => { + let nursery_id = self.channel_id_generator.generate(); + + let child_id = { + let mut heap = Heap::default(); + let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); + let nursery_send_port = heap.create_send_port(nursery_id); + let id = self.fiber_id_generator.generate(); + self.fibers.insert(id, FiberTree::SingleFiber(Fiber::new_for_running_closure(heap, body, &[nursery_send_port]))); + id + }; + + let nursery_id = { + let id = self.fiber_id_generator.generate(); + // TODO: Make it so that the initial fiber doesn't need a return channel. + let children = vec![Child { + fiber: child_id, + return_value_channel: return_channel, + }]; + self.channels.insert(id, Channel::Nursery { children }); + id + }; + + let paused_main_fiber = match self.fibers.remove(&fiber_id).unwrap() { + FiberTree::SingleFiber(fiber) => fiber, + _ => unreachable!(), + }; + self.fibers.insert(fiber_id, FiberTree::ParallelSection { paused_main_fiber, nursery: nursery_id }); + + // self.fibers.entry(fiber_id).and_modify(|fiber_tree| { + // let paused_main_fiber = match original {} + // FiberTree::ParallelSection { paused_main_fiber, nursery: nursery_id } + // }); + }, + fiber::Status::Done => { + info!("A fiber is done."); + }, + fiber::Status::Panicked { reason } => { + warn!("A fiber panicked because {reason}."); + }, + } + } + + fn send_to_channel(&mut self, performing_fiber: FiberId, channel: ChannelId, packet: Packet) { + match self.channels.get_mut(&channel).unwrap() { + Channel::Internal { buffer, pending_operations } => { + // TODO: Make multithreaded-working. + if buffer.is_full() { + pending_operations.push_back(Operation { + performing_fiber, + kind: OperationKind::Send { packet }, + }); + } else { + buffer.send(packet); + self.complete_send(performing_fiber); + } + }, + Channel::External(id) => { + let id = *id; + self.push_external_operation(id, Operation { + performing_fiber, + kind: OperationKind::Send { packet }, + }) + }, + Channel::Nursery { children } => { + todo!("Stuff is being sent to nursery."); + }, + } + } + + fn receive_from_channel(&mut self, performing_fiber: FiberId, channel: ChannelId) { + match self.channels.get_mut(&channel).unwrap() { + Channel::Internal { buffer, pending_operations } => { + if buffer.is_empty() { + pending_operations.push_back(Operation { + performing_fiber, + kind: OperationKind::Receive, + }); + } else { + let packet = buffer.receive().unwrap(); + self.complete_receive(performing_fiber, packet); + } + }, + Channel::External(id) => { + let id = *id; + self.push_external_operation(id, Operation { + performing_fiber, + kind: OperationKind::Receive, + }); + }, + Channel::Nursery { .. } => unreachable!("nurseries are only sent stuff"), + } + } + + fn push_external_operation(&mut self, channel: ChannelId, operation: Operation) { + self.external_operations.entry(channel).or_default().push(operation); + } +} + +impl Operation { + fn cancels_out(&self, other: &Self) -> bool { + matches!( + (&self.kind, &other.kind), + (OperationKind::Send { .. }, OperationKind::Receive) + | (OperationKind::Receive, OperationKind::Send { .. }) + ) + } +} + +impl fmt::Debug for Operation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.kind { + OperationKind::Send { packet } => write!(f, "{} sending {:?}", self.performing_fiber, packet), + OperationKind::Receive => write!(f, "{} receiving", self.performing_fiber), + } + } +} +impl fmt::Debug for FiberTree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SingleFiber(arg0) => f.debug_tuple("SingleFiber").finish(), + Self::ParallelSection { paused_main_fiber, nursery } => f.debug_struct("ParallelSection").field("nursery", nursery).finish(), + } + } +} +impl fmt::Debug for Vm { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Vm").field("fibers", &self.fibers).field("channels", &self.channels).field("external_operations", &self.external_operations).finish() + } +} + +#[derive(Clone)] +struct IdGenerator> { + next_id: usize, + _data: PhantomData, +} +impl> IdGenerator { + fn new() -> Self { + Self { + next_id: 0, + _data: Default::default(), + } + } + fn start_at(id: usize) -> Self { + Self { + next_id: id, + _data: Default::default(), + } + } + fn generate(&mut self) -> T { + let id = self.next_id; + self.next_id += 1; + id.into() + } +} diff --git a/compiler/src/vm/tree.rs b/compiler/src/vm/tree.rs deleted file mode 100644 index 227f9007a..000000000 --- a/compiler/src/vm/tree.rs +++ /dev/null @@ -1,709 +0,0 @@ -use super::{ - channel::{Channel, Packet}, - context::Context, - fiber::Fiber, - heap::{ChannelId, Data, ReceivePort, SendPort}, - tracer::Tracer, - Closure, Heap, Pointer, TearDownResult, -}; -use crate::vm::fiber; -use itertools::Itertools; -use rand::seq::IteratorRandom; -use core::fmt; -use std::{ - collections::{HashMap, VecDeque}, - marker::PhantomData, - mem, -}; -use tracing::{debug, info, warn}; - -/// A fiber tree a Candy program that thinks it's currently running. Everything -/// from a single fiber to a whole program spanning multiple nested parallel -/// scopes is represented as a fiber tree. Because fiber trees are first-class -/// Rust structs, they enable other code to store "freezed" programs and to -/// remain in control about when and for how long code runs. -/// -/// While fibers are "pure" virtual machines that manage a heap and stack, fiber -/// _trees_ encapsulate fibers and manage channels that are used by them. -/// -/// ## A Tree of Fibers -/// -/// As the name suggests, every Candy program can be represented by a tree at -/// any point in time. In particular, you can create new nodes in the tree by -/// entering a `core.parallel` scope. -/// -/// ```candy -/// core.parallel { nursery -> -/// banana = core.async { "Banana" } -/// peach = core.async { "Peach" } -/// } -/// ``` -/// -/// In this example, after `banana` and `peach` have been assigned, both those -/// closures run concurrently. Let's walk through what happens at each point in -/// time. Before entering the `core.parallel` scope, we only have a single fiber -/// managed by a `FiberTree`. -/// -/// FiberTree -/// | -/// Fiber -/// main -/// (running) -/// -/// As the program enters the `core.parallel` section, the fiber tree changes -/// its state. First, it generates a channel ID for a nursery; while a nursery -/// isn't a channel, it behaves just like one (you can send it closures). Then, -/// the fiber tree creates a send port for the nursery. Finally, it spawns a new -/// fiber tree with the body code of the parallel section, giving it a send port -/// of the nursery as an argument. -/// -/// When asked to run code, the fiber tree will not run the original main fiber, -/// but the body of the parallel section instead. -/// -/// FiberTree -/// | -/// +------+------+ -/// | | -/// Fiber FiberTree -/// main | -/// (parallel scope) Fiber -/// body -/// (running) -/// -/// Calls to `core.async` internally just send packets to the nursery containing -/// the closures to spawn. The fiber tree knows that the channel ID is that of -/// the nursery and instead of actually saving the packets spawns new fibers. -/// The packets also contain a send port of a channel that `core.async` creates -/// locally and that is expected to be sent the result of the running closure. -/// -/// After the two calls to `core.async` finished, the tree looks like this: -/// -/// FiberTree -/// | -/// +------+------+----------+----------+ -/// | | | | -/// Fiber FiberTree FiberTree FiberTree -/// main | | | -/// (parallel scope) Fiber Fiber Fiber -/// body banana peach -/// (running) (running) (running) -/// -/// Now, when the tree is asked to run code, it will run a random running fiber. -/// -/// Once a spawned fiber is done, its return value is stored in the -/// corresponding channel and the fiber is deleted. Once all fibers finished -/// running, the fiber tree exits the parallel section. The `core.parallel` and -/// `core.await` calls take care of actually returning the values put into the -/// channels. If any of the children panic, the parallel section itself will -/// immediately panic as well. -/// -/// The internal fiber trees can of course also start their own parallel -/// sections, resulting in a nested tree. -/// -/// ## Channels -/// -/// In Candy code, channels only appear via their ends, the send and receive -/// ports. Those are unlike other values. In particular, they have an identity -/// and are mutable. Operations like `channel.receive` are not pure and may -/// return different values every time. Also, operations on channels are -/// blocking. -/// -/// Unlike fibers, channels don't form a tree – they can go all over the place! -/// Because you can transmit ports over channels, any two parts of a fiber tree -/// could theoretically be connected via channels. -/// -/// In most programs, we expect channels to stay "relatively" local. In -/// particular, most channels don't escape the fiber tree that they are created -/// in. In order to get the most benefit out of actual paralellism, it's -/// beneficial to store channels as local as possible. For example, if two -/// completely different parts of a program use channels locally to model -/// mutable variables or some other data flow, all channel operations should be -/// local only and not need to be propagated to a central location, avoiding -/// contention. This becomes even more important when (if?) we distribute -/// programs across multiple machines; local channels shouldn't require any -/// communication whatsoever. -/// -/// That's why channels are stored in the local-most subtree of the Candy -/// program that has access to corresponding ports. Fibers themselves don't -/// store channels though – the surrounding nodes of the fiber trees take care -/// of managing channels. -/// -/// The identity of channels is modelled using a channel ID, which is unique -/// within a node of the fiber tree. Whenever data with ports is transmitted to -/// child or parent nodes in the tree, the channel IDs of the ports need to be -/// translated. -/// -/// TODO: Example -/// -/// ## Catching panics -/// -/// TODO: Implement -#[derive(Clone)] -pub struct FiberTree { - // status: Status, - state: Option, // Only `None` temporarily during state transitions. - - // Fiber trees communicate with the outer world using channels. Each channel - // is identified using an ID that is valid inside this particular tree node. - // Channels that are only used in this subtree are stored here directly. - pub internal_channels: HashMap)>, - internal_channel_id_generator: IdGenerator, - pub external_to_internal_channels: HashMap, - pub internal_to_external_channels: HashMap, - - // Channel operations may be long-running and complete in any order. That's - // why we expose operations to the parent node in this map. The parent can - // call our `complete_` methods with an operation ID to indicate that the - // operation is finished. - // The parent can remove entries from this map at will. We should be able - // to calling the `complete_*` methods. This makes it efficient to - // propagate operations up the tree. For example, a send operation - // containing a large value can just propagate upwards. - // pending_operations: HashMap, -} - -#[derive(Clone, Debug)] -pub enum Status { - Running, - WaitingForOperations, - Done, - Panicked { reason: String }, -} - -#[derive(Clone)] -enum State { - /// This tree is currently focused on running a single fiber. - /// - /// Since we only have at most one channel operation running at any given - /// time, the only valid operation ID is 0. - SingleFiber(Fiber), - - /// The fiber of this tree entered a `core.parallel` scope so that it's now - /// paused and waits for the parallel scope to end. Instead of the main - /// former single fiber, the tree now runs the closure passed to - /// `core.parallel` as well as any other spawned children. - ParallelSection { - paused_main_fiber: Fiber, // Should have Status::InParallelSection. - - nursery: ChannelId, - child_id_generator: IdGenerator, - - /// Children and a channels where to send the result of the child. The - /// channel's receive port is directly returned by the `core.async` function. - /// Here, we save the ID of the channel where the result of the VM will - /// be sent. - children: HashMap, - - operation_id_generator: IdGenerator, - operation_id_to_child_and_its_operation_id: HashMap, - }, -} - -type ChildId = usize; -type OperationId = usize; - -#[derive(Clone)] -pub struct Operation { - id: OperationId, - channel: ChannelId, - kind: OperationKind, -} -#[derive(Clone, Debug)] -pub enum OperationKind { - Send { packet: Packet }, - Receive, -} - -impl FiberTree { - fn new_with_fiber(mut fiber: Fiber) -> Self { - let mut tree = Self { - // status: match fiber.status { - // fiber::Status::Done => Status::Done, - // fiber::Status::Running => Status::Running, - // _ => panic!("Tried to create fiber tree with invalid fiber."), - // }, - state: None, - internal_channels: Default::default(), - internal_channel_id_generator: IdGenerator::new(), - external_to_internal_channels: Default::default(), - internal_to_external_channels: Default::default(), - // pending_operations: Default::default(), - }; - tree.create_channel_mappings_for(&fiber.heap); - fiber - .heap - .map_channel_ids(&tree.external_to_internal_channels); - tree.state = Some(State::SingleFiber(fiber)); - tree - } - pub fn new_for_running_closure(heap: Heap, closure: Pointer, arguments: &[Pointer]) -> Self { - Self::new_with_fiber(Fiber::new_for_running_closure(heap, closure, arguments)) - } - pub fn new_for_running_module_closure(closure: Closure) -> Self { - Self::new_with_fiber(Fiber::new_for_running_module_closure(closure)) - } - pub fn tear_down(self) -> TearDownResult { - debug!("FiberTree::tear_down called (our status = {:?}).", self.status()); - match self.into_state() { - State::SingleFiber(fiber) => fiber.tear_down(), - State::ParallelSection { .. } => { - panic!("Called `Vm::tear_down` while in parallel scope") - } - } - } - - pub fn status(&self) -> Status { - self.state.as_ref().unwrap().status() - } - fn is_running(&self) -> bool { - matches!(self.status(), Status::Running) - } - fn is_finished(&self) -> bool { - matches!(self.status(), Status::Done | Status::Panicked { .. }) - } - - pub fn fiber(&self) -> &Fiber { // TODO: Remove before merging the PR - match self.state() { - State::SingleFiber(fiber) => fiber, - State::ParallelSection { - paused_main_fiber, .. - } => paused_main_fiber, - } - } - pub fn cloned_tracer(&self) -> Tracer { - self.fiber().tracer.clone() - } - fn into_state(self) -> State { - self.state - .expect("Tried to get tree state during state transition") - } - fn state(&self) -> &State { - self.state - .as_ref() - .expect("Tried to get tree state during state transition") - } - - fn create_channel_mappings_for(&mut self, heap: &Heap) { - for object in heap.all_objects().values() { - if let Data::SendPort(SendPort { channel }) - | Data::ReceivePort(ReceivePort { channel }) = object.data - { - if !self.external_to_internal_channels.contains_key(&channel) { - let internal_id = self.internal_channel_id_generator.generate(); - self.external_to_internal_channels - .insert(channel, internal_id); - self.internal_to_external_channels - .insert(internal_id, channel); - } - } - } - } - - fn complete_canceling_operations(&mut self, a: Operation, b: Operation) { - assert_eq!(a.channel, b.channel); - match (a.kind, b.kind) { - (OperationKind::Send { packet }, OperationKind::Receive) => { - self.complete_send(a.id); - self.complete_receive(b.id, packet); - } - (OperationKind::Receive, OperationKind::Send { packet }) => { - self.complete_send(b.id); - self.complete_receive(a.id, packet); - } - _ => panic!("operations do not cancel each other out"), - } - } - fn complete_send(&mut self, id: OperationId) { - self.state.as_mut().unwrap().complete_send(id); - } - fn complete_receive(&mut self, id: OperationId, packet: Packet) { - self.state.as_mut().unwrap().complete_receive(id, packet); - } - - pub fn run(&mut self, context: &mut C) -> Vec { - assert!( - self.is_running(), - "Called FiberTree::run on a tree that is not ready to run." - ); - let mut state = mem::replace(&mut self.state, None).unwrap(); - let mut operations = vec![]; - // FIXME: Comment in before merging PR. - // while state.is_running() && context.should_continue_running() { - state = self.run_and_map_state(state, &mut operations, context); - // } - self.state = Some(state); - debug!("Finished running tree (status = {:?}).", self.status()); - - let mut external_operations = vec![]; - for operation in operations { - let (channel, channel_operations) = match self.internal_channels.get_mut(&operation.channel) { - Some(it) => it, - None => { - if let State::ParallelSection { nursery, children , ..} = self.state.as_mut().unwrap() && operation.channel == *nursery { - info!("Operation is for nursery."); - todo!("Handle message for nursery."); - continue; - } - info!("Operation is for channel ch#{}, which is an external channel: {operation:?}", operation.channel); - todo!("Migrate channels if necessary."); - external_operations.push(Operation { - channel: self.internal_to_external_channels[&operation.channel], - ..operation}); - info!("In particular, it corresponds to channel ch#{} in the outer node.", self.internal_to_external_channels[&operation.channel]); - continue; - }, - }; - - let was_completed = match &operation.kind { - OperationKind::Send { packet } => { - if channel.send(packet.clone()) { // TODO: Don't clone - self.state.as_mut().unwrap().complete_send(operation.id); - true - } else { - false - } - } - OperationKind::Receive => { - if let Some(packet) = channel.receive() { - self.state.as_mut().unwrap().complete_receive(operation.id, packet); - true - } else { - false - } - } - }; - - // TODO: Try canceling out with first operation. - // if let Some(operation) = operations.front() && new_operation.cancels_out(operation) { - // let operation = operations.pop_front().unwrap(); - // self.complete_canceling_operations(operation, new_operation); - // return; - // } - - if was_completed { - // TODO: Try completing more operations if that succeeded. - } else { - channel_operations.push_back(operation); - } - } - - external_operations - } - fn run_and_map_state(&mut self, state: State, operations: &mut Vec, context: &mut C) -> State { - match state { - State::SingleFiber (mut fiber) => { - debug!("Running fiber (status = {:?}).", fiber.status); - - fiber.run(context); - - match fiber.status() { - fiber::Status::Running => {} - fiber::Status::CreatingChannel { capacity } => { - let id = self.internal_channel_id_generator.generate(); - self.internal_channels - .insert(id, (Channel::new(capacity), VecDeque::new())); - fiber.complete_channel_create(id); - } - fiber::Status::Sending { channel, packet } => { - debug!("Sending packet to channel {channel}."); - operations.push(Operation { - id: 0, - channel, - kind: OperationKind::Send { packet }, - }); - } - fiber::Status::Receiving { channel } => { - debug!("Receiving packet from channel {channel}."); - operations.push(Operation { - id: 0, - channel, - kind: OperationKind::Receive, - }); - } - fiber::Status::InParallelScope { - body, - return_channel, - } => { - debug!("Entering parallel scope."); - let mut heap = Heap::default(); - let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); - let nursery = self.internal_channel_id_generator.generate(); - let nursery_send_port = heap.create_send_port(nursery); - let tree = - FiberTree::new_for_running_closure(heap, body, &[nursery_send_port]); - - let mut fiber_id_generator = IdGenerator::start_at(1); - let mut children = HashMap::new(); - children.insert(fiber_id_generator.generate(), (return_channel, tree)); - - return State::ParallelSection { - paused_main_fiber: fiber, - nursery, - child_id_generator: fiber_id_generator, - children, - operation_id_generator: IdGenerator::start_at(0), - operation_id_to_child_and_its_operation_id: Default::default(), - }; - } - fiber::Status::Done => info!("Fiber done."), - fiber::Status::Panicked { reason } => { - debug!("Fiber panicked because of {reason}.") - } - } - State::SingleFiber(fiber) - } - State::ParallelSection { - mut paused_main_fiber, - nursery, - child_id_generator: fiber_id_generator, - mut children, - mut operation_id_generator, - mut operation_id_to_child_and_its_operation_id, - .. - } => { - let (child_id, result_channel, vm) = children - .iter_mut() - .map(|(id, (channel, vm))| (*id, *channel, vm)) - .filter(|(_, _, vm)| matches!(vm.status(), Status::Running)) - .choose(&mut rand::thread_rng()) - .expect("Tried to run Vm, but no child can run."); - - debug!("Running child VM."); - let new_operations = vm.run(context); - - for operation in new_operations { - let id = operation_id_generator.generate(); - operation_id_to_child_and_its_operation_id.insert(id, (child_id, operation.id)); - operations.push(Operation { - id, - ..operation - }); - } - - // If the child finished executing, the result should be - // transmitted to the channel that's returned by the - // `core.async` call. - let packet = match vm.status() { - Status::Done => { - debug!("Child done."); - let (_, vm) = children.remove(&child_id).unwrap(); - let TearDownResult { - heap: vm_heap, - result, - .. - } = vm.tear_down(); - let return_value = result.unwrap(); - let mut heap = Heap::default(); - let return_value = - vm_heap.clone_single_to_other_heap(&mut heap, return_value); - let value = heap.create_result(Ok(return_value)); - Some(Packet { heap, value }) - } - Status::Panicked { reason } => { - warn!("Child panicked with reason {reason}"); - let mut heap = Heap::default(); - let reason = heap.create_text(reason); - let value = heap.create_result(Err(reason)); - Some(Packet { heap, value }) - } - _ => None, - }; - if let Some(packet) = packet { - operations.push(Operation { - id: operation_id_generator.generate(), - channel: result_channel, - kind: OperationKind::Send { packet }, - }); - } - - // Update status and state. - if children.values().all(|(_, tree)| tree.is_finished()) { - paused_main_fiber.complete_parallel_scope(); - return State::SingleFiber(paused_main_fiber); - } - - State::ParallelSection { - paused_main_fiber, - nursery, - child_id_generator: fiber_id_generator, - children, - operation_id_generator, - operation_id_to_child_and_its_operation_id, - } - } - } - } -} - -impl State { - pub fn status(&self) -> Status { - match self { - State::SingleFiber(fiber) => match &fiber.status { - fiber::Status::Running => Status::Running, - fiber::Status::Sending { .. } | - fiber::Status::Receiving { .. } => Status::WaitingForOperations, - fiber::Status::CreatingChannel { .. } | - fiber::Status::InParallelScope { .. } => unreachable!(), - fiber::Status::Done => Status::Done, - fiber::Status::Panicked { reason } => Status::Panicked { reason: reason.clone() }, - }, - State::ParallelSection { children, .. } => { - for (_, child) in children.values() { - return match child.status() { - Status::Running => Status::Running, - Status::WaitingForOperations => Status::WaitingForOperations, - Status::Done => continue, - Status::Panicked { reason } => Status::Panicked { reason }, - }; - } - unreachable!("We should have exited the parallel section") - }, - } - } - fn is_running(&self) -> bool { - matches!(self.status(), Status::Running) - } - - fn complete_send(&mut self, id: OperationId) { - debug!("Completing send {id}."); - match self { - State::SingleFiber(fiber) => { - assert_eq!(id, 0); - fiber.complete_send(); - } - State::ParallelSection { - children, - operation_id_to_child_and_its_operation_id, - .. - } => { - let (child_id, operation_id) = operation_id_to_child_and_its_operation_id[&id]; - children.get_mut(&child_id).unwrap().1.complete_send(operation_id); - } - } - } - fn complete_receive(&mut self, id: OperationId, packet: Packet) { - debug!("Completing receive {id}."); - match self { - State::SingleFiber (fiber) => { - assert_eq!(id, 0); - fiber.complete_receive(packet); - } - State::ParallelSection { - children, - operation_id_to_child_and_its_operation_id, - .. - } => { - let (child_id, operation_id) = operation_id_to_child_and_its_operation_id[&id]; - children.get_mut(&child_id).unwrap().1.complete_receive(operation_id, packet); - } - } - } -} - -impl fmt::Debug for FiberTree { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let formatted_channels = { - let mut out = String::from(""); - if !self.internal_to_external_channels.is_empty() { - out.push_str(&format!(" internal to external: {}\n", self.internal_to_external_channels.iter().map(|(internal, external)| format!("{internal}->{external}")).join(", "))); - } - for (id, (channel, pending_operations)) in &self.internal_channels { - out.push_str(&format!(" ch#{id}: {channel:?}")); - if !pending_operations.is_empty() { - out.push_str(&format!(" pending: ")); - for operation in pending_operations { - out.push_str(&format!("{operation:?}")); - } - } - out.push('\n'); - } - out - }; - - match self.state.as_ref().unwrap() { - State::SingleFiber(fiber) => { - writeln!(f, "SingleFiber {{")?; - write!(f, "{formatted_channels}")?; - writeln!(f, " status: {:?}", fiber.status)?; - write!(f, "}}")?; - }, - State::ParallelSection { nursery, children, .. } => { - writeln!(f, "ParallelSection {{")?; - write!(f, "{formatted_channels}")?; - writeln!(f, " nursery: ch#{nursery}")?; - for (id, (result_channel, child)) in children { - write!(f, " child#{id} completing to ch#{result_channel}: ")?; - let child = format!("{child:?}"); - writeln!(f, "{}\n{}", child.lines().nth(0).unwrap(), child.lines().skip(1).map(|line| format!(" {line}")).join("\n"))?; - } - write!(f, "}}")?; - }, - } - Ok(()) - } -} -impl fmt::Debug for State { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::SingleFiber(fiber) => write!(f, "SingleFiber ({:?})", fiber.status), - Self::ParallelSection { children, .. } => f.debug_struct("ParallelSection") - .field("children", children).finish(), - } - } -} -impl fmt::Debug for Operation { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "op#{} {} ch#{}", self.id, match &self.kind { - OperationKind::Send { packet } => format!("sending {packet:?} to"), - OperationKind::Receive => format!("receiving from"), - }, self.channel) - } -} - - -impl Operation { - fn cancels_out(&self, other: &Self) -> bool { - matches!( - (&self.kind, &other.kind), - (OperationKind::Send { .. }, OperationKind::Receive) - | (OperationKind::Receive, OperationKind::Send { .. }) - ) - } -} - -impl Heap { - fn map_channel_ids(&mut self, mapping: &HashMap) { - for object in self.all_objects_mut().values_mut() { - if let Data::SendPort(SendPort { channel }) - | Data::ReceivePort(ReceivePort { channel }) = &mut object.data - { - *channel = mapping[channel]; - } - } - } -} - -#[derive(Clone)] -struct IdGenerator> { - next_id: usize, - _data: PhantomData, -} -impl> IdGenerator { - fn new() -> Self { - Self { - next_id: 0, - _data: Default::default(), - } - } - fn start_at(id: usize) -> Self { - Self { - next_id: id, - _data: Default::default(), - } - } - fn generate(&mut self) -> T { - let id = self.next_id; - self.next_id += 1; - id.into() - } -} From 8c9ef936dcc46796c68bc946f3fbb7927b1d0ee3 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Wed, 7 Sep 2022 12:13:16 +0200 Subject: [PATCH 24/59] Implement spawning of fibers --- compiler/src/vm/mod.rs | 54 +++++++++++++++++++++++++++------ packages/Core/Concurrency.candy | 20 ++++++------ 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 6f5952051..77f2e3f80 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -11,7 +11,9 @@ pub use fiber::{Fiber, TearDownResult}; pub use heap::{Closure, Heap, Object, Pointer}; use rand::seq::SliceRandom; use tracing::{info, warn}; -use self::{heap::ChannelId, channel::{ChannelBuf, Packet}, context::Context, tracer::Tracer}; +use crate::vm::heap::Struct; + +use self::{heap::{ChannelId, Symbol, SendPort}, channel::{ChannelBuf, Packet}, context::Context, tracer::Tracer}; /// A fiber tree a Candy program that thinks it's currently running. Everything /// from a single fiber to a whole program spanning multiple nested parallel @@ -170,7 +172,7 @@ enum Channel { #[derive(Clone, Debug)] struct Child { fiber: FiberId, - return_value_channel: ChannelId, + return_channel: ChannelId, } #[derive(Clone)] @@ -259,14 +261,14 @@ impl Vm { _ => unreachable!(), }; for child in children { - return match self.status_of(child.fiber) { - Status::Running => Status::Running, - Status::WaitingForOperations => Status::WaitingForOperations, + match self.status_of(child.fiber) { + Status::Running => return Status::Running, + Status::WaitingForOperations => {}, Status::Done => continue, - Status::Panicked { reason } => Status::Panicked { reason }, + Status::Panicked { reason } => return Status::Panicked { reason }, }; } - unreachable!("We should have exited the parallel section") + Status::WaitingForOperations }, } } @@ -322,6 +324,10 @@ impl Vm { } }; + if !matches!(fiber.status(), fiber::Status::Running) { + return; + } + fiber.run(context); match fiber.status() { @@ -352,7 +358,7 @@ impl Vm { // TODO: Make it so that the initial fiber doesn't need a return channel. let children = vec![Child { fiber: child_id, - return_value_channel: return_channel, + return_channel: return_channel, }]; self.channels.insert(id, Channel::Nursery { children }); id @@ -400,10 +406,38 @@ impl Vm { }) }, Channel::Nursery { children } => { - todo!("Stuff is being sent to nursery."); + info!("Nursery received packet {:?}", packet); + let (heap, closure_to_spawn, return_channel) = match Self::parse_spawn_packet(packet) { + Some(it) => it, + None => { + // The nursery received an invalid message. TODO: Panic. + panic!("A nursery received an invalid message."); + } + }; + let fiber_id = self.fiber_id_generator.generate(); + self.fibers.insert(fiber_id, FiberTree::SingleFiber(Fiber::new_for_running_closure(heap, closure_to_spawn, &[]))); + children.push(Child { fiber: fiber_id, return_channel }); + self.complete_send(performing_fiber); }, } } + fn parse_spawn_packet(packet: Packet) -> Option<(Heap, Pointer, usize)> { + let Packet { mut heap, value } = packet; + let arguments: Struct = heap.get(value).data.clone().try_into().ok()?; + + let closure_symbol = heap.create_symbol("Closure".to_string()); + let closure_address = arguments.get(&heap, closure_symbol)?; + let closure: Closure = heap.get(closure_address).data.clone().try_into().ok()?; + if closure.num_args > 0 { + return None; + } + + let return_channel_symbol = heap.create_symbol("ReturnChannel".to_string()); + let return_channel_address = arguments.get(&heap, return_channel_symbol)?; + let return_channel: SendPort = heap.get(return_channel_address).data.clone().try_into().ok()?; + + Some((heap, closure_address, return_channel.channel)) + } fn receive_from_channel(&mut self, performing_fiber: FiberId, channel: ChannelId) { match self.channels.get_mut(&channel).unwrap() { @@ -455,7 +489,7 @@ impl fmt::Debug for Operation { impl fmt::Debug for FiberTree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::SingleFiber(arg0) => f.debug_tuple("SingleFiber").finish(), + Self::SingleFiber(fiber) => f.debug_tuple("SingleFiber").field(&fiber.status()).finish(), Self::ParallelSection { paused_main_fiber, nursery } => f.debug_struct("ParallelSection").field("nursery", nursery).finish(), } } diff --git a/packages/Core/Concurrency.candy b/packages/Core/Concurrency.candy index 65c8fc0ec..6f89391dc 100644 --- a/packages/Core/Concurrency.candy +++ b/packages/Core/Concurrency.candy @@ -4,20 +4,20 @@ structGet = (use "..Struct").getUnwrap parallel body := needs (function.is1 body) - returnValueChannel = channel.create 1 - returnValueSendPort = structGet returnValueChannel 0 - returnValueReceivePort = structGet returnValueChannel 1 - ✨.parallel body returnValueSendPort - channel.receive returnValueReceivePort + returnChannel = channel.create 1 + returnSendPort = structGet returnChannel 0 + returnReceivePort = structGet returnChannel 1 + ✨.parallel body returnSendPort + channel.receive returnReceivePort async nursery body := needs (channel.isSendPort nursery) needs (function.is0 body) - returnValueChannel = channel.create 1 - returnValueSendPort = structGet returnValueChannel 0 - returnValueReceivePort = structGet returnValueChannel 1 - channel.send nursery [Spawn, body, returnValueSendPort] - returnValueReceivePort + returnChannel = channel.create 1 + returnSendPort = structGet returnChannel 0 + returnReceivePort = structGet returnChannel 1 + channel.send nursery [Closure: body, ReturnChannel: returnSendPort] + returnReceivePort await fiber := needs (channel.isReceivePort fiber) From a762c531e4a51412f65f8622184a93403ff77fcb Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Wed, 7 Sep 2022 17:28:16 +0200 Subject: [PATCH 25/59] Continue implementing channels --- compiler/src/main.rs | 1 + compiler/src/vm/channel.rs | 9 +- compiler/src/vm/mod.rs | 406 +++++++++++++++++-------------------- 3 files changed, 192 insertions(+), 224 deletions(-) diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 23ec7934b..f7046e6d1 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -222,6 +222,7 @@ fn run(options: CandyRunOptions) { _ => break, } } + info!("Tree: {:#?}", vm); let TearDownResult { tracer, result, diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index fddba8641..85164ba17 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -39,16 +39,15 @@ impl ChannelBuf { self.packets.len() == self.capacity } - pub fn send(&mut self, packet: Packet) -> bool { + pub fn send(&mut self, packet: Packet) { if self.is_full() { - return false; + panic!("Tried to send on channel that is full."); } self.packets.push_back(packet); - return true; } - pub fn receive(&mut self) -> Option { - self.packets.pop_front() + pub fn receive(&mut self) -> Packet { + self.packets.pop_front().expect("Tried to receive from channel that is empty.") } } diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 77f2e3f80..aed4c9e45 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -15,134 +15,9 @@ use crate::vm::heap::Struct; use self::{heap::{ChannelId, Symbol, SendPort}, channel::{ChannelBuf, Packet}, context::Context, tracer::Tracer}; -/// A fiber tree a Candy program that thinks it's currently running. Everything -/// from a single fiber to a whole program spanning multiple nested parallel -/// scopes is represented as a fiber tree. Because fiber trees are first-class -/// Rust structs, they enable other code to store "freezed" programs and to -/// remain in control about when and for how long code runs. -/// -/// While fibers are "pure" virtual machines that manage a heap and stack, fiber -/// _trees_ encapsulate fibers and manage channels that are used by them. -/// -/// ## A Tree of Fibers -/// -/// As the name suggests, every Candy program can be represented by a tree at -/// any point in time. In particular, you can create new nodes in the tree by -/// entering a `core.parallel` scope. -/// -/// ```candy -/// core.parallel { nursery -> -/// banana = core.async { "Banana" } -/// peach = core.async { "Peach" } -/// } -/// ``` -/// -/// In this example, after `banana` and `peach` have been assigned, both those -/// closures run concurrently. Let's walk through what happens at each point in -/// time. Before entering the `core.parallel` scope, we only have a single fiber -/// managed by a `FiberTree`. -/// -/// FiberTree -/// | -/// Fiber -/// main -/// (running) -/// -/// As the program enters the `core.parallel` section, the fiber tree changes -/// its state. First, it generates a channel ID for a nursery; while a nursery -/// isn't a channel, it behaves just like one (you can send it closures). Then, -/// the fiber tree creates a send port for the nursery. Finally, it spawns a new -/// fiber tree with the body code of the parallel section, giving it a send port -/// of the nursery as an argument. -/// -/// When asked to run code, the fiber tree will not run the original main fiber, -/// but the body of the parallel section instead. -/// -/// FiberTree -/// | -/// +------+------+ -/// | | -/// Fiber FiberTree -/// main | -/// (parallel scope) Fiber -/// body -/// (running) -/// -/// Calls to `core.async` internally just send packets to the nursery containing -/// the closures to spawn. The fiber tree knows that the channel ID is that of -/// the nursery and instead of actually saving the packets spawns new fibers. -/// The packets also contain a send port of a channel that `core.async` creates -/// locally and that is expected to be sent the result of the running closure. -/// -/// After the two calls to `core.async` finished, the tree looks like this: -/// -/// FiberTree -/// | -/// +------+------+----------+----------+ -/// | | | | -/// Fiber FiberTree FiberTree FiberTree -/// main | | | -/// (parallel scope) Fiber Fiber Fiber -/// body banana peach -/// (running) (running) (running) -/// -/// Now, when the tree is asked to run code, it will run a random running fiber. -/// -/// Once a spawned fiber is done, its return value is stored in the -/// corresponding channel and the fiber is deleted. Once all fibers finished -/// running, the fiber tree exits the parallel section. The `core.parallel` and -/// `core.await` calls take care of actually returning the values put into the -/// channels. If any of the children panic, the parallel section itself will -/// immediately panic as well. -/// -/// The internal fiber trees can of course also start their own parallel -/// sections, resulting in a nested tree. -/// -/// ## Channels -/// -/// In Candy code, channels only appear via their ends, the send and receive -/// ports. Those are unlike other values. In particular, they have an identity -/// and are mutable. Operations like `channel.receive` are not pure and may -/// return different values every time. Also, operations on channels are -/// blocking. -/// -/// Unlike fibers, channels don't form a tree – they can go all over the place! -/// Because you can transmit ports over channels, any two parts of a fiber tree -/// could theoretically be connected via channels. -/// -/// In most programs, we expect channels to stay "relatively" local. In -/// particular, most channels don't escape the fiber tree that they are created -/// in. In order to get the most benefit out of actual paralellism, it's -/// beneficial to store channels as local as possible. For example, if two -/// completely different parts of a program use channels locally to model -/// mutable variables or some other data flow, all channel operations should be -/// local only and not need to be propagated to a central location, avoiding -/// contention. This becomes even more important when (if?) we distribute -/// programs across multiple machines; local channels shouldn't require any -/// communication whatsoever. -/// -/// That's why channels are stored in the local-most subtree of the Candy -/// program that has access to corresponding ports. Fibers themselves don't -/// store channels though – the surrounding nodes of the fiber trees take care -/// of managing channels. -/// -/// The identity of channels is modelled using a channel ID, which is unique -/// within a node of the fiber tree. Whenever data with ports is transmitted to -/// child or parent nodes in the tree, the channel IDs of the ports need to be -/// translated. -/// -/// TODO: Example -/// -/// ## Catching panics -/// -/// TODO: Implement -/// - - - - - - +/// A VM represents a Candy program that thinks it's currently running. Because +/// VMs are first-class Rust structs, they enable other code to store "freezed" +/// programs and to remain in control about when and for how long code runs. #[derive(Clone)] pub struct Vm { fibers: HashMap, @@ -160,14 +35,15 @@ type OperationId = usize; #[derive(Clone, Debug)] enum Channel { - Internal { - buffer: ChannelBuf, - pending_operations: VecDeque, - }, + Internal(InternalChannel), External(ChannelId), - Nursery { - children: Vec, - } + Nursery(Vec), +} +#[derive(Clone, Debug)] +struct InternalChannel { + buffer: ChannelBuf, + pending_sends: VecDeque<(Option, Packet)>, + pending_receives: VecDeque>, } #[derive(Clone, Debug)] struct Child { @@ -177,7 +53,7 @@ struct Child { #[derive(Clone)] pub struct Operation { - performing_fiber: FiberId, + performing_fiber: Option, kind: OperationKind, } #[derive(Clone, Debug)] @@ -189,7 +65,10 @@ pub enum OperationKind { #[derive(Clone)] enum FiberTree { /// This tree is currently focused on running a single fiber. - SingleFiber(Fiber), + SingleFiber { + fiber: Fiber, + parent_nursery: Option, + }, /// The fiber of this tree entered a `core.parallel` scope so that it's now /// paused and waits for the parallel scope to end. Instead of the main @@ -209,12 +88,9 @@ pub enum Status { Panicked { reason: String }, } - - - impl Vm { fn new_with_fiber(mut fiber: Fiber) -> Self { - let fiber = FiberTree::SingleFiber(fiber); + let fiber = FiberTree::SingleFiber { fiber, parent_nursery: None }; let mut fiber_id_generator = IdGenerator::start_at(0); let root_fiber_id = fiber_id_generator.generate(); Self { @@ -233,11 +109,7 @@ impl Vm { Self::new_with_fiber(Fiber::new_for_running_module_closure(closure)) } pub fn tear_down(mut self) -> TearDownResult { - let fiber = self.fibers.remove(&self.root_fiber).unwrap(); - let fiber = match fiber { - FiberTree::SingleFiber(fiber) => fiber, - FiberTree::ParallelSection { .. } => unreachable!(), - }; + let fiber = self.fibers.remove(&self.root_fiber).unwrap().into_single_fiber().unwrap(); fiber.tear_down() } @@ -246,7 +118,7 @@ impl Vm { } fn status_of(&self, fiber: FiberId) -> Status { match &self.fibers[&fiber] { - FiberTree::SingleFiber(fiber) => match &fiber.status { + FiberTree::SingleFiber { fiber, .. } => match &fiber.status { fiber::Status::Running => Status::Running, fiber::Status::Sending { .. } | fiber::Status::Receiving { .. } => Status::WaitingForOperations, @@ -256,10 +128,10 @@ impl Vm { fiber::Status::Panicked { reason } => Status::Panicked { reason: reason.clone() }, }, FiberTree::ParallelSection { nursery, .. } => { - let children = match &self.channels[nursery] { - Channel::Nursery { children } => children, - _ => unreachable!(), - }; + let children = self.channels[nursery].as_nursery().unwrap(); + if children.is_empty() { + return Status::Done; + } for child in children { match self.status_of(child.fiber) { Status::Running => return Status::Running, @@ -286,38 +158,15 @@ impl Vm { self.fiber().tracer.clone() } - fn complete_send(&mut self, fiber: FiberId) { - let fiber = self.fibers.get_mut(&fiber).unwrap(); - let fiber = match fiber { - FiberTree::SingleFiber(fiber) => fiber, - FiberTree::ParallelSection { .. } => unreachable!(), - }; - fiber.complete_send(); - } - fn complete_receive(&mut self, fiber: FiberId, packet: Packet) { - let fiber = self.fibers.get_mut(&fiber).unwrap(); - let fiber = match fiber { - FiberTree::SingleFiber(fiber) => fiber, - FiberTree::ParallelSection { .. } => unreachable!(), - }; - fiber.complete_receive(packet); - } - pub fn run(&mut self, context: &mut C) { - assert!( - self.is_running(), - "Called Vm::run on a VM that is not ready to run." - ); + assert!(self.is_running(), "Called Vm::run on a VM that is not ready to run."); let mut fiber_id = self.root_fiber; let fiber = loop { match self.fibers.get_mut(&fiber_id).unwrap() { - FiberTree::SingleFiber(fiber) => break fiber, + FiberTree::SingleFiber { fiber, .. } => break fiber, FiberTree::ParallelSection { nursery, .. } => { - let children = match &self.channels[&nursery] { - Channel::Nursery { children } => children, - _ => unreachable!(), - }; + let children = self.channels.get_mut(&nursery).unwrap().as_nursery_mut().unwrap(); fiber_id = children .choose(&mut rand::thread_rng()).unwrap().fiber; }, @@ -328,19 +177,25 @@ impl Vm { return; } + // TODO: Limit context. fiber.run(context); - match fiber.status() { - fiber::Status::Running => {}, + let is_finished = match fiber.status() { + fiber::Status::Running => false, fiber::Status::CreatingChannel { capacity } => { let channel_id = self.channel_id_generator.generate(); - self.channels.insert(channel_id, Channel::Internal { buffer: ChannelBuf::new(capacity), pending_operations: Default::default() }); + self.channels.insert(channel_id, Channel::Internal(InternalChannel { buffer: ChannelBuf::new(capacity), pending_sends: Default::default(), pending_receives: Default::default() })); fiber.complete_channel_create(channel_id); + false }, - fiber::Status::Sending { channel, packet } => - self.send_to_channel(fiber_id, channel, packet), - fiber::Status::Receiving { channel } => - self.receive_from_channel(fiber_id, channel), + fiber::Status::Sending { channel, packet } => { + self.send_to_channel(Some(fiber_id), channel, packet); + false + } + fiber::Status::Receiving { channel } => { + self.receive_from_channel(Some(fiber_id), channel); + false + } fiber::Status::InParallelScope { body, return_channel } => { let nursery_id = self.channel_id_generator.generate(); @@ -349,7 +204,7 @@ impl Vm { let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); let nursery_send_port = heap.create_send_port(nursery_id); let id = self.fiber_id_generator.generate(); - self.fibers.insert(id, FiberTree::SingleFiber(Fiber::new_for_running_closure(heap, body, &[nursery_send_port]))); + self.fibers.insert(id, FiberTree::SingleFiber { fiber: Fiber::new_for_running_closure(heap, body, &[nursery_send_port]), parent_nursery: Some(nursery_id) }); id }; @@ -360,43 +215,76 @@ impl Vm { fiber: child_id, return_channel: return_channel, }]; - self.channels.insert(id, Channel::Nursery { children }); + self.channels.insert(id, Channel::Nursery(children)); id }; - let paused_main_fiber = match self.fibers.remove(&fiber_id).unwrap() { - FiberTree::SingleFiber(fiber) => fiber, - _ => unreachable!(), - }; + let paused_main_fiber = self.fibers.remove(&fiber_id).unwrap().into_single_fiber().unwrap(); self.fibers.insert(fiber_id, FiberTree::ParallelSection { paused_main_fiber, nursery: nursery_id }); // self.fibers.entry(fiber_id).and_modify(|fiber_tree| { // let paused_main_fiber = match original {} // FiberTree::ParallelSection { paused_main_fiber, nursery: nursery_id } // }); + + false }, fiber::Status::Done => { info!("A fiber is done."); + true }, fiber::Status::Panicked { reason } => { warn!("A fiber panicked because {reason}."); + true }, + }; + + if is_finished { + let fiber = self.fibers.remove(&fiber_id).unwrap(); + let (fiber, parent_nursery) = match fiber { + FiberTree::SingleFiber { fiber, parent_nursery } => (fiber, parent_nursery), + _ => unreachable!(), + }; + let TearDownResult { mut heap, result, fuzzable_closures, tracer } = fiber.tear_down(); + + if let Some(nursery) = parent_nursery { + let children = self.channels.get_mut(&nursery).unwrap().as_nursery_mut().unwrap(); + // TODO: Turn children into map to make this less awkward. + let index = children.iter_mut().position(|child| child.fiber == fiber_id).unwrap(); + let child = children.remove(index); + + let result = match result { + Ok(return_value) => self.send_to_channel(None, child.return_channel, Packet { heap, value: return_value }), + Err(panic_reason) => { + // TODO: Handle panicking parallel section. + }, + }; + + if children.is_empty() { + let nursery = self.channels.remove(*nursery).unwrap(); + } + } } } - fn send_to_channel(&mut self, performing_fiber: FiberId, channel: ChannelId, packet: Packet) { + fn try_() { + // let result = match result { + // Ok(return_value) => { + // let ok = heap.create_symbol("Ok".to_string()); + // heap.create_list(&[ok, return_value]) + // }, + // Err(panic_reason) => { + // let err = heap.create_symbol("Err".to_string()); + // let reason = heap.create_text(panic_reason); + // heap.create_list(&[err, reason]) + // }, + // }; + } + + fn send_to_channel(&mut self, performing_fiber: Option, channel: ChannelId, packet: Packet) { match self.channels.get_mut(&channel).unwrap() { - Channel::Internal { buffer, pending_operations } => { - // TODO: Make multithreaded-working. - if buffer.is_full() { - pending_operations.push_back(Operation { - performing_fiber, - kind: OperationKind::Send { packet }, - }); - } else { - buffer.send(packet); - self.complete_send(performing_fiber); - } + Channel::Internal(channel) => { + channel.send(&mut self.fibers, performing_fiber, packet); }, Channel::External(id) => { let id = *id; @@ -405,7 +293,7 @@ impl Vm { kind: OperationKind::Send { packet }, }) }, - Channel::Nursery { children } => { + Channel::Nursery(children) => { info!("Nursery received packet {:?}", packet); let (heap, closure_to_spawn, return_channel) = match Self::parse_spawn_packet(packet) { Some(it) => it, @@ -415,9 +303,9 @@ impl Vm { } }; let fiber_id = self.fiber_id_generator.generate(); - self.fibers.insert(fiber_id, FiberTree::SingleFiber(Fiber::new_for_running_closure(heap, closure_to_spawn, &[]))); + self.fibers.insert(fiber_id, FiberTree::SingleFiber { fiber: Fiber::new_for_running_closure(heap, closure_to_spawn, &[]), parent_nursery: Some(channel) }); children.push(Child { fiber: fiber_id, return_channel }); - self.complete_send(performing_fiber); + InternalChannel::complete_send(&mut self.fibers, performing_fiber); }, } } @@ -439,18 +327,10 @@ impl Vm { Some((heap, closure_address, return_channel.channel)) } - fn receive_from_channel(&mut self, performing_fiber: FiberId, channel: ChannelId) { + fn receive_from_channel(&mut self, performing_fiber: Option, channel: ChannelId) { match self.channels.get_mut(&channel).unwrap() { - Channel::Internal { buffer, pending_operations } => { - if buffer.is_empty() { - pending_operations.push_back(Operation { - performing_fiber, - kind: OperationKind::Receive, - }); - } else { - let packet = buffer.receive().unwrap(); - self.complete_receive(performing_fiber, packet); - } + Channel::Internal(channel) => { + channel.receive(&mut self.fibers, performing_fiber); }, Channel::External(id) => { let id = *id; @@ -478,18 +358,106 @@ impl Operation { } } +impl InternalChannel { + fn send(&mut self, fibers: &mut HashMap, performing_fiber: Option, packet: Packet) { + self.pending_sends.push_back((performing_fiber, packet)); + self.work_on_pending_operations(fibers); + } + + fn receive(&mut self, fibers: &mut HashMap, performing_fiber: Option) { + self.pending_receives.push_back(performing_fiber); + self.work_on_pending_operations(fibers); + } + + fn work_on_pending_operations(&mut self, fibers: &mut HashMap) { + if self.buffer.capacity == 0 { + while !self.pending_sends.is_empty() && !self.pending_receives.is_empty() { + let (send_id, packet) = self.pending_sends.pop_front().unwrap(); + let (receive_id) = self.pending_receives.pop_front().unwrap(); + Self::complete_send(fibers, send_id); + Self::complete_receive(fibers, receive_id, packet); + } + } else { + loop { + let mut did_perform_operation = false; + + if !self.buffer.is_full() && let Some((fiber, packet)) = self.pending_sends.pop_front() { + self.buffer.send(packet); + Self::complete_send(fibers, fiber); + did_perform_operation = true; + } + + if !self.buffer.is_empty() && let Some(fiber) = self.pending_receives.pop_front() { + let packet = self.buffer.receive(); + Self::complete_receive(fibers, fiber, packet); + did_perform_operation = true; + } + + if !did_perform_operation { + break; + } + } + } + } + + fn complete_send(fibers: &mut HashMap, fiber: Option) { + if let Some(fiber) = fiber { + let fiber = fibers.get_mut(&fiber).unwrap().as_single_fiber_mut().unwrap(); + fiber.complete_send(); + } + } + fn complete_receive(fibers: &mut HashMap, fiber: Option, packet: Packet) { + if let Some(fiber) = fiber { + let fiber = fibers.get_mut(&fiber).unwrap().as_single_fiber_mut().unwrap(); + fiber.complete_receive(packet); + } + } +} + +impl Channel { + fn as_nursery(&self) -> Option<&Vec> { + match self { + Channel::Nursery(children) => Some(children), + _ => None, + } + } + fn as_nursery_mut(&mut self) -> Option<&mut Vec> { + match self { + Channel::Nursery(children) => Some(children), + _ => None, + } + } +} +impl FiberTree { + fn into_single_fiber(self) -> Option { + match self { + FiberTree::SingleFiber { fiber, .. } => Some(fiber), + _ => None + } + } + fn as_single_fiber_mut(&mut self) -> Option<&mut Fiber> { + match self { + FiberTree::SingleFiber { fiber, .. } => Some(fiber), + _ => None + } + } +} + impl fmt::Debug for Operation { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(fiber) = self.performing_fiber { + write!(f, "{} ", fiber)?; + } match &self.kind { - OperationKind::Send { packet } => write!(f, "{} sending {:?}", self.performing_fiber, packet), - OperationKind::Receive => write!(f, "{} receiving", self.performing_fiber), + OperationKind::Send { packet } => write!(f, "sending {:?}", packet), + OperationKind::Receive => write!(f, "receiving"), } } } impl fmt::Debug for FiberTree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::SingleFiber(fiber) => f.debug_tuple("SingleFiber").field(&fiber.status()).finish(), + Self::SingleFiber { fiber, parent_nursery } => f.debug_struct("SingleFiber").field("status", &fiber.status()).field("parent_nursery", parent_nursery).finish(), Self::ParallelSection { paused_main_fiber, nursery } => f.debug_struct("ParallelSection").field("nursery", nursery).finish(), } } From 0602c32ec511623267a41d0044687082155b7554 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Wed, 7 Sep 2022 17:46:58 +0200 Subject: [PATCH 26/59] Make channels work --- compiler/src/vm/mod.rs | 52 ++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index aed4c9e45..b2d8956da 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -37,7 +37,7 @@ type OperationId = usize; enum Channel { Internal(InternalChannel), External(ChannelId), - Nursery(Vec), + Nursery { parent: FiberId, children: Vec }, } #[derive(Clone, Debug)] struct InternalChannel { @@ -75,7 +75,7 @@ enum FiberTree { /// former single fiber, the tree now runs the closure passed to /// `core.parallel` as well as any other spawned children. ParallelSection { - paused_main_fiber: Fiber, // Should have Status::InParallelSection. + paused_tree: Box, nursery: ChannelId, }, } @@ -128,7 +128,7 @@ impl Vm { fiber::Status::Panicked { reason } => Status::Panicked { reason: reason.clone() }, }, FiberTree::ParallelSection { nursery, .. } => { - let children = self.channels[nursery].as_nursery().unwrap(); + let children = self.channels[nursery].as_nursery_children().unwrap(); if children.is_empty() { return Status::Done; } @@ -166,7 +166,7 @@ impl Vm { match self.fibers.get_mut(&fiber_id).unwrap() { FiberTree::SingleFiber { fiber, .. } => break fiber, FiberTree::ParallelSection { nursery, .. } => { - let children = self.channels.get_mut(&nursery).unwrap().as_nursery_mut().unwrap(); + let children = self.channels.get_mut(&nursery).unwrap().as_nursery_children_mut().unwrap(); fiber_id = children .choose(&mut rand::thread_rng()).unwrap().fiber; }, @@ -215,12 +215,12 @@ impl Vm { fiber: child_id, return_channel: return_channel, }]; - self.channels.insert(id, Channel::Nursery(children)); + self.channels.insert(id, Channel::Nursery { parent: fiber_id, children }); id }; - let paused_main_fiber = self.fibers.remove(&fiber_id).unwrap().into_single_fiber().unwrap(); - self.fibers.insert(fiber_id, FiberTree::ParallelSection { paused_main_fiber, nursery: nursery_id }); + let paused_tree = self.fibers.remove(&fiber_id).unwrap(); + self.fibers.insert(fiber_id, FiberTree::ParallelSection { paused_tree: Box::new(paused_tree), nursery: nursery_id }); // self.fibers.entry(fiber_id).and_modify(|fiber_tree| { // let paused_main_fiber = match original {} @@ -239,7 +239,7 @@ impl Vm { }, }; - if is_finished { + if is_finished && fiber_id != self.root_fiber { let fiber = self.fibers.remove(&fiber_id).unwrap(); let (fiber, parent_nursery) = match fiber { FiberTree::SingleFiber { fiber, parent_nursery } => (fiber, parent_nursery), @@ -248,10 +248,11 @@ impl Vm { let TearDownResult { mut heap, result, fuzzable_closures, tracer } = fiber.tear_down(); if let Some(nursery) = parent_nursery { - let children = self.channels.get_mut(&nursery).unwrap().as_nursery_mut().unwrap(); + let children = self.channels.get_mut(&nursery).unwrap().as_nursery_children_mut().unwrap(); // TODO: Turn children into map to make this less awkward. let index = children.iter_mut().position(|child| child.fiber == fiber_id).unwrap(); let child = children.remove(index); + let is_finished = children.is_empty(); let result = match result { Ok(return_value) => self.send_to_channel(None, child.return_channel, Packet { heap, value: return_value }), @@ -260,8 +261,12 @@ impl Vm { }, }; - if children.is_empty() { - let nursery = self.channels.remove(*nursery).unwrap(); + if is_finished { + let (parent, _) = self.channels.remove(&nursery).unwrap().into_nursery().unwrap(); + + let (mut paused_tree, _) = self.fibers.remove(&parent).unwrap().into_parallel_section().unwrap(); + paused_tree.as_single_fiber_mut().unwrap().complete_parallel_scope(); + self.fibers.insert(parent, paused_tree); } } } @@ -293,7 +298,7 @@ impl Vm { kind: OperationKind::Send { packet }, }) }, - Channel::Nursery(children) => { + Channel::Nursery { children, .. } => { info!("Nursery received packet {:?}", packet); let (heap, closure_to_spawn, return_channel) = match Self::parse_spawn_packet(packet) { Some(it) => it, @@ -415,15 +420,21 @@ impl InternalChannel { } impl Channel { - fn as_nursery(&self) -> Option<&Vec> { + fn into_nursery(self) -> Option<(FiberId, Vec)> { + match self { + Channel::Nursery { parent, children } => Some((parent, children)), + _ => None, + } + } + fn as_nursery_children(&self) -> Option<&Vec> { match self { - Channel::Nursery(children) => Some(children), + Channel::Nursery { children, .. } => Some(children), _ => None, } } - fn as_nursery_mut(&mut self) -> Option<&mut Vec> { + fn as_nursery_children_mut(&mut self) -> Option<&mut Vec> { match self { - Channel::Nursery(children) => Some(children), + Channel::Nursery { children, .. } => Some(children), _ => None, } } @@ -441,6 +452,13 @@ impl FiberTree { _ => None } } + + fn into_parallel_section(self) -> Option<(FiberTree, ChannelId)> { + match self { + FiberTree::ParallelSection { paused_tree, nursery } => Some((*paused_tree, nursery)), + _ => None, + } + } } impl fmt::Debug for Operation { @@ -458,7 +476,7 @@ impl fmt::Debug for FiberTree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::SingleFiber { fiber, parent_nursery } => f.debug_struct("SingleFiber").field("status", &fiber.status()).field("parent_nursery", parent_nursery).finish(), - Self::ParallelSection { paused_main_fiber, nursery } => f.debug_struct("ParallelSection").field("nursery", nursery).finish(), + Self::ParallelSection { paused_tree, nursery } => f.debug_struct("ParallelSection").field("nursery", nursery).finish(), } } } From b09a84e1c54895cf4d024c8f8a4ea225a95fdbbf Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 8 Sep 2022 00:19:13 +0200 Subject: [PATCH 27/59] Improve code quality --- compiler/src/vm/heap/object.rs | 19 ++++++++-- compiler/src/vm/mod.rs | 67 +++++++++++++++------------------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/compiler/src/vm/heap/object.rs b/compiler/src/vm/heap/object.rs index ef5062104..ac56398f9 100644 --- a/compiler/src/vm/heap/object.rs +++ b/compiler/src/vm/heap/object.rs @@ -14,6 +14,7 @@ use std::{ collections::{hash_map::DefaultHasher, HashMap}, hash::{Hash, Hasher}, ops::Deref, + fmt, }; #[derive(Clone)] @@ -162,7 +163,19 @@ pub struct ReceivePort { pub channel: ChannelId, } -pub type ChannelId = usize; +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct ChannelId(usize); + +impl From for ChannelId { + fn from(id: usize) -> Self { + Self(id) + } +} +impl fmt::Debug for ChannelId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:x}", self.0) + } +} impl SendPort { pub fn new(channel: ChannelId) -> Self { @@ -238,8 +251,8 @@ impl Data { ), Data::Closure(_) => "{…}".to_string(), Data::Builtin(builtin) => format!("builtin{:?}", builtin.function), - Data::SendPort(port) => format!("", port.channel), - Data::ReceivePort(port) => format!("", port.channel), + Data::SendPort(port) => format!("sendPort {:?}", port.channel), + Data::ReceivePort(port) => format!("receivePort {:?}", port.channel), } } diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index b2d8956da..a1bcfc763 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -12,8 +12,7 @@ pub use heap::{Closure, Heap, Object, Pointer}; use rand::seq::SliceRandom; use tracing::{info, warn}; use crate::vm::heap::Struct; - -use self::{heap::{ChannelId, Symbol, SendPort}, channel::{ChannelBuf, Packet}, context::Context, tracer::Tracer}; +use self::{heap::{ChannelId, SendPort}, channel::{ChannelBuf, Packet}, context::Context, tracer::Tracer}; /// A VM represents a Candy program that thinks it's currently running. Because /// VMs are first-class Rust structs, they enable other code to store "freezed" @@ -30,8 +29,11 @@ pub struct Vm { channel_id_generator: IdGenerator, } -type FiberId = usize; -type OperationId = usize; +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct FiberId(usize); + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct OperationId(usize); #[derive(Clone, Debug)] enum Channel { @@ -89,7 +91,7 @@ pub enum Status { } impl Vm { - fn new_with_fiber(mut fiber: Fiber) -> Self { + fn new_with_fiber(fiber: Fiber) -> Self { let fiber = FiberTree::SingleFiber { fiber, parent_nursery: None }; let mut fiber_id_generator = IdGenerator::start_at(0); let root_fiber_id = fiber_id_generator.generate(); @@ -208,16 +210,12 @@ impl Vm { id }; - let nursery_id = { - let id = self.fiber_id_generator.generate(); - // TODO: Make it so that the initial fiber doesn't need a return channel. - let children = vec![Child { - fiber: child_id, - return_channel: return_channel, - }]; - self.channels.insert(id, Channel::Nursery { parent: fiber_id, children }); - id - }; + // TODO: Make it so that the initial fiber doesn't need a return channel. + let children = vec![Child { + fiber: child_id, + return_channel: return_channel, + }]; + self.channels.insert(nursery_id, Channel::Nursery { parent: fiber_id, children }); let paused_tree = self.fibers.remove(&fiber_id).unwrap(); self.fibers.insert(fiber_id, FiberTree::ParallelSection { paused_tree: Box::new(paused_tree), nursery: nursery_id }); @@ -245,7 +243,7 @@ impl Vm { FiberTree::SingleFiber { fiber, parent_nursery } => (fiber, parent_nursery), _ => unreachable!(), }; - let TearDownResult { mut heap, result, fuzzable_closures, tracer } = fiber.tear_down(); + let TearDownResult { heap, result, .. } = fiber.tear_down(); if let Some(nursery) = parent_nursery { let children = self.channels.get_mut(&nursery).unwrap().as_nursery_children_mut().unwrap(); @@ -254,7 +252,7 @@ impl Vm { let child = children.remove(index); let is_finished = children.is_empty(); - let result = match result { + match result { Ok(return_value) => self.send_to_channel(None, child.return_channel, Packet { heap, value: return_value }), Err(panic_reason) => { // TODO: Handle panicking parallel section. @@ -314,7 +312,7 @@ impl Vm { }, } } - fn parse_spawn_packet(packet: Packet) -> Option<(Heap, Pointer, usize)> { + fn parse_spawn_packet(packet: Packet) -> Option<(Heap, Pointer, ChannelId)> { let Packet { mut heap, value } = packet; let arguments: Struct = heap.get(value).data.clone().try_into().ok()?; @@ -353,16 +351,6 @@ impl Vm { } } -impl Operation { - fn cancels_out(&self, other: &Self) -> bool { - matches!( - (&self.kind, &other.kind), - (OperationKind::Send { .. }, OperationKind::Receive) - | (OperationKind::Receive, OperationKind::Send { .. }) - ) - } -} - impl InternalChannel { fn send(&mut self, fibers: &mut HashMap, performing_fiber: Option, packet: Packet) { self.pending_sends.push_back((performing_fiber, packet)); @@ -378,7 +366,7 @@ impl InternalChannel { if self.buffer.capacity == 0 { while !self.pending_sends.is_empty() && !self.pending_receives.is_empty() { let (send_id, packet) = self.pending_sends.pop_front().unwrap(); - let (receive_id) = self.pending_receives.pop_front().unwrap(); + let receive_id = self.pending_receives.pop_front().unwrap(); Self::complete_send(fibers, send_id); Self::complete_receive(fibers, receive_id, packet); } @@ -464,7 +452,7 @@ impl FiberTree { impl fmt::Debug for Operation { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if let Some(fiber) = self.performing_fiber { - write!(f, "{} ", fiber)?; + write!(f, "{:?} ", fiber)?; } match &self.kind { OperationKind::Send { packet } => write!(f, "sending {:?}", packet), @@ -476,7 +464,7 @@ impl fmt::Debug for FiberTree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::SingleFiber { fiber, parent_nursery } => f.debug_struct("SingleFiber").field("status", &fiber.status()).field("parent_nursery", parent_nursery).finish(), - Self::ParallelSection { paused_tree, nursery } => f.debug_struct("ParallelSection").field("nursery", nursery).finish(), + Self::ParallelSection { nursery, .. } => f.debug_struct("ParallelSection").field("nursery", nursery).finish(), } } } @@ -485,6 +473,17 @@ impl fmt::Debug for Vm { f.debug_struct("Vm").field("fibers", &self.fibers).field("channels", &self.channels).field("external_operations", &self.external_operations).finish() } } +impl fmt::Debug for FiberId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:x}", self.0) + } +} + +impl From for FiberId { + fn from(id: usize) -> Self { + Self(id) + } +} #[derive(Clone)] struct IdGenerator> { @@ -492,12 +491,6 @@ struct IdGenerator> { _data: PhantomData, } impl> IdGenerator { - fn new() -> Self { - Self { - next_id: 0, - _data: Default::default(), - } - } fn start_at(id: usize) -> Self { Self { next_id: id, From 11cbc291d8b5e25f5b8918d8fbafa142b53ba47f Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 8 Sep 2022 00:43:07 +0200 Subject: [PATCH 28/59] Improve debug output --- compiler/src/vm/channel.rs | 6 +----- compiler/src/vm/heap/object.rs | 2 +- compiler/src/vm/mod.rs | 30 +++++++++++++++++++++++++++--- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index 85164ba17..49b7f0b1f 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -53,11 +53,7 @@ impl ChannelBuf { impl fmt::Debug for ChannelBuf { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.is_empty() { - write!(f, "") - } else { - write!(f, "{}", self.packets.iter().map(|packet| format!("{:?}", packet)).join(", ")) - } + f.debug_list().entries(self.packets.iter()).finish() } } impl fmt::Debug for Packet { diff --git a/compiler/src/vm/heap/object.rs b/compiler/src/vm/heap/object.rs index ac56398f9..7bd041047 100644 --- a/compiler/src/vm/heap/object.rs +++ b/compiler/src/vm/heap/object.rs @@ -173,7 +173,7 @@ impl From for ChannelId { } impl fmt::Debug for ChannelId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:x}", self.0) + write!(f, "channel_{:x}", self.0) } } diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index a1bcfc763..442052ba8 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -9,6 +9,7 @@ mod use_module; use std::{marker::PhantomData, collections::{HashMap, VecDeque}, fmt}; pub use fiber::{Fiber, TearDownResult}; pub use heap::{Closure, Heap, Object, Pointer}; +use itertools::Itertools; use rand::seq::SliceRandom; use tracing::{info, warn}; use crate::vm::heap::Struct; @@ -35,7 +36,7 @@ struct FiberId(usize); #[derive(Clone, Copy, PartialEq, Eq, Hash)] struct OperationId(usize); -#[derive(Clone, Debug)] +#[derive(Clone)] enum Channel { Internal(InternalChannel), External(ChannelId), @@ -47,7 +48,7 @@ struct InternalChannel { pending_sends: VecDeque<(Option, Packet)>, pending_receives: VecDeque>, } -#[derive(Clone, Debug)] +#[derive(Clone)] struct Child { fiber: FiberId, return_channel: ChannelId, @@ -460,6 +461,29 @@ impl fmt::Debug for Operation { } } } +impl fmt::Debug for Channel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Internal(InternalChannel { buffer, pending_sends, pending_receives }) => + f.debug_struct("InternalChannel") + .field("buffer", buffer) + .field("operations", + &pending_sends.iter() + .map(|(fiber, packet)| Operation { performing_fiber: fiber.clone(), kind: OperationKind::Send { packet: packet.clone() } }) + .chain(pending_receives.iter().map(|fiber| Operation { performing_fiber: fiber.clone(), kind: OperationKind::Receive })) + .collect_vec() + ) + .finish(), + Self::External(arg0) => f.debug_tuple("External").field(arg0).finish(), + Self::Nursery { parent, children } => f.debug_struct("Nursery").field("parent", parent).field("children", children).finish(), + } + } +} +impl fmt::Debug for Child { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?} returning to {:?}", self.fiber, self.return_channel) + } +} impl fmt::Debug for FiberTree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -475,7 +499,7 @@ impl fmt::Debug for Vm { } impl fmt::Debug for FiberId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:x}", self.0) + write!(f, "fiber_{:x}", self.0) } } From 3b8d79127a0a1add2209075c323860a1d7e23fcf Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 8 Sep 2022 21:59:04 +0200 Subject: [PATCH 29/59] Restructure code --- compiler/src/vm/mod.rs | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 442052ba8..508884f04 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -33,8 +33,23 @@ pub struct Vm { #[derive(Clone, Copy, PartialEq, Eq, Hash)] struct FiberId(usize); -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -struct OperationId(usize); +#[derive(Clone)] +enum FiberTree { + /// This tree is currently focused on running a single fiber. + SingleFiber { + fiber: Fiber, + parent_nursery: Option, + }, + + /// The fiber of this tree entered a `core.parallel` scope so that it's now + /// paused and waits for the parallel scope to end. Instead of the main + /// former single fiber, the tree now runs the closure passed to + /// `core.parallel` as well as any other spawned children. + ParallelSection { + paused_tree: Box, + nursery: ChannelId, + }, +} #[derive(Clone)] enum Channel { @@ -54,6 +69,9 @@ struct Child { return_channel: ChannelId, } +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct OperationId(usize); + #[derive(Clone)] pub struct Operation { performing_fiber: Option, @@ -65,24 +83,6 @@ pub enum OperationKind { Receive, } -#[derive(Clone)] -enum FiberTree { - /// This tree is currently focused on running a single fiber. - SingleFiber { - fiber: Fiber, - parent_nursery: Option, - }, - - /// The fiber of this tree entered a `core.parallel` scope so that it's now - /// paused and waits for the parallel scope to end. Instead of the main - /// former single fiber, the tree now runs the closure passed to - /// `core.parallel` as well as any other spawned children. - ParallelSection { - paused_tree: Box, - nursery: ChannelId, - }, -} - #[derive(Clone, Debug)] pub enum Status { Running, From 0e019799e152d66d7d80cd60110890e3d62405dd Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 8 Sep 2022 22:35:22 +0200 Subject: [PATCH 30/59] Propagate panics happening in parallel scope --- compiler/src/builtin_functions.rs | 8 ++- compiler/src/fuzzer/fuzzer.rs | 2 +- .../hints/constant_evaluator.rs | 2 +- compiler/src/main.rs | 2 +- compiler/src/vm/fiber.rs | 11 +++- compiler/src/vm/mod.rs | 64 +++++++++++-------- 6 files changed, 55 insertions(+), 34 deletions(-) diff --git a/compiler/src/builtin_functions.rs b/compiler/src/builtin_functions.rs index 223b5d23f..f76e35d4f 100644 --- a/compiler/src/builtin_functions.rs +++ b/compiler/src/builtin_functions.rs @@ -25,7 +25,13 @@ pub enum BuiltinFunction { IntShiftLeft, // (value: int) (amount: int) -> (shifted: int) IntShiftRight, // (value: int) (amount: int) -> (shifted: int) IntSubtract, // (minuend: int) (subtrahend: int) -> (difference: int) - Parallel, // bodyClosureTakingNursery -> resultOfClosure + + /// Implementation note: Why does `✨.parallel` not return the value + /// directly? The current architecture is chosen to make the VM's code more + /// uniform – every nursery child just sends its result to a channel and + /// there's no special casing for the first child, the closure passed to + /// `✨.parallel`. + Parallel, // (body: Closure, result: SendPort) -> Nothing Print, // message -> Nothing StructGet, // struct key -> value StructGetKeys, // struct -> listOfKeys diff --git a/compiler/src/fuzzer/fuzzer.rs b/compiler/src/fuzzer/fuzzer.rs index e07c251cb..1b257009b 100644 --- a/compiler/src/fuzzer/fuzzer.rs +++ b/compiler/src/fuzzer/fuzzer.rs @@ -76,7 +76,7 @@ impl Fuzzer { fn map_status(&self, status: Status, db: &Database, context: &mut C) -> Status { match status { Status::StillFuzzing { mut vm, arguments } => match vm.status() { - vm::Status::Running => { + vm::Status::CanRun => { vm.run(context); Status::StillFuzzing { vm, arguments } } diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index 679dacf1c..dd0dbf80b 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -42,7 +42,7 @@ impl ConstantEvaluator { let mut running_vms = self .vms .iter_mut() - .filter(|(_, vm)| matches!(vm.status(), vm::Status::Running)) + .filter(|(_, vm)| matches!(vm.status(), vm::Status::CanRun)) .collect_vec(); trace!( "Constant evaluator running. {} running VMs, {} in total.", diff --git a/compiler/src/main.rs b/compiler/src/main.rs index f7046e6d1..76bf25cf5 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -208,7 +208,7 @@ fn run(options: CandyRunOptions) { loop { info!("Tree: {:#?}", vm); match vm.status() { - Status::Running => { + Status::CanRun => { debug!("VM still running."); vm.run(&mut ModularContext { use_provider: DbUseProvider { db: &db }, diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index aa034e397..d045757ef 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -166,9 +166,14 @@ impl Fiber { self.data_stack.push(address); self.status = Status::Running; } - pub fn complete_parallel_scope(&mut self) { - self.data_stack.push(self.heap.create_nothing()); - self.status = Status::Running; + pub fn complete_parallel_scope(&mut self, result: Result<(), String>) { + match result { + Ok(()) => { + self.data_stack.push(self.heap.create_nothing()); + self.status = Status::Running; + }, + Err(reason) => self.panic(reason), + } } fn get_from_data_stack(&self, offset: usize) -> Pointer { diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 508884f04..53288ceb4 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -85,7 +85,7 @@ pub enum OperationKind { #[derive(Clone, Debug)] pub enum Status { - Running, + CanRun, WaitingForOperations, Done, Panicked { reason: String }, @@ -122,7 +122,7 @@ impl Vm { fn status_of(&self, fiber: FiberId) -> Status { match &self.fibers[&fiber] { FiberTree::SingleFiber { fiber, .. } => match &fiber.status { - fiber::Status::Running => Status::Running, + fiber::Status::Running => Status::CanRun, fiber::Status::Sending { .. } | fiber::Status::Receiving { .. } => Status::WaitingForOperations, fiber::Status::CreatingChannel { .. } | @@ -131,16 +131,17 @@ impl Vm { fiber::Status::Panicked { reason } => Status::Panicked { reason: reason.clone() }, }, FiberTree::ParallelSection { nursery, .. } => { - let children = self.channels[nursery].as_nursery_children().unwrap(); - if children.is_empty() { + // The section is still running, otherwise it would have been removed. + let (_, children) = self.channels[nursery].as_nursery().unwrap(); + if children.is_empty() { // TODO: Remove return Status::Done; } for child in children { match self.status_of(child.fiber) { - Status::Running => return Status::Running, + Status::CanRun => return Status::CanRun, Status::WaitingForOperations => {}, - Status::Done => continue, - Status::Panicked { reason } => return Status::Panicked { reason }, + Status::Done => continue, // TODO: Make unreachable + Status::Panicked { reason } => return Status::Panicked { reason }, // TODO: Make unreachable }; } Status::WaitingForOperations @@ -148,7 +149,7 @@ impl Vm { } } fn is_running(&self) -> bool { - matches!(self.status(), Status::Running) + matches!(self.status(), Status::CanRun) } fn is_finished(&self) -> bool { matches!(self.status(), Status::Done | Status::Panicked { .. }) @@ -169,7 +170,7 @@ impl Vm { match self.fibers.get_mut(&fiber_id).unwrap() { FiberTree::SingleFiber { fiber, .. } => break fiber, FiberTree::ParallelSection { nursery, .. } => { - let children = self.channels.get_mut(&nursery).unwrap().as_nursery_children_mut().unwrap(); + let (_, children) = self.channels.get_mut(&nursery).unwrap().as_nursery_mut().unwrap(); fiber_id = children .choose(&mut rand::thread_rng()).unwrap().fiber; }, @@ -237,36 +238,45 @@ impl Vm { true }, }; - + if is_finished && fiber_id != self.root_fiber { let fiber = self.fibers.remove(&fiber_id).unwrap(); - let (fiber, parent_nursery) = match fiber { + let (fiber, parent_nursery_id) = match fiber { FiberTree::SingleFiber { fiber, parent_nursery } => (fiber, parent_nursery), _ => unreachable!(), }; let TearDownResult { heap, result, .. } = fiber.tear_down(); - if let Some(nursery) = parent_nursery { - let children = self.channels.get_mut(&nursery).unwrap().as_nursery_children_mut().unwrap(); + if let Some(parent_nursery_id) = parent_nursery_id { + let (parent_fiber_id, children) = self.channels.get_mut(&parent_nursery_id).unwrap().as_nursery_mut().unwrap(); // TODO: Turn children into map to make this less awkward. let index = children.iter_mut().position(|child| child.fiber == fiber_id).unwrap(); let child = children.remove(index); - let is_finished = children.is_empty(); - match result { - Ok(return_value) => self.send_to_channel(None, child.return_channel, Packet { heap, value: return_value }), + let parallel_section_result = match result { + Ok(return_value) => { + let is_finished = children.is_empty(); + self.send_to_channel(None, child.return_channel, Packet { heap, value: return_value }); + + if is_finished { + Some(Ok(())) + } else { + None + } + }, Err(panic_reason) => { - // TODO: Handle panicking parallel section. + // TODO: Cancel other children. + Some(Err(panic_reason)) }, }; - if is_finished { - let (parent, _) = self.channels.remove(&nursery).unwrap().into_nursery().unwrap(); - - let (mut paused_tree, _) = self.fibers.remove(&parent).unwrap().into_parallel_section().unwrap(); - paused_tree.as_single_fiber_mut().unwrap().complete_parallel_scope(); - self.fibers.insert(parent, paused_tree); + if let Some(result) = parallel_section_result { + let (parent_fiber_id, _) = self.channels.remove(&parent_nursery_id).unwrap().into_nursery().unwrap(); + let (mut paused_tree, _) = self.fibers.remove(&parent_fiber_id).unwrap().into_parallel_section().unwrap(); + paused_tree.as_single_fiber_mut().unwrap().complete_parallel_scope(result); + self.fibers.insert(parent_fiber_id, paused_tree); } + } } } @@ -415,15 +425,15 @@ impl Channel { _ => None, } } - fn as_nursery_children(&self) -> Option<&Vec> { + fn as_nursery(&self) -> Option<(&FiberId, &Vec)> { match self { - Channel::Nursery { children, .. } => Some(children), + Channel::Nursery { parent, children } => Some((parent, children)), _ => None, } } - fn as_nursery_children_mut(&mut self) -> Option<&mut Vec> { + fn as_nursery_mut(&mut self) -> Option<(&mut FiberId, &mut Vec)> { match self { - Channel::Nursery { children, .. } => Some(children), + Channel::Nursery { parent, children } => Some((parent, children)), _ => None, } } From d5f95e9fc2295e3e74adb42380a78d5f05f9b88b Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 8 Sep 2022 22:41:38 +0200 Subject: [PATCH 31/59] Remove canceled fibers --- compiler/src/vm/mod.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 53288ceb4..376f89248 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -23,6 +23,7 @@ pub struct Vm { fibers: HashMap, root_fiber: FiberId, + // TODO: Drop channels channels: HashMap, pub external_operations: HashMap>, @@ -265,7 +266,9 @@ impl Vm { } }, Err(panic_reason) => { - // TODO: Cancel other children. + for child in children.clone() { + self.cancel(child.fiber); + } Some(Err(panic_reason)) }, }; @@ -276,10 +279,20 @@ impl Vm { paused_tree.as_single_fiber_mut().unwrap().complete_parallel_scope(result); self.fibers.insert(parent_fiber_id, paused_tree); } - } } } + fn cancel(&mut self, fiber: FiberId) { + match self.fibers.remove(&fiber).unwrap() { + FiberTree::SingleFiber { .. } => {}, + FiberTree::ParallelSection { nursery, .. } => { + let (_, children) = self.channels.remove(&nursery).unwrap().into_nursery().unwrap(); + for child in children { + self.cancel(child.fiber); + } + }, + } + } fn try_() { // let result = match result { From 48d5dff251e7acc7c43333bf5327d341dd8f5ea7 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 8 Sep 2022 23:35:24 +0200 Subject: [PATCH 32/59] Clean up code --- compiler/src/vm/mod.rs | 248 ++++++++++++++++++++--------------------- 1 file changed, 120 insertions(+), 128 deletions(-) diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 376f89248..0280943a4 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -9,7 +9,6 @@ mod use_module; use std::{marker::PhantomData, collections::{HashMap, VecDeque}, fmt}; pub use fiber::{Fiber, TearDownResult}; pub use heap::{Closure, Heap, Object, Pointer}; -use itertools::Itertools; use rand::seq::SliceRandom; use tracing::{info, warn}; use crate::vm::heap::Struct; @@ -37,26 +36,36 @@ struct FiberId(usize); #[derive(Clone)] enum FiberTree { /// This tree is currently focused on running a single fiber. - SingleFiber { - fiber: Fiber, - parent_nursery: Option, - }, + Single(Single), /// The fiber of this tree entered a `core.parallel` scope so that it's now /// paused and waits for the parallel scope to end. Instead of the main /// former single fiber, the tree now runs the closure passed to /// `core.parallel` as well as any other spawned children. - ParallelSection { - paused_tree: Box, - nursery: ChannelId, - }, + Parallel(Parallel), +} +#[derive(Clone)] +struct Single { + fiber: Fiber, + parent: Option, +} +#[derive(Clone)] +struct Parallel { + paused_fiber: Single, + children: Vec, + nursery: ChannelId, +} +#[derive(Clone)] +struct Child { + fiber: FiberId, + return_channel: ChannelId, } #[derive(Clone)] enum Channel { Internal(InternalChannel), - External(ChannelId), - Nursery { parent: FiberId, children: Vec }, + External, + Nursery(FiberId), } #[derive(Clone, Debug)] struct InternalChannel { @@ -64,11 +73,6 @@ struct InternalChannel { pending_sends: VecDeque<(Option, Packet)>, pending_receives: VecDeque>, } -#[derive(Clone)] -struct Child { - fiber: FiberId, - return_channel: ChannelId, -} #[derive(Clone, Copy, PartialEq, Eq, Hash)] struct OperationId(usize); @@ -94,7 +98,7 @@ pub enum Status { impl Vm { fn new_with_fiber(fiber: Fiber) -> Self { - let fiber = FiberTree::SingleFiber { fiber, parent_nursery: None }; + let fiber = FiberTree::Single(Single { fiber, parent: None }); let mut fiber_id_generator = IdGenerator::start_at(0); let root_fiber_id = fiber_id_generator.generate(); Self { @@ -113,8 +117,8 @@ impl Vm { Self::new_with_fiber(Fiber::new_for_running_module_closure(closure)) } pub fn tear_down(mut self) -> TearDownResult { - let fiber = self.fibers.remove(&self.root_fiber).unwrap().into_single_fiber().unwrap(); - fiber.tear_down() + let fiber = self.fibers.remove(&self.root_fiber).unwrap().into_single().unwrap(); + fiber.fiber.tear_down() } pub fn status(&self) -> Status { @@ -122,7 +126,7 @@ impl Vm { } fn status_of(&self, fiber: FiberId) -> Status { match &self.fibers[&fiber] { - FiberTree::SingleFiber { fiber, .. } => match &fiber.status { + FiberTree::Single(Single { fiber, .. }) => match &fiber.status { fiber::Status::Running => Status::CanRun, fiber::Status::Sending { .. } | fiber::Status::Receiving { .. } => Status::WaitingForOperations, @@ -131,20 +135,17 @@ impl Vm { fiber::Status::Done => Status::Done, fiber::Status::Panicked { reason } => Status::Panicked { reason: reason.clone() }, }, - FiberTree::ParallelSection { nursery, .. } => { - // The section is still running, otherwise it would have been removed. - let (_, children) = self.channels[nursery].as_nursery().unwrap(); - if children.is_empty() { // TODO: Remove - return Status::Done; - } + FiberTree::Parallel(Parallel { children, .. }) => { for child in children { match self.status_of(child.fiber) { Status::CanRun => return Status::CanRun, Status::WaitingForOperations => {}, - Status::Done => continue, // TODO: Make unreachable - Status::Panicked { reason } => return Status::Panicked { reason }, // TODO: Make unreachable + Status::Done | Status::Panicked { .. } => unreachable!(), }; } + // The section is still running, otherwise it would have been + // removed. Thus, there was at least one child and all children + // were waiting for operations. Status::WaitingForOperations }, } @@ -169,11 +170,9 @@ impl Vm { let mut fiber_id = self.root_fiber; let fiber = loop { match self.fibers.get_mut(&fiber_id).unwrap() { - FiberTree::SingleFiber { fiber, .. } => break fiber, - FiberTree::ParallelSection { nursery, .. } => { - let (_, children) = self.channels.get_mut(&nursery).unwrap().as_nursery_mut().unwrap(); - fiber_id = children - .choose(&mut rand::thread_rng()).unwrap().fiber; + FiberTree::Single(Single { fiber, .. }) => break fiber, + FiberTree::Parallel(Parallel { children, .. }) => { + fiber_id = children.choose(&mut rand::thread_rng()).unwrap().fiber; }, } }; @@ -203,30 +202,31 @@ impl Vm { } fiber::Status::InParallelScope { body, return_channel } => { let nursery_id = self.channel_id_generator.generate(); + self.channels.insert(nursery_id, Channel::Nursery(fiber_id)); - let child_id = { + let first_child_id = { let mut heap = Heap::default(); let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); let nursery_send_port = heap.create_send_port(nursery_id); let id = self.fiber_id_generator.generate(); - self.fibers.insert(id, FiberTree::SingleFiber { fiber: Fiber::new_for_running_closure(heap, body, &[nursery_send_port]), parent_nursery: Some(nursery_id) }); + self.fibers.insert(id, FiberTree::Single(Single{ + fiber: Fiber::new_for_running_closure(heap, body, &[nursery_send_port]), + parent: Some(fiber_id) + })); id }; + // TODO: Create utility method for removing and adding under same ID. // TODO: Make it so that the initial fiber doesn't need a return channel. - let children = vec![Child { - fiber: child_id, - return_channel: return_channel, - }]; - self.channels.insert(nursery_id, Channel::Nursery { parent: fiber_id, children }); - - let paused_tree = self.fibers.remove(&fiber_id).unwrap(); - self.fibers.insert(fiber_id, FiberTree::ParallelSection { paused_tree: Box::new(paused_tree), nursery: nursery_id }); - - // self.fibers.entry(fiber_id).and_modify(|fiber_tree| { - // let paused_main_fiber = match original {} - // FiberTree::ParallelSection { paused_main_fiber, nursery: nursery_id } - // }); + let paused_fiber = self.fibers.remove(&fiber_id).unwrap().into_single().unwrap(); + self.fibers.insert(fiber_id, FiberTree::Parallel(Parallel { + paused_fiber, + children: vec![Child { + fiber: first_child_id, + return_channel: return_channel, + }], + nursery: nursery_id, + })); false }, @@ -241,20 +241,17 @@ impl Vm { }; if is_finished && fiber_id != self.root_fiber { - let fiber = self.fibers.remove(&fiber_id).unwrap(); - let (fiber, parent_nursery_id) = match fiber { - FiberTree::SingleFiber { fiber, parent_nursery } => (fiber, parent_nursery), - _ => unreachable!(), - }; - let TearDownResult { heap, result, .. } = fiber.tear_down(); - - if let Some(parent_nursery_id) = parent_nursery_id { - let (parent_fiber_id, children) = self.channels.get_mut(&parent_nursery_id).unwrap().as_nursery_mut().unwrap(); + let single = self.fibers.remove(&fiber_id).unwrap().into_single().unwrap(); + let TearDownResult { heap, result, .. } = single.fiber.tear_down(); + + if let Some(parent_id) = single.parent { + let Parallel { children, nursery, .. } = self.fibers.get_mut(&parent_id).unwrap().as_parallel_mut().unwrap(); // TODO: Turn children into map to make this less awkward. let index = children.iter_mut().position(|child| child.fiber == fiber_id).unwrap(); let child = children.remove(index); + let nursery = *nursery; - let parallel_section_result = match result { + let parallel_result = match result { Ok(return_value) => { let is_finished = children.is_empty(); self.send_to_channel(None, child.return_channel, Packet { heap, value: return_value }); @@ -273,20 +270,20 @@ impl Vm { }, }; - if let Some(result) = parallel_section_result { - let (parent_fiber_id, _) = self.channels.remove(&parent_nursery_id).unwrap().into_nursery().unwrap(); - let (mut paused_tree, _) = self.fibers.remove(&parent_fiber_id).unwrap().into_parallel_section().unwrap(); - paused_tree.as_single_fiber_mut().unwrap().complete_parallel_scope(result); - self.fibers.insert(parent_fiber_id, paused_tree); + if let Some(result) = parallel_result { + self.channels.remove(&nursery).unwrap().to_nursery().unwrap(); + let Parallel { mut paused_fiber, .. } = self.fibers.remove(&parent_id).unwrap().into_parallel().unwrap(); + paused_fiber.fiber.complete_parallel_scope(result); + self.fibers.insert(parent_id, FiberTree::Single(paused_fiber)); } } } } fn cancel(&mut self, fiber: FiberId) { match self.fibers.remove(&fiber).unwrap() { - FiberTree::SingleFiber { .. } => {}, - FiberTree::ParallelSection { nursery, .. } => { - let (_, children) = self.channels.remove(&nursery).unwrap().into_nursery().unwrap(); + FiberTree::Single(_) => {}, + FiberTree::Parallel(Parallel { children, nursery, .. }) => { + self.channels.remove(&nursery).unwrap().to_nursery().unwrap(); for child in children { self.cancel(child.fiber); } @@ -313,14 +310,13 @@ impl Vm { Channel::Internal(channel) => { channel.send(&mut self.fibers, performing_fiber, packet); }, - Channel::External(id) => { - let id = *id; - self.push_external_operation(id, Operation { + Channel::External => { + self.push_external_operation(channel, Operation { performing_fiber, kind: OperationKind::Send { packet }, }) }, - Channel::Nursery { children, .. } => { + Channel::Nursery(parent_id) => { info!("Nursery received packet {:?}", packet); let (heap, closure_to_spawn, return_channel) = match Self::parse_spawn_packet(packet) { Some(it) => it, @@ -329,9 +325,12 @@ impl Vm { panic!("A nursery received an invalid message."); } }; - let fiber_id = self.fiber_id_generator.generate(); - self.fibers.insert(fiber_id, FiberTree::SingleFiber { fiber: Fiber::new_for_running_closure(heap, closure_to_spawn, &[]), parent_nursery: Some(channel) }); - children.push(Child { fiber: fiber_id, return_channel }); + let child_id = self.fiber_id_generator.generate(); + self.fibers.insert(child_id, FiberTree::Single(Single { fiber: Fiber::new_for_running_closure(heap, closure_to_spawn, &[]), parent: Some(*parent_id) })); + + let parent = self.fibers.get_mut(parent_id).unwrap().as_parallel_mut().unwrap(); + parent.children.push(Child { fiber: child_id, return_channel }); + InternalChannel::complete_send(&mut self.fibers, performing_fiber); }, } @@ -359,9 +358,8 @@ impl Vm { Channel::Internal(channel) => { channel.receive(&mut self.fibers, performing_fiber); }, - Channel::External(id) => { - let id = *id; - self.push_external_operation(id, Operation { + Channel::External => { + self.push_external_operation(channel, Operation { performing_fiber, kind: OperationKind::Receive, }); @@ -419,71 +417,77 @@ impl InternalChannel { fn complete_send(fibers: &mut HashMap, fiber: Option) { if let Some(fiber) = fiber { - let fiber = fibers.get_mut(&fiber).unwrap().as_single_fiber_mut().unwrap(); - fiber.complete_send(); + let fiber = fibers.get_mut(&fiber).unwrap().as_single_mut().unwrap(); + fiber.fiber.complete_send(); } } fn complete_receive(fibers: &mut HashMap, fiber: Option, packet: Packet) { if let Some(fiber) = fiber { - let fiber = fibers.get_mut(&fiber).unwrap().as_single_fiber_mut().unwrap(); - fiber.complete_receive(packet); + let fiber = fibers.get_mut(&fiber).unwrap().as_single_mut().unwrap(); + fiber.fiber.complete_receive(packet); } } } impl Channel { - fn into_nursery(self) -> Option<(FiberId, Vec)> { + fn to_nursery(&self) -> Option { match self { - Channel::Nursery { parent, children } => Some((parent, children)), - _ => None, - } - } - fn as_nursery(&self) -> Option<(&FiberId, &Vec)> { - match self { - Channel::Nursery { parent, children } => Some((parent, children)), - _ => None, - } - } - fn as_nursery_mut(&mut self) -> Option<(&mut FiberId, &mut Vec)> { - match self { - Channel::Nursery { parent, children } => Some((parent, children)), + Channel::Nursery(fiber) => Some(*fiber), _ => None, } } } impl FiberTree { - fn into_single_fiber(self) -> Option { + fn into_single(self) -> Option { match self { - FiberTree::SingleFiber { fiber, .. } => Some(fiber), + FiberTree::Single(single) => Some(single), _ => None } } - fn as_single_fiber_mut(&mut self) -> Option<&mut Fiber> { + fn as_single_mut(&mut self) -> Option<&mut Single> { match self { - FiberTree::SingleFiber { fiber, .. } => Some(fiber), + FiberTree::Single(single) => Some(single), _ => None } } - fn into_parallel_section(self) -> Option<(FiberTree, ChannelId)> { + fn into_parallel(self) -> Option { + match self { + FiberTree::Parallel(parallel) => Some(parallel), + _ => None, + } + } + fn as_parallel_mut(&mut self) -> Option<&mut Parallel> { match self { - FiberTree::ParallelSection { paused_tree, nursery } => Some((*paused_tree, nursery)), + FiberTree::Parallel(parallel) => Some(parallel), _ => None, } } } -impl fmt::Debug for Operation { +impl fmt::Debug for Vm { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if let Some(fiber) = self.performing_fiber { - write!(f, "{:?} ", fiber)?; - } - match &self.kind { - OperationKind::Send { packet } => write!(f, "sending {:?}", packet), - OperationKind::Receive => write!(f, "receiving"), + f.debug_struct("Vm").field("fibers", &self.fibers).field("channels", &self.channels).field("external_operations", &self.external_operations).finish() + } +} +impl fmt::Debug for FiberId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "fiber_{:x}", self.0) + } +} +impl fmt::Debug for FiberTree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Single(Single { fiber, parent }) => f.debug_struct("SingleFiber").field("status", &fiber.status()).field("parent", parent).finish(), + Self::Parallel(Parallel { children, nursery, .. }) => f.debug_struct("ParallelSection").field("children", children).field("nursery", nursery).finish(), } } } +impl fmt::Debug for Child { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?} returning to {:?}", self.fiber, self.return_channel) + } +} impl fmt::Debug for Channel { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -497,32 +501,20 @@ impl fmt::Debug for Channel { .collect_vec() ) .finish(), - Self::External(arg0) => f.debug_tuple("External").field(arg0).finish(), - Self::Nursery { parent, children } => f.debug_struct("Nursery").field("parent", parent).field("children", children).finish(), + Self::External => f.debug_tuple("External").finish(), + Self::Nursery(fiber) => f.debug_tuple("Nursery").field(fiber).finish(), } } } -impl fmt::Debug for Child { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?} returning to {:?}", self.fiber, self.return_channel) - } -} -impl fmt::Debug for FiberTree { +impl fmt::Debug for Operation { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::SingleFiber { fiber, parent_nursery } => f.debug_struct("SingleFiber").field("status", &fiber.status()).field("parent_nursery", parent_nursery).finish(), - Self::ParallelSection { nursery, .. } => f.debug_struct("ParallelSection").field("nursery", nursery).finish(), + if let Some(fiber) = self.performing_fiber { + write!(f, "{:?} ", fiber)?; + } + match &self.kind { + OperationKind::Send { packet } => write!(f, "sending {:?}", packet), + OperationKind::Receive => write!(f, "receiving"), } - } -} -impl fmt::Debug for Vm { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Vm").field("fibers", &self.fibers).field("channels", &self.channels).field("external_operations", &self.external_operations).finish() - } -} -impl fmt::Debug for FiberId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "fiber_{:x}", self.0) } } From 14230e8d0266e6e6b0cb3ff5e7df6e3a8bfdb204 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sat, 10 Sep 2022 22:35:59 +0200 Subject: [PATCH 33/59] Implement try --- compiler/src/builtin_functions.rs | 1 + compiler/src/vm/builtin_functions.rs | 16 +++- compiler/src/vm/fiber.rs | 17 ++++ compiler/src/vm/mod.rs | 118 +++++++++++++++++---------- 4 files changed, 106 insertions(+), 46 deletions(-) diff --git a/compiler/src/builtin_functions.rs b/compiler/src/builtin_functions.rs index f76e35d4f..c816b9e87 100644 --- a/compiler/src/builtin_functions.rs +++ b/compiler/src/builtin_functions.rs @@ -46,6 +46,7 @@ pub enum BuiltinFunction { TextStartsWith, // text (pattern: text) -> booleanSymbol TextTrimEnd, // text -> text TextTrimStart, // text -> text + Try, // closure -> okWithClosureResultOrErrorWithPanicReason TypeOf, // any -> typeSymbol } lazy_static! { diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index 5490ccbbf..e2360f31c 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -58,6 +58,7 @@ impl Fiber { BuiltinFunction::TextStartsWith => self.heap.text_starts_with(args), BuiltinFunction::TextTrimEnd => self.heap.text_trim_end(args), BuiltinFunction::TextTrimStart => self.heap.text_trim_start(args), + BuiltinFunction::Try => self.heap.try_(args), BuiltinFunction::TypeOf => self.heap.type_of(args), }); match result { @@ -69,15 +70,15 @@ impl Fiber { Ok(CreateChannel { capacity }) => self.status = Status::CreatingChannel { capacity }, Ok(Send { channel, packet }) => self.status = Status::Sending { channel, packet }, Ok(Receive { channel }) => self.status = Status::Receiving { channel }, - Ok(Parallel { - body, - return_channel, - }) => { + Ok(Parallel { body, return_channel }) => { self.status = Status::InParallelScope { body, return_channel, } } + Ok(Try { body }) => { + self.status = Status::InTry { body } + } Err(reason) => self.panic(reason), } } @@ -103,6 +104,7 @@ enum SuccessfulBehavior { body: Pointer, return_channel: ChannelId, }, + Try { body: Pointer } } use SuccessfulBehavior::*; @@ -420,6 +422,12 @@ impl Heap { }) } + fn try_(&mut self, args: &[Pointer]) -> BuiltinResult { + unpack!(self, args, |body: Closure| { + Try { body: body.address } + }) + } + fn type_of(&mut self, args: &[Pointer]) -> BuiltinResult { unpack_and_later_drop!(self, args, |value: Any| { let symbol = match **value { diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index d045757ef..bce2c2300 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -52,6 +52,7 @@ pub enum Status { body: Pointer, return_channel: ChannelId, }, + InTry { body: Pointer }, Done, Panicked { reason: String, @@ -175,6 +176,22 @@ impl Fiber { Err(reason) => self.panic(reason), } } + pub fn complete_try(&mut self, result: Result) { + let result = match result { + Ok(Packet { heap, value: return_value }) => { + let ok = self.heap.create_symbol("Ok".to_string()); + let return_value = heap.clone_single_to_other_heap(&mut self.heap, return_value); + self.heap.create_list(&[ok, return_value]) + }, + Err(panic_reason) => { + let err = self.heap.create_symbol("Err".to_string()); + let reason = self.heap.create_text(panic_reason); + self.heap.create_list(&[err, reason]) + }, + }; + self.data_stack.push(result); + self.status = Status::Running; + } fn get_from_data_stack(&self, offset: usize) -> Pointer { self.data_stack[self.data_stack.len() - 1 - offset as usize] diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 0280943a4..d37b0c943 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -9,6 +9,7 @@ mod use_module; use std::{marker::PhantomData, collections::{HashMap, VecDeque}, fmt}; pub use fiber::{Fiber, TearDownResult}; pub use heap::{Closure, Heap, Object, Pointer}; +use itertools::Itertools; use rand::seq::SliceRandom; use tracing::{info, warn}; use crate::vm::heap::Struct; @@ -43,6 +44,8 @@ enum FiberTree { /// former single fiber, the tree now runs the closure passed to /// `core.parallel` as well as any other spawned children. Parallel(Parallel), + + Try(Try), } #[derive(Clone)] struct Single { @@ -60,6 +63,11 @@ struct Child { fiber: FiberId, return_channel: ChannelId, } +#[derive(Clone)] +struct Try { + paused_fiber: Single, + child: FiberId, +} #[derive(Clone)] enum Channel { @@ -131,7 +139,7 @@ impl Vm { fiber::Status::Sending { .. } | fiber::Status::Receiving { .. } => Status::WaitingForOperations, fiber::Status::CreatingChannel { .. } | - fiber::Status::InParallelScope { .. } => unreachable!(), + fiber::Status::InParallelScope { .. } | fiber::Status::InTry { .. } => unreachable!(), fiber::Status::Done => Status::Done, fiber::Status::Panicked { reason } => Status::Panicked { reason: reason.clone() }, }, @@ -148,6 +156,7 @@ impl Vm { // were waiting for operations. Status::WaitingForOperations }, + FiberTree::Try(Try { child, .. }) => self.status_of(*child), } } fn is_running(&self) -> bool { @@ -174,6 +183,7 @@ impl Vm { FiberTree::Parallel(Parallel { children, .. }) => { fiber_id = children.choose(&mut rand::thread_rng()).unwrap().fiber; }, + FiberTree::Try(Try { child, .. }) => fiber_id = *child, } }; @@ -230,6 +240,26 @@ impl Vm { false }, + fiber::Status::InTry { body } => { + let child_id = { + let mut heap = Heap::default(); + let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); + let id = self.fiber_id_generator.generate(); + self.fibers.insert(id, FiberTree::Single(Single { + fiber: Fiber::new_for_running_closure(heap, body, &[]), + parent: Some(fiber_id), + })); + id + }; + + let paused_fiber = self.fibers.remove(&fiber_id).unwrap().into_single().unwrap(); + self.fibers.insert(fiber_id, FiberTree::Try(Try { + paused_fiber, + child: child_id, + })); + + false + }, fiber::Status::Done => { info!("A fiber is done."); true @@ -244,37 +274,46 @@ impl Vm { let single = self.fibers.remove(&fiber_id).unwrap().into_single().unwrap(); let TearDownResult { heap, result, .. } = single.fiber.tear_down(); + // TODO: Are there non-root fibers without parents? if let Some(parent_id) = single.parent { - let Parallel { children, nursery, .. } = self.fibers.get_mut(&parent_id).unwrap().as_parallel_mut().unwrap(); - // TODO: Turn children into map to make this less awkward. - let index = children.iter_mut().position(|child| child.fiber == fiber_id).unwrap(); - let child = children.remove(index); - let nursery = *nursery; - - let parallel_result = match result { - Ok(return_value) => { - let is_finished = children.is_empty(); - self.send_to_channel(None, child.return_channel, Packet { heap, value: return_value }); - - if is_finished { - Some(Ok(())) - } else { - None + match self.fibers.get_mut(&parent_id).unwrap() { + FiberTree::Single(_) => unreachable!(), + FiberTree::Parallel(Parallel { children, nursery, .. }) => { + // TODO: Turn children into map to make this less awkward. + let index = children.iter_mut().position(|child| child.fiber == fiber_id).unwrap(); + let child = children.remove(index); + let nursery = *nursery; + + let result_of_parallel = match result { + Ok(return_value) => { + let is_finished = children.is_empty(); + self.send_to_channel(None, child.return_channel, Packet { heap, value: return_value }); + + if is_finished { + Some(Ok(())) + } else { + None + } + }, + Err(panic_reason) => { + for child in children.clone() { + self.cancel(child.fiber); + } + Some(Err(panic_reason)) + }, + }; + + if let Some(result) = result_of_parallel { + self.channels.remove(&nursery).unwrap().to_nursery().unwrap(); + let Parallel { mut paused_fiber, .. } = self.fibers.remove(&parent_id).unwrap().into_parallel().unwrap(); + paused_fiber.fiber.complete_parallel_scope(result); + self.fibers.insert(parent_id, FiberTree::Single(paused_fiber)); } }, - Err(panic_reason) => { - for child in children.clone() { - self.cancel(child.fiber); - } - Some(Err(panic_reason)) + FiberTree::Try(Try { .. }) => { + let Try { mut paused_fiber, .. } = self.fibers.remove(&parent_id).unwrap().into_try().unwrap(); + paused_fiber.fiber.complete_try(result.map(|value| Packet { heap, value })); }, - }; - - if let Some(result) = parallel_result { - self.channels.remove(&nursery).unwrap().to_nursery().unwrap(); - let Parallel { mut paused_fiber, .. } = self.fibers.remove(&parent_id).unwrap().into_parallel().unwrap(); - paused_fiber.fiber.complete_parallel_scope(result); - self.fibers.insert(parent_id, FiberTree::Single(paused_fiber)); } } } @@ -288,23 +327,10 @@ impl Vm { self.cancel(child.fiber); } }, + FiberTree::Try(Try { child, .. }) => self.cancel(child), } } - fn try_() { - // let result = match result { - // Ok(return_value) => { - // let ok = heap.create_symbol("Ok".to_string()); - // heap.create_list(&[ok, return_value]) - // }, - // Err(panic_reason) => { - // let err = heap.create_symbol("Err".to_string()); - // let reason = heap.create_text(panic_reason); - // heap.create_list(&[err, reason]) - // }, - // }; - } - fn send_to_channel(&mut self, performing_fiber: Option, channel: ChannelId, packet: Packet) { match self.channels.get_mut(&channel).unwrap() { Channel::Internal(channel) => { @@ -463,6 +489,13 @@ impl FiberTree { _ => None, } } + + fn into_try(self) -> Option { + match self { + FiberTree::Try(try_) => Some(try_), + _ => None, + } + } } impl fmt::Debug for Vm { @@ -480,6 +513,7 @@ impl fmt::Debug for FiberTree { match self { Self::Single(Single { fiber, parent }) => f.debug_struct("SingleFiber").field("status", &fiber.status()).field("parent", parent).finish(), Self::Parallel(Parallel { children, nursery, .. }) => f.debug_struct("ParallelSection").field("children", children).field("nursery", nursery).finish(), + Self::Try(Try { child, .. }) => f.debug_struct("Try").field("child", child).finish(), } } } From 8f89e960429cda020e2028f6f463144a63b84b7f Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sat, 10 Sep 2022 22:58:19 +0200 Subject: [PATCH 34/59] Call main function --- compiler/src/main.rs | 71 +++++++++++++++++++++++++++++++++++++----- compiler/src/vm/mod.rs | 9 ++++-- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 76bf25cf5..dab76c2b3 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -28,7 +28,7 @@ use crate::{ module::{Module, ModuleKind}, vm::{ context::{DbUseProvider, ModularContext, RunForever}, - Closure, Vm, TearDownResult, Status, + Closure, Vm, TearDownResult, Status, Struct, }, }; use compiler::lir::Lir; @@ -39,7 +39,7 @@ use std::{ env::current_dir, path::PathBuf, sync::{mpsc::channel, Arc}, - time::Duration, + time::Duration, convert::TryInto, collections::HashMap, }; use structopt::StructOpt; use tower_lsp::{LspService, Server}; @@ -223,6 +223,64 @@ fn run(options: CandyRunOptions) { } } info!("Tree: {:#?}", vm); + let TearDownResult { + tracer, + result, + mut heap, + .. + } = vm.tear_down(); + + if options.debug { + module.dump_associated_debug_file("trace", &tracer.format_call_tree(&heap)); + } + + let exported_definitions: Struct = match result { + Ok(return_value) => { + info!( + "The module exports these definitions: {}", + return_value.format(&heap), + ); + heap.get(return_value).data.clone().try_into().unwrap() + }, + Err(reason) => { + error!("The module panicked because {reason}."); + error!("This is the stack trace:"); + tracer.dump_stack_trace(&db, &heap); + return; + } + }; + + let main = heap.create_symbol("Main".to_string()); + let main = match exported_definitions.get(&heap, main) { + Some(main) => main, + None => { + error!("The module doesn't contain a main function."); + return; + }, + }; + + info!("Running main function."); + // TODO: Add environment stuff. + let environment = heap.create_struct(HashMap::new()); + let mut vm = Vm::new_for_running_closure(heap, main, &[environment]); + loop { + info!("Tree: {:#?}", vm); + match vm.status() { + Status::CanRun => { + debug!("VM still running."); + vm.run(&mut ModularContext { + use_provider: DbUseProvider { db: &db }, + execution_controller: RunForever, + }); + // TODO: handle operations + }, + Status::WaitingForOperations => { + todo!("VM can't proceed until some operations complete."); + }, + _ => break, + } + } + info!("Tree: {:#?}", vm); let TearDownResult { tracer, result, @@ -232,19 +290,16 @@ fn run(options: CandyRunOptions) { match result { Ok(return_value) => info!( - "The module exports these definitions: {}", + "The main function returned: {}", return_value.format(&heap) ), Err(reason) => { - error!("The module panicked because {reason}."); + error!("The main function panicked because {reason}."); error!("This is the stack trace:"); tracer.dump_stack_trace(&db, &heap); + return; } } - - if options.debug { - module.dump_associated_debug_file("trace", &tracer.format_call_tree(&heap)); - } } async fn fuzz(options: CandyFuzzOptions) { diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index d37b0c943..ec6f3834a 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -8,11 +8,10 @@ mod use_module; use std::{marker::PhantomData, collections::{HashMap, VecDeque}, fmt}; pub use fiber::{Fiber, TearDownResult}; -pub use heap::{Closure, Heap, Object, Pointer}; +pub use heap::{Closure, Heap, Object, Pointer, Struct}; use itertools::Itertools; use rand::seq::SliceRandom; use tracing::{info, warn}; -use crate::vm::heap::Struct; use self::{heap::{ChannelId, SendPort}, channel::{ChannelBuf, Packet}, context::Context, tracer::Tracer}; /// A VM represents a Candy program that thinks it's currently running. Because @@ -173,6 +172,12 @@ impl Vm { self.fiber().tracer.clone() } + pub fn create_channel(&mut self) -> ChannelId { + let id = self.channel_id_generator.generate(); + self.channels.insert(id, Channel::External); + id + } + pub fn run(&mut self, context: &mut C) { assert!(self.is_running(), "Called Vm::run on a VM that is not ready to run."); From 8391286c4a2f4d87008ba603cb12bb0658c6f381 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sun, 11 Sep 2022 22:08:57 +0200 Subject: [PATCH 35/59] Clean up cargo lints --- compiler/src/fuzzer/mod.rs | 7 +- .../hints/constant_evaluator.rs | 10 +- compiler/src/main.rs | 24 +- compiler/src/vm/builtin_functions.rs | 17 +- compiler/src/vm/channel.rs | 5 +- compiler/src/vm/fiber.rs | 15 +- compiler/src/vm/mod.rs | 386 +++++++++++++----- compiler/src/vm/utils.rs | 20 - 8 files changed, 318 insertions(+), 166 deletions(-) delete mode 100644 compiler/src/vm/utils.rs diff --git a/compiler/src/fuzzer/mod.rs b/compiler/src/fuzzer/mod.rs index 512c401c0..8ad87c9a0 100644 --- a/compiler/src/fuzzer/mod.rs +++ b/compiler/src/fuzzer/mod.rs @@ -16,13 +16,12 @@ use tracing::info; pub async fn fuzz(db: &Database, module: Module) { let (fuzzables_heap, fuzzables) = { - let mut vm = Vm::new_for_running_module_closure( - Closure::of_module(db, module.clone()).unwrap(), - ); + let mut vm = + Vm::new_for_running_module_closure(Closure::of_module(db, module.clone()).unwrap()); vm.run(&mut ModularContext { use_provider: DbUseProvider { db }, execution_controller: RunForever, - }, todo!()); + }); let result = vm.tear_down(); (result.heap, result.fuzzable_closures) }; diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index d309110c6..feeb49e82 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -10,9 +10,10 @@ use crate::{ language_server::hints::{utils::id_to_end_of_line, HintKind}, module::Module, vm::{ + self, context::{DbUseProvider, ModularContext, RunLimitedNumberOfInstructions}, tracer::TraceEntry, - Closure, Vm, Heap, Pointer, self, + Closure, Heap, Pointer, Vm, }, }; use itertools::Itertools; @@ -27,9 +28,8 @@ pub struct ConstantEvaluator { impl ConstantEvaluator { pub fn update_module(&mut self, db: &Database, module: Module) { - let vm = Vm::new_for_running_module_closure( - Closure::of_module(db, module.clone()).unwrap(), - ); + let vm = + Vm::new_for_running_module_closure(Closure::of_module(db, module.clone()).unwrap()); self.vms.insert(module, vm); } @@ -54,7 +54,7 @@ impl ConstantEvaluator { vm.run(&mut ModularContext { use_provider: DbUseProvider { db }, execution_controller: RunLimitedNumberOfInstructions::new(500), - }, todo!()); + }); Some(module.clone()) } else { None diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 2dfefe156..a09208010 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -28,7 +28,7 @@ use crate::{ module::{Module, ModuleKind}, vm::{ context::{DbUseProvider, ModularContext, RunForever}, - Closure, Vm, Status, TearDownResult, Struct, + Closure, Status, Struct, TearDownResult, Vm, }, }; use compiler::lir::Lir; @@ -36,10 +36,12 @@ use itertools::Itertools; use language_server::CandyLanguageServer; use notify::{watcher, RecursiveMode, Watcher}; use std::{ + collections::HashMap, + convert::TryInto, env::current_dir, path::PathBuf, sync::{mpsc::channel, Arc}, - time::Duration, convert::TryInto, collections::HashMap, + time::Duration, }; use structopt::StructOpt; use tower_lsp::{LspService, Server}; @@ -215,10 +217,10 @@ fn run(options: CandyRunOptions) { execution_controller: RunForever, }); // TODO: handle operations - }, + } Status::WaitingForOperations => { todo!("VM can't proceed until some operations complete."); - }, + } _ => break, } } @@ -241,7 +243,7 @@ fn run(options: CandyRunOptions) { return_value.format(&heap), ); heap.get(return_value).data.clone().try_into().unwrap() - }, + } Err(reason) => { error!("The module panicked because {reason}."); error!("This is the stack trace:"); @@ -256,7 +258,7 @@ fn run(options: CandyRunOptions) { None => { error!("The module doesn't contain a main function."); return; - }, + } }; info!("Running main function."); @@ -273,10 +275,10 @@ fn run(options: CandyRunOptions) { execution_controller: RunForever, }); // TODO: handle operations - }, + } Status::WaitingForOperations => { todo!("VM can't proceed until some operations complete."); - }, + } _ => break, } } @@ -289,15 +291,11 @@ fn run(options: CandyRunOptions) { } = vm.tear_down(); match result { - Ok(return_value) => info!( - "The main function returned: {}", - return_value.format(&heap) - ), + Ok(return_value) => info!("The main function returned: {}", return_value.format(&heap)), Err(reason) => { error!("The main function panicked because {reason}."); error!("This is the stack trace:"); tracer.dump_stack_trace(&db, &heap); - return; } } } diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index e2360f31c..db0abaf1a 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -70,15 +70,16 @@ impl Fiber { Ok(CreateChannel { capacity }) => self.status = Status::CreatingChannel { capacity }, Ok(Send { channel, packet }) => self.status = Status::Sending { channel, packet }, Ok(Receive { channel }) => self.status = Status::Receiving { channel }, - Ok(Parallel { body, return_channel }) => { + Ok(Parallel { + body, + return_channel, + }) => { self.status = Status::InParallelScope { body, return_channel, } } - Ok(Try { body }) => { - self.status = Status::InTry { body } - } + Ok(Try { body }) => self.status = Status::InTry { body }, Err(reason) => self.panic(reason), } } @@ -104,7 +105,9 @@ enum SuccessfulBehavior { body: Pointer, return_channel: ChannelId, }, - Try { body: Pointer } + Try { + body: Pointer, + }, } use SuccessfulBehavior::*; @@ -423,9 +426,7 @@ impl Heap { } fn try_(&mut self, args: &[Pointer]) -> BuiltinResult { - unpack!(self, args, |body: Closure| { - Try { body: body.address } - }) + unpack!(self, args, |body: Closure| { Try { body: body.address } }) } fn type_of(&mut self, args: &[Pointer]) -> BuiltinResult { diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index 49b7f0b1f..4ae8b00c4 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -1,4 +1,3 @@ -use itertools::Itertools; use super::{Heap, Pointer}; use std::{collections::VecDeque, fmt}; @@ -47,7 +46,9 @@ impl ChannelBuf { } pub fn receive(&mut self) -> Packet { - self.packets.pop_front().expect("Tried to receive from channel that is empty.") + self.packets + .pop_front() + .expect("Tried to receive from channel that is empty.") } } diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index bce2c2300..0bef6094f 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -52,7 +52,9 @@ pub enum Status { body: Pointer, return_channel: ChannelId, }, - InTry { body: Pointer }, + InTry { + body: Pointer, + }, Done, Panicked { reason: String, @@ -172,22 +174,25 @@ impl Fiber { Ok(()) => { self.data_stack.push(self.heap.create_nothing()); self.status = Status::Running; - }, + } Err(reason) => self.panic(reason), } } pub fn complete_try(&mut self, result: Result) { let result = match result { - Ok(Packet { heap, value: return_value }) => { + Ok(Packet { + heap, + value: return_value, + }) => { let ok = self.heap.create_symbol("Ok".to_string()); let return_value = heap.clone_single_to_other_heap(&mut self.heap, return_value); self.heap.create_list(&[ok, return_value]) - }, + } Err(panic_reason) => { let err = self.heap.create_symbol("Err".to_string()); let reason = self.heap.create_text(panic_reason); self.heap.create_list(&[err, reason]) - }, + } }; self.data_stack.push(result); self.status = Status::Running; diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 08a9835a9..9adb32c0c 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -5,15 +5,23 @@ mod fiber; mod heap; pub mod tracer; mod use_module; -pub mod utils; -use std::{marker::PhantomData, collections::{HashMap, VecDeque}, fmt}; +use self::{ + channel::{ChannelBuf, Packet}, + context::Context, + heap::{ChannelId, SendPort}, + tracer::Tracer, +}; pub use fiber::{Fiber, TearDownResult}; pub use heap::{Closure, Heap, Object, Pointer, Struct}; use itertools::Itertools; use rand::seq::SliceRandom; +use std::{ + collections::{HashMap, VecDeque}, + fmt, + marker::PhantomData, +}; use tracing::{info, warn}; -use self::{heap::{ChannelId, SendPort}, channel::{ChannelBuf, Packet}, context::Context, tracer::Tracer}; /// A VM represents a Candy program that thinks it's currently running. Because /// VMs are first-class Rust structs, they enable other code to store "freezed" @@ -22,7 +30,7 @@ use self::{heap::{ChannelId, SendPort}, channel::{ChannelBuf, Packet}, context:: pub struct Vm { fibers: HashMap, root_fiber: FiberId, - + // TODO: Drop channels channels: HashMap, pub external_operations: HashMap>, @@ -106,7 +114,10 @@ pub enum Status { impl Vm { fn new_with_fiber(fiber: Fiber) -> Self { - let fiber = FiberTree::Single(Single { fiber, parent: None }); + let fiber = FiberTree::Single(Single { + fiber, + parent: None, + }); let mut fiber_id_generator = IdGenerator::start_at(0); let root_fiber_id = fiber_id_generator.generate(); Self { @@ -125,7 +136,12 @@ impl Vm { Self::new_with_fiber(Fiber::new_for_running_module_closure(closure)) } pub fn tear_down(mut self) -> TearDownResult { - let fiber = self.fibers.remove(&self.root_fiber).unwrap().into_single().unwrap(); + let fiber = self + .fibers + .remove(&self.root_fiber) + .unwrap() + .into_single() + .unwrap(); fiber.fiber.tear_down() } @@ -136,18 +152,22 @@ impl Vm { match &self.fibers[&fiber] { FiberTree::Single(Single { fiber, .. }) => match &fiber.status { fiber::Status::Running => Status::CanRun, - fiber::Status::Sending { .. } | - fiber::Status::Receiving { .. } => Status::WaitingForOperations, - fiber::Status::CreatingChannel { .. } | - fiber::Status::InParallelScope { .. } | fiber::Status::InTry { .. } => unreachable!(), + fiber::Status::Sending { .. } | fiber::Status::Receiving { .. } => { + Status::WaitingForOperations + } + fiber::Status::CreatingChannel { .. } + | fiber::Status::InParallelScope { .. } + | fiber::Status::InTry { .. } => unreachable!(), fiber::Status::Done => Status::Done, - fiber::Status::Panicked { reason } => Status::Panicked { reason: reason.clone() }, + fiber::Status::Panicked { reason } => Status::Panicked { + reason: reason.clone(), + }, }, FiberTree::Parallel(Parallel { children, .. }) => { for child in children { match self.status_of(child.fiber) { Status::CanRun => return Status::CanRun, - Status::WaitingForOperations => {}, + Status::WaitingForOperations => {} Status::Done | Status::Panicked { .. } => unreachable!(), }; } @@ -155,7 +175,7 @@ impl Vm { // removed. Thus, there was at least one child and all children // were waiting for operations. Status::WaitingForOperations - }, + } FiberTree::Try(Try { child, .. }) => self.status_of(*child), } } @@ -166,7 +186,8 @@ impl Vm { matches!(self.status(), Status::Done | Status::Panicked { .. }) } - pub fn fiber(&self) -> &Fiber { // TODO: Remove before merging the PR + pub fn fiber(&self) -> &Fiber { + // TODO: Remove before merging the PR todo!() } pub fn cloned_tracer(&self) -> Tracer { @@ -180,7 +201,10 @@ impl Vm { } pub fn run(&mut self, context: &mut C) { - assert!(self.is_running(), "Called Vm::run on a VM that is not ready to run."); + assert!( + self.is_running(), + "Called Vm::run on a VM that is not ready to run." + ); let mut fiber_id = self.root_fiber; let fiber = loop { @@ -188,7 +212,7 @@ impl Vm { FiberTree::Single(Single { fiber, .. }) => break fiber, FiberTree::Parallel(Parallel { children, .. }) => { fiber_id = children.choose(&mut rand::thread_rng()).unwrap().fiber; - }, + } FiberTree::Try(Try { child, .. }) => fiber_id = *child, } }; @@ -204,10 +228,17 @@ impl Vm { fiber::Status::Running => false, fiber::Status::CreatingChannel { capacity } => { let channel_id = self.channel_id_generator.generate(); - self.channels.insert(channel_id, Channel::Internal(InternalChannel { buffer: ChannelBuf::new(capacity), pending_sends: Default::default(), pending_receives: Default::default() })); + self.channels.insert( + channel_id, + Channel::Internal(InternalChannel { + buffer: ChannelBuf::new(capacity), + pending_sends: Default::default(), + pending_receives: Default::default(), + }), + ); fiber.complete_channel_create(channel_id); false - }, + } fiber::Status::Sending { channel, packet } => { self.send_to_channel(Some(fiber_id), channel, packet); false @@ -216,7 +247,10 @@ impl Vm { self.receive_from_channel(Some(fiber_id), channel); false } - fiber::Status::InParallelScope { body, return_channel } => { + fiber::Status::InParallelScope { + body, + return_channel, + } => { let nursery_id = self.channel_id_generator.generate(); self.channels.insert(nursery_id, Channel::Nursery(fiber_id)); @@ -225,152 +259,234 @@ impl Vm { let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); let nursery_send_port = heap.create_send_port(nursery_id); let id = self.fiber_id_generator.generate(); - self.fibers.insert(id, FiberTree::Single(Single{ - fiber: Fiber::new_for_running_closure(heap, body, &[nursery_send_port]), - parent: Some(fiber_id) - })); + self.fibers.insert( + id, + FiberTree::Single(Single { + fiber: Fiber::new_for_running_closure(heap, body, &[nursery_send_port]), + parent: Some(fiber_id), + }), + ); id }; // TODO: Create utility method for removing and adding under same ID. // TODO: Make it so that the initial fiber doesn't need a return channel. - let paused_fiber = self.fibers.remove(&fiber_id).unwrap().into_single().unwrap(); - self.fibers.insert(fiber_id, FiberTree::Parallel(Parallel { - paused_fiber, - children: vec![Child { - fiber: first_child_id, - return_channel: return_channel, - }], - nursery: nursery_id, - })); + let paused_fiber = self + .fibers + .remove(&fiber_id) + .unwrap() + .into_single() + .unwrap(); + self.fibers.insert( + fiber_id, + FiberTree::Parallel(Parallel { + paused_fiber, + children: vec![Child { + fiber: first_child_id, + return_channel, + }], + nursery: nursery_id, + }), + ); false - }, + } fiber::Status::InTry { body } => { let child_id = { let mut heap = Heap::default(); let body = fiber.heap.clone_single_to_other_heap(&mut heap, body); let id = self.fiber_id_generator.generate(); - self.fibers.insert(id, FiberTree::Single(Single { - fiber: Fiber::new_for_running_closure(heap, body, &[]), - parent: Some(fiber_id), - })); + self.fibers.insert( + id, + FiberTree::Single(Single { + fiber: Fiber::new_for_running_closure(heap, body, &[]), + parent: Some(fiber_id), + }), + ); id }; - let paused_fiber = self.fibers.remove(&fiber_id).unwrap().into_single().unwrap(); - self.fibers.insert(fiber_id, FiberTree::Try(Try { - paused_fiber, - child: child_id, - })); - + let paused_fiber = self + .fibers + .remove(&fiber_id) + .unwrap() + .into_single() + .unwrap(); + self.fibers.insert( + fiber_id, + FiberTree::Try(Try { + paused_fiber, + child: child_id, + }), + ); + false - }, + } fiber::Status::Done => { info!("A fiber is done."); true - }, + } fiber::Status::Panicked { reason } => { warn!("A fiber panicked because {reason}."); true - }, + } }; if is_finished && fiber_id != self.root_fiber { - let single = self.fibers.remove(&fiber_id).unwrap().into_single().unwrap(); + let single = self + .fibers + .remove(&fiber_id) + .unwrap() + .into_single() + .unwrap(); let TearDownResult { heap, result, .. } = single.fiber.tear_down(); // TODO: Are there non-root fibers without parents? if let Some(parent_id) = single.parent { match self.fibers.get_mut(&parent_id).unwrap() { FiberTree::Single(_) => unreachable!(), - FiberTree::Parallel(Parallel { children, nursery, .. }) => { + FiberTree::Parallel(Parallel { + children, nursery, .. + }) => { // TODO: Turn children into map to make this less awkward. - let index = children.iter_mut().position(|child| child.fiber == fiber_id).unwrap(); + let index = children + .iter_mut() + .position(|child| child.fiber == fiber_id) + .unwrap(); let child = children.remove(index); let nursery = *nursery; let result_of_parallel = match result { Ok(return_value) => { let is_finished = children.is_empty(); - self.send_to_channel(None, child.return_channel, Packet { heap, value: return_value }); + self.send_to_channel( + None, + child.return_channel, + Packet { + heap, + value: return_value, + }, + ); if is_finished { Some(Ok(())) } else { None } - }, + } Err(panic_reason) => { for child in children.clone() { self.cancel(child.fiber); } Some(Err(panic_reason)) - }, + } }; if let Some(result) = result_of_parallel { - self.channels.remove(&nursery).unwrap().to_nursery().unwrap(); - let Parallel { mut paused_fiber, .. } = self.fibers.remove(&parent_id).unwrap().into_parallel().unwrap(); + self.channels + .remove(&nursery) + .unwrap() + .to_nursery() + .unwrap(); + let Parallel { + mut paused_fiber, .. + } = self + .fibers + .remove(&parent_id) + .unwrap() + .into_parallel() + .unwrap(); paused_fiber.fiber.complete_parallel_scope(result); - self.fibers.insert(parent_id, FiberTree::Single(paused_fiber)); + self.fibers + .insert(parent_id, FiberTree::Single(paused_fiber)); } - }, + } FiberTree::Try(Try { .. }) => { - let Try { mut paused_fiber, .. } = self.fibers.remove(&parent_id).unwrap().into_try().unwrap(); - paused_fiber.fiber.complete_try(result.map(|value| Packet { heap, value })); - }, + let Try { + mut paused_fiber, .. + } = self.fibers.remove(&parent_id).unwrap().into_try().unwrap(); + paused_fiber + .fiber + .complete_try(result.map(|value| Packet { heap, value })); + } } } } } fn cancel(&mut self, fiber: FiberId) { match self.fibers.remove(&fiber).unwrap() { - FiberTree::Single(_) => {}, - FiberTree::Parallel(Parallel { children, nursery, .. }) => { - self.channels.remove(&nursery).unwrap().to_nursery().unwrap(); + FiberTree::Single(_) => {} + FiberTree::Parallel(Parallel { + children, nursery, .. + }) => { + self.channels + .remove(&nursery) + .unwrap() + .to_nursery() + .unwrap(); for child in children { self.cancel(child.fiber); } - }, + } FiberTree::Try(Try { child, .. }) => self.cancel(child), } } - fn send_to_channel(&mut self, performing_fiber: Option, channel: ChannelId, packet: Packet) { + fn send_to_channel( + &mut self, + performing_fiber: Option, + channel: ChannelId, + packet: Packet, + ) { match self.channels.get_mut(&channel).unwrap() { Channel::Internal(channel) => { channel.send(&mut self.fibers, performing_fiber, packet); - }, - Channel::External => { - self.push_external_operation(channel, Operation { + } + Channel::External => self.push_external_operation( + channel, + Operation { performing_fiber, kind: OperationKind::Send { packet }, - }) - }, + }, + ), Channel::Nursery(parent_id) => { info!("Nursery received packet {:?}", packet); - let (heap, closure_to_spawn, return_channel) = match Self::parse_spawn_packet(packet) { - Some(it) => it, - None => { - // The nursery received an invalid message. TODO: Panic. - panic!("A nursery received an invalid message."); - } - }; + let (heap, closure_to_spawn, return_channel) = + match Self::parse_spawn_packet(packet) { + Some(it) => it, + None => { + // The nursery received an invalid message. TODO: Panic. + panic!("A nursery received an invalid message."); + } + }; let child_id = self.fiber_id_generator.generate(); - self.fibers.insert(child_id, FiberTree::Single(Single { fiber: Fiber::new_for_running_closure(heap, closure_to_spawn, &[]), parent: Some(*parent_id) })); - - let parent = self.fibers.get_mut(parent_id).unwrap().as_parallel_mut().unwrap(); - parent.children.push(Child { fiber: child_id, return_channel }); + self.fibers.insert( + child_id, + FiberTree::Single(Single { + fiber: Fiber::new_for_running_closure(heap, closure_to_spawn, &[]), + parent: Some(*parent_id), + }), + ); + + let parent = self + .fibers + .get_mut(parent_id) + .unwrap() + .as_parallel_mut() + .unwrap(); + parent.children.push(Child { + fiber: child_id, + return_channel, + }); InternalChannel::complete_send(&mut self.fibers, performing_fiber); - }, + } } } fn parse_spawn_packet(packet: Packet) -> Option<(Heap, Pointer, ChannelId)> { let Packet { mut heap, value } = packet; let arguments: Struct = heap.get(value).data.clone().try_into().ok()?; - + let closure_symbol = heap.create_symbol("Closure".to_string()); let closure_address = arguments.get(&heap, closure_symbol)?; let closure: Closure = heap.get(closure_address).data.clone().try_into().ok()?; @@ -380,7 +496,12 @@ impl Vm { let return_channel_symbol = heap.create_symbol("ReturnChannel".to_string()); let return_channel_address = arguments.get(&heap, return_channel_symbol)?; - let return_channel: SendPort = heap.get(return_channel_address).data.clone().try_into().ok()?; + let return_channel: SendPort = heap + .get(return_channel_address) + .data + .clone() + .try_into() + .ok()?; Some((heap, closure_address, return_channel.channel)) } @@ -389,29 +510,44 @@ impl Vm { match self.channels.get_mut(&channel).unwrap() { Channel::Internal(channel) => { channel.receive(&mut self.fibers, performing_fiber); - }, + } Channel::External => { - self.push_external_operation(channel, Operation { - performing_fiber, - kind: OperationKind::Receive, - }); - }, + self.push_external_operation( + channel, + Operation { + performing_fiber, + kind: OperationKind::Receive, + }, + ); + } Channel::Nursery { .. } => unreachable!("nurseries are only sent stuff"), } } fn push_external_operation(&mut self, channel: ChannelId, operation: Operation) { - self.external_operations.entry(channel).or_default().push(operation); + self.external_operations + .entry(channel) + .or_default() + .push(operation); } } impl InternalChannel { - fn send(&mut self, fibers: &mut HashMap, performing_fiber: Option, packet: Packet) { + fn send( + &mut self, + fibers: &mut HashMap, + performing_fiber: Option, + packet: Packet, + ) { self.pending_sends.push_back((performing_fiber, packet)); self.work_on_pending_operations(fibers); } - fn receive(&mut self, fibers: &mut HashMap, performing_fiber: Option) { + fn receive( + &mut self, + fibers: &mut HashMap, + performing_fiber: Option, + ) { self.pending_receives.push_back(performing_fiber); self.work_on_pending_operations(fibers); } @@ -453,7 +589,11 @@ impl InternalChannel { fiber.fiber.complete_send(); } } - fn complete_receive(fibers: &mut HashMap, fiber: Option, packet: Packet) { + fn complete_receive( + fibers: &mut HashMap, + fiber: Option, + packet: Packet, + ) { if let Some(fiber) = fiber { let fiber = fibers.get_mut(&fiber).unwrap().as_single_mut().unwrap(); fiber.fiber.complete_receive(packet); @@ -473,13 +613,13 @@ impl FiberTree { fn into_single(self) -> Option { match self { FiberTree::Single(single) => Some(single), - _ => None + _ => None, } } fn as_single_mut(&mut self) -> Option<&mut Single> { match self { FiberTree::Single(single) => Some(single), - _ => None + _ => None, } } @@ -506,7 +646,11 @@ impl FiberTree { impl fmt::Debug for Vm { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Vm").field("fibers", &self.fibers).field("channels", &self.channels).field("external_operations", &self.external_operations).finish() + f.debug_struct("Vm") + .field("fibers", &self.fibers) + .field("channels", &self.channels) + .field("external_operations", &self.external_operations) + .finish() } } impl fmt::Debug for FiberId { @@ -517,9 +661,19 @@ impl fmt::Debug for FiberId { impl fmt::Debug for FiberTree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Single(Single { fiber, parent }) => f.debug_struct("SingleFiber").field("status", &fiber.status()).field("parent", parent).finish(), - Self::Parallel(Parallel { children, nursery, .. }) => f.debug_struct("ParallelSection").field("children", children).field("nursery", nursery).finish(), - Self::Try(Try { child, .. }) => f.debug_struct("Try").field("child", child).finish(), + Self::Single(Single { fiber, parent }) => f + .debug_struct("SingleFiber") + .field("status", &fiber.status()) + .field("parent", parent) + .finish(), + Self::Parallel(Parallel { + children, nursery, .. + }) => f + .debug_struct("ParallelSection") + .field("children", children) + .field("nursery", nursery) + .finish(), + Self::Try(Try { child, .. }) => f.debug_struct("Try").field("child", child).finish(), } } } @@ -531,16 +685,30 @@ impl fmt::Debug for Child { impl fmt::Debug for Channel { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Internal(InternalChannel { buffer, pending_sends, pending_receives }) => - f.debug_struct("InternalChannel") - .field("buffer", buffer) - .field("operations", - &pending_sends.iter() - .map(|(fiber, packet)| Operation { performing_fiber: fiber.clone(), kind: OperationKind::Send { packet: packet.clone() } }) - .chain(pending_receives.iter().map(|fiber| Operation { performing_fiber: fiber.clone(), kind: OperationKind::Receive })) - .collect_vec() - ) - .finish(), + Self::Internal(InternalChannel { + buffer, + pending_sends, + pending_receives, + }) => f + .debug_struct("InternalChannel") + .field("buffer", buffer) + .field( + "operations", + &pending_sends + .iter() + .map(|(fiber, packet)| Operation { + performing_fiber: fiber.clone(), + kind: OperationKind::Send { + packet: packet.clone(), + }, + }) + .chain(pending_receives.iter().map(|fiber| Operation { + performing_fiber: fiber.clone(), + kind: OperationKind::Receive, + })) + .collect_vec(), + ) + .finish(), Self::External => f.debug_tuple("External").finish(), Self::Nursery(fiber) => f.debug_tuple("Nursery").field(fiber).finish(), } diff --git a/compiler/src/vm/utils.rs b/compiler/src/vm/utils.rs deleted file mode 100644 index ba4c1a6d2..000000000 --- a/compiler/src/vm/utils.rs +++ /dev/null @@ -1,20 +0,0 @@ -use std::marker::PhantomData; - -#[derive(Clone)] -pub struct IdGenerator> { - next_id: usize, - _data: PhantomData, -} -impl> IdGenerator { - pub fn start_at(id: usize) -> Self { - Self { - next_id: id, - _data: Default::default(), - } - } - pub fn generate(&mut self) -> T { - let id = self.next_id; - self.next_id += 1; - id.into() - } -} From f3f7d49700fa38f54c80477f312d1f0c4fa1433c Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sun, 11 Sep 2022 22:09:07 +0200 Subject: [PATCH 36/59] Add main function to benchmark --- packages/Benchmark.candy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/Benchmark.candy b/packages/Benchmark.candy index 18078db2b..8642a9297 100644 --- a/packages/Benchmark.candy +++ b/packages/Benchmark.candy @@ -33,3 +33,6 @@ core.parallel { nursery -> core.async nursery { ✨.print "Banana" } ✨.print "Hi" } + +main environment := + ✨.print "Hello, world!" From a22f716b3d7b5105d381a4128e1ac227ae971307 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sun, 11 Sep 2022 22:35:54 +0200 Subject: [PATCH 37/59] Implement parallel builtin without channels --- compiler/src/builtin_functions.rs | 8 +--- compiler/src/vm/builtin_functions.rs | 53 +++++++------------------- compiler/src/vm/fiber.rs | 26 +++---------- compiler/src/vm/mod.rs | 56 +++++++++++++++------------- packages/Core/Concurrency.candy | 6 +-- 5 files changed, 53 insertions(+), 96 deletions(-) diff --git a/compiler/src/builtin_functions.rs b/compiler/src/builtin_functions.rs index c816b9e87..8924acb60 100644 --- a/compiler/src/builtin_functions.rs +++ b/compiler/src/builtin_functions.rs @@ -25,13 +25,7 @@ pub enum BuiltinFunction { IntShiftLeft, // (value: int) (amount: int) -> (shifted: int) IntShiftRight, // (value: int) (amount: int) -> (shifted: int) IntSubtract, // (minuend: int) (subtrahend: int) -> (difference: int) - - /// Implementation note: Why does `✨.parallel` not return the value - /// directly? The current architecture is chosen to make the VM's code more - /// uniform – every nursery child just sends its result to a channel and - /// there's no special casing for the first child, the closure passed to - /// `✨.parallel`. - Parallel, // (body: Closure, result: SendPort) -> Nothing + Parallel, // body: Closure -> returnValueOfClosure Print, // message -> Nothing StructGet, // struct key -> value StructGetKeys, // struct -> listOfKeys diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index db0abaf1a..c00d837d2 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -70,15 +70,7 @@ impl Fiber { Ok(CreateChannel { capacity }) => self.status = Status::CreatingChannel { capacity }, Ok(Send { channel, packet }) => self.status = Status::Sending { channel, packet }, Ok(Receive { channel }) => self.status = Status::Receiving { channel }, - Ok(Parallel { - body, - return_channel, - }) => { - self.status = Status::InParallelScope { - body, - return_channel, - } - } + Ok(Parallel { body }) => self.status = Status::InParallelScope { body }, Ok(Try { body }) => self.status = Status::InTry { body }, Err(reason) => self.panic(reason), } @@ -88,26 +80,12 @@ impl Fiber { type BuiltinResult = Result; enum SuccessfulBehavior { Return(Pointer), - DivergeControlFlow { - closure: Pointer, - }, - CreateChannel { - capacity: Capacity, - }, - Send { - channel: ChannelId, - packet: Packet, - }, - Receive { - channel: ChannelId, - }, - Parallel { - body: Pointer, - return_channel: ChannelId, - }, - Try { - body: Pointer, - }, + DivergeControlFlow { closure: Pointer }, + CreateChannel { capacity: Capacity }, + Send { channel: ChannelId, packet: Packet }, + Receive { channel: ChannelId }, + Parallel { body: Pointer }, + Try { body: Pointer }, } use SuccessfulBehavior::*; @@ -310,17 +288,14 @@ impl Heap { } fn parallel(&mut self, args: &[Pointer]) -> BuiltinResult { - unpack_and_later_drop!( - self, - args, - |body_taking_nursery: Closure, return_channel: SendPort| { - self.dup(body_taking_nursery.address); - Parallel { - body: body_taking_nursery.address, - return_channel: return_channel.channel, - } + unpack!(self, args, |body_taking_nursery: Closure| { + if body_taking_nursery.num_args != 1 { + return Err("Parallel expects a closure that takes a nursery.".to_string()); } - ) + Parallel { + body: body_taking_nursery.address, + } + }) } fn print(&mut self, args: &[Pointer]) -> BuiltinResult { diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index 0bef6094f..dfb0e2af9 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -38,27 +38,13 @@ pub struct Fiber { #[derive(Clone, Debug)] pub enum Status { Running, - CreatingChannel { - capacity: Capacity, - }, - Sending { - channel: ChannelId, - packet: Packet, - }, - Receiving { - channel: ChannelId, - }, - InParallelScope { - body: Pointer, - return_channel: ChannelId, - }, - InTry { - body: Pointer, - }, + CreatingChannel { capacity: Capacity }, + Sending { channel: ChannelId, packet: Packet }, + Receiving { channel: ChannelId }, + InParallelScope { body: Pointer }, + InTry { body: Pointer }, Done, - Panicked { - reason: String, - }, + Panicked { reason: String }, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 9adb32c0c..f47327639 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -64,12 +64,20 @@ struct Single { struct Parallel { paused_fiber: Single, children: Vec, + return_value: Option, nursery: ChannelId, } #[derive(Clone)] struct Child { fiber: FiberId, - return_channel: ChannelId, + + /// Most children of a nursery are spawned by sending a packet of the form + /// `[Closure: someClosure, ReturnChannel: someSendPort]` to the nursery – + /// the `core.async` function internally creates a channel, then sends this + /// packet, and finally returns the channel's receive port. + /// The child that doesn't have a return channel is the first child, which + /// is the body of the `✨.parallel` call. + return_channel: Option, } #[derive(Clone)] struct Try { @@ -247,10 +255,7 @@ impl Vm { self.receive_from_channel(Some(fiber_id), channel); false } - fiber::Status::InParallelScope { - body, - return_channel, - } => { + fiber::Status::InParallelScope { body } => { let nursery_id = self.channel_id_generator.generate(); self.channels.insert(nursery_id, Channel::Nursery(fiber_id)); @@ -283,8 +288,9 @@ impl Vm { paused_fiber, children: vec![Child { fiber: first_child_id, - return_channel, + return_channel: None, }], + return_value: None, nursery: nursery_id, }), ); @@ -345,28 +351,28 @@ impl Vm { if let Some(parent_id) = single.parent { match self.fibers.get_mut(&parent_id).unwrap() { FiberTree::Single(_) => unreachable!(), - FiberTree::Parallel(Parallel { - children, nursery, .. - }) => { + FiberTree::Parallel(parallel) => { // TODO: Turn children into map to make this less awkward. - let index = children + let index = parallel + .children .iter_mut() .position(|child| child.fiber == fiber_id) .unwrap(); - let child = children.remove(index); - let nursery = *nursery; + let child = parallel.children.remove(index); + let nursery = parallel.nursery; let result_of_parallel = match result { Ok(return_value) => { - let is_finished = children.is_empty(); - self.send_to_channel( - None, - child.return_channel, - Packet { - heap, - value: return_value, - }, - ); + let is_finished = parallel.children.is_empty(); + let packet = Packet { + heap, + value: return_value, + }; + if let Some(return_channel) = child.return_channel { + self.send_to_channel(None, return_channel, packet); + } else { + parallel.return_value = Some(packet); + } if is_finished { Some(Ok(())) @@ -375,7 +381,7 @@ impl Vm { } } Err(panic_reason) => { - for child in children.clone() { + for child in parallel.children.clone() { self.cancel(child.fiber); } Some(Err(panic_reason)) @@ -476,7 +482,7 @@ impl Vm { .unwrap(); parent.children.push(Child { fiber: child_id, - return_channel, + return_channel: Some(return_channel), }); InternalChannel::complete_send(&mut self.fibers, performing_fiber); @@ -697,13 +703,13 @@ impl fmt::Debug for Channel { &pending_sends .iter() .map(|(fiber, packet)| Operation { - performing_fiber: fiber.clone(), + performing_fiber: *fiber, kind: OperationKind::Send { packet: packet.clone(), }, }) .chain(pending_receives.iter().map(|fiber| Operation { - performing_fiber: fiber.clone(), + performing_fiber: *fiber, kind: OperationKind::Receive, })) .collect_vec(), diff --git a/packages/Core/Concurrency.candy b/packages/Core/Concurrency.candy index 6f89391dc..fe29d1fed 100644 --- a/packages/Core/Concurrency.candy +++ b/packages/Core/Concurrency.candy @@ -4,11 +4,7 @@ structGet = (use "..Struct").getUnwrap parallel body := needs (function.is1 body) - returnChannel = channel.create 1 - returnSendPort = structGet returnChannel 0 - returnReceivePort = structGet returnChannel 1 - ✨.parallel body returnSendPort - channel.receive returnReceivePort + ✨.parallel body async nursery body := needs (channel.isSendPort nursery) From d5b5f6ec89162e622dd551db13be13b4509ad1ec Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sun, 11 Sep 2022 23:03:09 +0200 Subject: [PATCH 38/59] Improve code quality --- compiler/src/vm/mod.rs | 179 +++++++++++++++++++++-------------------- 1 file changed, 90 insertions(+), 89 deletions(-) diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index f47327639..60f084c0f 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -55,30 +55,34 @@ enum FiberTree { Try(Try), } + +/// Single fibers are the leafs of the fiber tree. #[derive(Clone)] struct Single { fiber: Fiber, parent: Option, } + +/// When a parallel section is entered, the fiber that started the section is +/// paused. Instead, the children of the parallel section are run. Initially, +/// there's only one child – the closure given to the parallel builtin function. +/// Using the nursery parameter (a nursery can be thought of as a pointer to a +/// parallel section), you can also spawn other fibers. In contrast to the first +/// child, those children also have an explicit send port where the closure's +/// result is sent to. #[derive(Clone)] struct Parallel { paused_fiber: Single, - children: Vec, - return_value: Option, + children: HashMap, + return_value: Option, // will later contain the body's return value nursery: ChannelId, } #[derive(Clone)] -struct Child { - fiber: FiberId, - - /// Most children of a nursery are spawned by sending a packet of the form - /// `[Closure: someClosure, ReturnChannel: someSendPort]` to the nursery – - /// the `core.async` function internally creates a channel, then sends this - /// packet, and finally returns the channel's receive port. - /// The child that doesn't have a return channel is the first child, which - /// is the body of the `✨.parallel` call. - return_channel: Option, +enum ChildKind { + InitialChild, + SpawnedChild(ChannelId), } + #[derive(Clone)] struct Try { paused_fiber: Single, @@ -172,8 +176,8 @@ impl Vm { }, }, FiberTree::Parallel(Parallel { children, .. }) => { - for child in children { - match self.status_of(child.fiber) { + for child_fiber in children.keys() { + match self.status_of(*child_fiber) { Status::CanRun => return Status::CanRun, Status::WaitingForOperations => {} Status::Done | Status::Panicked { .. } => unreachable!(), @@ -219,7 +223,9 @@ impl Vm { match self.fibers.get_mut(&fiber_id).unwrap() { FiberTree::Single(Single { fiber, .. }) => break fiber, FiberTree::Parallel(Parallel { children, .. }) => { - fiber_id = children.choose(&mut rand::thread_rng()).unwrap().fiber; + let children_as_vec = children.iter().collect_vec(); + let random_child = children_as_vec.choose(&mut rand::thread_rng()).unwrap(); + fiber_id = *random_child.0 } FiberTree::Try(Try { child, .. }) => fiber_id = *child, } @@ -275,7 +281,6 @@ impl Vm { }; // TODO: Create utility method for removing and adding under same ID. - // TODO: Make it so that the initial fiber doesn't need a return channel. let paused_fiber = self .fibers .remove(&fiber_id) @@ -286,10 +291,9 @@ impl Vm { fiber_id, FiberTree::Parallel(Parallel { paused_fiber, - children: vec![Child { - fiber: first_child_id, - return_channel: None, - }], + children: [(first_child_id, ChildKind::InitialChild)] + .into_iter() + .collect(), return_value: None, nursery: nursery_id, }), @@ -347,75 +351,70 @@ impl Vm { .unwrap(); let TearDownResult { heap, result, .. } = single.fiber.tear_down(); - // TODO: Are there non-root fibers without parents? - if let Some(parent_id) = single.parent { - match self.fibers.get_mut(&parent_id).unwrap() { - FiberTree::Single(_) => unreachable!(), - FiberTree::Parallel(parallel) => { - // TODO: Turn children into map to make this less awkward. - let index = parallel - .children - .iter_mut() - .position(|child| child.fiber == fiber_id) - .unwrap(); - let child = parallel.children.remove(index); - let nursery = parallel.nursery; - - let result_of_parallel = match result { - Ok(return_value) => { - let is_finished = parallel.children.is_empty(); - let packet = Packet { - heap, - value: return_value, - }; - if let Some(return_channel) = child.return_channel { - self.send_to_channel(None, return_channel, packet); - } else { - parallel.return_value = Some(packet); + let parent_id = single + .parent + .expect("we already checked we're not the root fiber"); + match self.fibers.get_mut(&parent_id).unwrap() { + FiberTree::Single(_) => unreachable!(), + FiberTree::Parallel(parallel) => { + let child = parallel.children.remove(&fiber_id).unwrap(); + let nursery = parallel.nursery; + + let result_of_parallel = match result { + Ok(return_value) => { + let is_finished = parallel.children.is_empty(); + let packet = Packet { + heap, + value: return_value, + }; + match child { + ChildKind::InitialChild => parallel.return_value = Some(packet), + ChildKind::SpawnedChild(return_channel) => { + self.send_to_channel(None, return_channel, packet) } + } - if is_finished { - Some(Ok(())) - } else { - None - } + if is_finished { + Some(Ok(())) + } else { + None } - Err(panic_reason) => { - for child in parallel.children.clone() { - self.cancel(child.fiber); - } - Some(Err(panic_reason)) + } + Err(panic_reason) => { + for fiber_id in parallel.children.clone().into_keys() { + self.cancel(fiber_id); } - }; - - if let Some(result) = result_of_parallel { - self.channels - .remove(&nursery) - .unwrap() - .to_nursery() - .unwrap(); - let Parallel { - mut paused_fiber, .. - } = self - .fibers - .remove(&parent_id) - .unwrap() - .into_parallel() - .unwrap(); - paused_fiber.fiber.complete_parallel_scope(result); - self.fibers - .insert(parent_id, FiberTree::Single(paused_fiber)); + Some(Err(panic_reason)) } - } - FiberTree::Try(Try { .. }) => { - let Try { + }; + + if let Some(result) = result_of_parallel { + self.channels + .remove(&nursery) + .unwrap() + .to_nursery() + .unwrap(); + let Parallel { mut paused_fiber, .. - } = self.fibers.remove(&parent_id).unwrap().into_try().unwrap(); - paused_fiber - .fiber - .complete_try(result.map(|value| Packet { heap, value })); + } = self + .fibers + .remove(&parent_id) + .unwrap() + .into_parallel() + .unwrap(); + paused_fiber.fiber.complete_parallel_scope(result); + self.fibers + .insert(parent_id, FiberTree::Single(paused_fiber)); } } + FiberTree::Try(Try { .. }) => { + let Try { + mut paused_fiber, .. + } = self.fibers.remove(&parent_id).unwrap().into_try().unwrap(); + paused_fiber + .fiber + .complete_try(result.map(|value| Packet { heap, value })); + } } } } @@ -430,8 +429,8 @@ impl Vm { .unwrap() .to_nursery() .unwrap(); - for child in children { - self.cancel(child.fiber); + for child_fiber in children.keys() { + self.cancel(*child_fiber); } } FiberTree::Try(Try { child, .. }) => self.cancel(child), @@ -480,10 +479,9 @@ impl Vm { .unwrap() .as_parallel_mut() .unwrap(); - parent.children.push(Child { - fiber: child_id, - return_channel: Some(return_channel), - }); + parent + .children + .insert(child_id, ChildKind::SpawnedChild(return_channel)); InternalChannel::complete_send(&mut self.fibers, performing_fiber); } @@ -683,9 +681,12 @@ impl fmt::Debug for FiberTree { } } } -impl fmt::Debug for Child { +impl fmt::Debug for ChildKind { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?} returning to {:?}", self.fiber, self.return_channel) + match self { + ChildKind::InitialChild => write!(f, "is initial child"), + ChildKind::SpawnedChild(return_channel) => write!(f, "returns to {:?}", return_channel), + } } } impl fmt::Debug for Channel { From da2400aa5ea60267c0a8aa5c6a800e25576a2aa2 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sun, 11 Sep 2022 23:19:23 +0200 Subject: [PATCH 39/59] Improve code quality --- compiler/src/vm/mod.rs | 100 +++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 59 deletions(-) diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 60f084c0f..cc6f35662 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -19,6 +19,7 @@ use rand::seq::SliceRandom; use std::{ collections::{HashMap, VecDeque}, fmt, + hash::Hash, marker::PhantomData, }; use tracing::{info, warn}; @@ -148,13 +149,9 @@ impl Vm { Self::new_with_fiber(Fiber::new_for_running_module_closure(closure)) } pub fn tear_down(mut self) -> TearDownResult { - let fiber = self - .fibers - .remove(&self.root_fiber) - .unwrap() - .into_single() - .unwrap(); - fiber.fiber.tear_down() + let tree = self.fibers.remove(&self.root_fiber).unwrap(); + let single = tree.into_single().unwrap(); + single.fiber.tear_down() } pub fn status(&self) -> Status { @@ -280,24 +277,17 @@ impl Vm { id }; - // TODO: Create utility method for removing and adding under same ID. - let paused_fiber = self - .fibers - .remove(&fiber_id) - .unwrap() - .into_single() - .unwrap(); - self.fibers.insert( - fiber_id, + self.fibers.replace(fiber_id, |tree| { + let single = tree.into_single().unwrap(); FiberTree::Parallel(Parallel { - paused_fiber, + paused_fiber: single, children: [(first_child_id, ChildKind::InitialChild)] .into_iter() .collect(), return_value: None, nursery: nursery_id, - }), - ); + }) + }); false } @@ -316,19 +306,13 @@ impl Vm { id }; - let paused_fiber = self - .fibers - .remove(&fiber_id) - .unwrap() - .into_single() - .unwrap(); - self.fibers.insert( - fiber_id, + self.fibers.replace(fiber_id, |tree| { + let single = tree.into_single().unwrap(); FiberTree::Try(Try { - paused_fiber, + paused_fiber: single, child: child_id, - }), - ); + }) + }); false } @@ -343,12 +327,8 @@ impl Vm { }; if is_finished && fiber_id != self.root_fiber { - let single = self - .fibers - .remove(&fiber_id) - .unwrap() - .into_single() - .unwrap(); + let tree = self.fibers.remove(&fiber_id).unwrap(); + let single = tree.into_single().unwrap(); let TearDownResult { heap, result, .. } = single.fiber.tear_down(); let parent_id = single @@ -389,31 +369,22 @@ impl Vm { }; if let Some(result) = result_of_parallel { - self.channels - .remove(&nursery) - .unwrap() - .to_nursery() - .unwrap(); - let Parallel { - mut paused_fiber, .. - } = self - .fibers - .remove(&parent_id) - .unwrap() - .into_parallel() - .unwrap(); - paused_fiber.fiber.complete_parallel_scope(result); - self.fibers - .insert(parent_id, FiberTree::Single(paused_fiber)); + self.channels.remove(&nursery).unwrap(); + self.fibers.replace(parent_id, |tree| { + let mut paused_fiber = tree.into_parallel().unwrap().paused_fiber; + paused_fiber.fiber.complete_parallel_scope(result); + FiberTree::Single(paused_fiber) + }); } } FiberTree::Try(Try { .. }) => { - let Try { - mut paused_fiber, .. - } = self.fibers.remove(&parent_id).unwrap().into_try().unwrap(); - paused_fiber - .fiber - .complete_try(result.map(|value| Packet { heap, value })); + self.fibers.replace(parent_id, |tree| { + let mut paused_fiber = tree.into_try().unwrap().paused_fiber; + paused_fiber + .fiber + .complete_try(result.map(|value| Packet { heap, value })); + FiberTree::Single(paused_fiber) + }); } } } @@ -460,7 +431,7 @@ impl Vm { match Self::parse_spawn_packet(packet) { Some(it) => it, None => { - // The nursery received an invalid message. TODO: Panic. + // The nursery received an invalid message. TODO: Handle this. panic!("A nursery received an invalid message."); } }; @@ -757,3 +728,14 @@ impl> IdGenerator { id.into() } } + +trait ReplaceHashMapValue { + fn replace V>(&mut self, key: K, replacer: F); +} +impl ReplaceHashMapValue for HashMap { + fn replace V>(&mut self, key: K, replacer: F) { + let value = self.remove(&key).unwrap(); + let value = replacer(value); + self.insert(key, value); + } +} From 2d15b869b049200dc24275abf3b6400bd6c38759 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 12 Sep 2022 22:39:20 +0200 Subject: [PATCH 40/59] Implement external channels --- compiler/src/fuzzer/fuzzer.rs | 5 +- compiler/src/fuzzer/mod.rs | 4 +- .../hints/constant_evaluator.rs | 4 +- compiler/src/main.rs | 33 +++- compiler/src/vm/builtin_functions.rs | 2 +- compiler/src/vm/fiber.rs | 13 +- compiler/src/vm/heap/mod.rs | 19 +- compiler/src/vm/heap/object.rs | 12 +- compiler/src/vm/mod.rs | 177 ++++++++++++------ 9 files changed, 195 insertions(+), 74 deletions(-) diff --git a/compiler/src/fuzzer/fuzzer.rs b/compiler/src/fuzzer/fuzzer.rs index 1b257009b..ffcb41ac1 100644 --- a/compiler/src/fuzzer/fuzzer.rs +++ b/compiler/src/fuzzer/fuzzer.rs @@ -2,7 +2,7 @@ use super::{generator::generate_n_values, utils::did_need_in_closure_cause_panic use crate::{ compiler::hir, database::Database, - vm::{context::Context, tracer::Tracer, Closure, Vm, Heap, Pointer, self}, + vm::{self, context::Context, tracer::Tracer, Closure, Heap, Pointer, Vm}, }; use std::mem; @@ -42,7 +42,8 @@ impl Status { let closure = closure_heap.clone_single_to_other_heap(&mut vm_heap, closure); let arguments = generate_n_values(&mut vm_heap, num_args); - let vm = Vm::new_for_running_closure(vm_heap, closure, &arguments); + let mut vm = Vm::new(); + vm.set_up_for_running_closure(vm_heap, closure, &arguments); Status::StillFuzzing { vm, arguments } } diff --git a/compiler/src/fuzzer/mod.rs b/compiler/src/fuzzer/mod.rs index 8ad87c9a0..ded5cbb9e 100644 --- a/compiler/src/fuzzer/mod.rs +++ b/compiler/src/fuzzer/mod.rs @@ -16,8 +16,8 @@ use tracing::info; pub async fn fuzz(db: &Database, module: Module) { let (fuzzables_heap, fuzzables) = { - let mut vm = - Vm::new_for_running_module_closure(Closure::of_module(db, module.clone()).unwrap()); + let mut vm = Vm::new(); + vm.set_up_for_running_module_closure(Closure::of_module(db, module.clone()).unwrap()); vm.run(&mut ModularContext { use_provider: DbUseProvider { db }, execution_controller: RunForever, diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index feeb49e82..8bfeea0d9 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -28,8 +28,8 @@ pub struct ConstantEvaluator { impl ConstantEvaluator { pub fn update_module(&mut self, db: &Database, module: Module) { - let vm = - Vm::new_for_running_module_closure(Closure::of_module(db, module.clone()).unwrap()); + let mut vm = Vm::new(); + vm.set_up_for_running_module_closure(Closure::of_module(db, module.clone()).unwrap()); self.vms.insert(module, vm); } diff --git a/compiler/src/main.rs b/compiler/src/main.rs index a09208010..8234f9aa6 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -36,7 +36,6 @@ use itertools::Itertools; use language_server::CandyLanguageServer; use notify::{watcher, RecursiveMode, Watcher}; use std::{ - collections::HashMap, convert::TryInto, env::current_dir, path::PathBuf, @@ -206,7 +205,8 @@ fn run(options: CandyRunOptions) { let path_string = options.file.to_string_lossy(); info!("Running `{path_string}`."); - let mut vm = Vm::new_for_running_module_closure(module_closure); + let mut vm = Vm::new(); + vm.set_up_for_running_module_closure(module_closure); loop { info!("Tree: {:#?}", vm); match vm.status() { @@ -263,8 +263,14 @@ fn run(options: CandyRunOptions) { info!("Running main function."); // TODO: Add environment stuff. - let environment = heap.create_struct(HashMap::new()); - let mut vm = Vm::new_for_running_closure(heap, main, &[environment]); + let mut vm = Vm::new(); + let stdout = vm.create_channel(); + let environment = { + let stdout_symbol = heap.create_symbol("Stdout".to_string()); + let stdout_port = heap.create_send_port(stdout); + heap.create_struct([(stdout_symbol, stdout_port)].into_iter().collect()) + }; + vm.set_up_for_running_closure(heap, main, &[environment]); loop { info!("Tree: {:#?}", vm); match vm.status() { @@ -281,6 +287,25 @@ fn run(options: CandyRunOptions) { } _ => break, } + let stdout_operations = vm + .external_operations + .get_mut(&stdout) + .unwrap() + .drain(..) + .collect_vec(); + for operation in stdout_operations { + match operation { + vm::Operation::Send { + performing_fiber, + packet, + } => { + info!("Sent to stdout: {}", packet.value.format(&packet.heap)); + vm.complete_send(performing_fiber); + } + vm::Operation::Receive { .. } => unreachable!(), + vm::Operation::Drop => vm.free_channel(stdout), + } + } } info!("Tree: {:#?}", vm); let TearDownResult { diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index c00d837d2..dc51cce76 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -72,7 +72,7 @@ impl Fiber { Ok(Receive { channel }) => self.status = Status::Receiving { channel }, Ok(Parallel { body }) => self.status = Status::InParallelScope { body }, Ok(Try { body }) => self.status = Status::InTry { body }, - Err(reason) => self.panic(reason), + Err(reason) => self.status = Status::Panicked { reason }, } } } diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index dfb0e2af9..34fc5a2c5 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -117,6 +117,7 @@ impl Fiber { let closure = heap.create_closure(closure); Self::new_for_running_closure(heap, closure, &[]) } + pub fn tear_down(mut self) -> TearDownResult { let result = match self.status { Status::Done => Ok(self.data_stack.pop().unwrap()), @@ -134,9 +135,12 @@ impl Fiber { pub fn status(&self) -> Status { self.status.clone() } - pub fn panic(&mut self, reason: String) { - self.status = Status::Panicked { reason }; - } + + // If the status of this fiber is something else than `Status::Running` + // after running, then the VM that manages this fiber is expected to perform + // some action and to then call the corresponding `complete_*` method before + // calling `run` again. + pub fn complete_channel_create(&mut self, channel: ChannelId) { let send_port = self.heap.create_send_port(channel); let receive_port = self.heap.create_receive_port(channel); @@ -187,6 +191,9 @@ impl Fiber { fn get_from_data_stack(&self, offset: usize) -> Pointer { self.data_stack[self.data_stack.len() - 1 - offset as usize] } + fn panic(&mut self, reason: String) { + self.status = Status::Panicked { reason }; + } pub fn run(&mut self, context: &mut C) { assert!( diff --git a/compiler/src/vm/heap/mod.rs b/compiler/src/vm/heap/mod.rs index 836ccc0e2..f0269bb85 100644 --- a/compiler/src/vm/heap/mod.rs +++ b/compiler/src/vm/heap/mod.rs @@ -10,7 +10,10 @@ pub use self::{ use crate::builtin_functions::BuiltinFunction; use itertools::Itertools; use num_bigint::BigInt; -use std::{cmp::Ordering, collections::HashMap}; +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, +}; const TRACE: bool = false; @@ -92,6 +95,7 @@ impl Heap { self.get(address).reference_count - 1, address.format(self), ); + let object = self.get_mut(address); object.reference_count -= 1; if object.reference_count == 0 { @@ -200,11 +204,14 @@ impl Heap { .unwrap() } - pub fn all_objects(&self) -> &HashMap { - &self.objects - } - pub fn all_objects_mut(&mut self) -> &mut HashMap { - &mut self.objects + pub fn known_channels(&self) -> HashSet { + let mut known = HashSet::new(); + for object in self.objects.values() { + if let Some(channel) = object.data.channel() { + known.insert(channel); + } + } + known } pub fn create_int(&mut self, int: BigInt) -> Pointer { diff --git a/compiler/src/vm/heap/object.rs b/compiler/src/vm/heap/object.rs index 7bd041047..bae25b248 100644 --- a/compiler/src/vm/heap/object.rs +++ b/compiler/src/vm/heap/object.rs @@ -12,9 +12,9 @@ use itertools::Itertools; use num_bigint::BigInt; use std::{ collections::{hash_map::DefaultHasher, HashMap}, + fmt, hash::{Hash, Hasher}, ops::Deref, - fmt, }; #[derive(Clone)] @@ -271,6 +271,16 @@ impl Data { Data::Closure(closure) => closure.captured.clone(), } } + + pub fn channel(&self) -> Option { + if let Data::SendPort(SendPort { channel }) | Data::ReceivePort(ReceivePort { channel }) = + self + { + Some(*channel) + } else { + None + } + } } impl Deref for Object { diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index cc6f35662..71464fb63 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -17,7 +17,7 @@ pub use heap::{Closure, Heap, Object, Pointer, Struct}; use itertools::Itertools; use rand::seq::SliceRandom; use std::{ - collections::{HashMap, VecDeque}, + collections::{HashMap, HashSet, VecDeque}, fmt, hash::Hash, marker::PhantomData, @@ -30,7 +30,7 @@ use tracing::{info, warn}; #[derive(Clone)] pub struct Vm { fibers: HashMap, - root_fiber: FiberId, + root_fiber: Option, // only None when no fiber is created yet // TODO: Drop channels channels: HashMap, @@ -41,7 +41,7 @@ pub struct Vm { } #[derive(Clone, Copy, PartialEq, Eq, Hash)] -struct FiberId(usize); +pub struct FiberId(usize); #[derive(Clone)] enum FiberTree { @@ -103,18 +103,16 @@ struct InternalChannel { pending_receives: VecDeque>, } -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -struct OperationId(usize); - #[derive(Clone)] -pub struct Operation { - performing_fiber: Option, - kind: OperationKind, -} -#[derive(Clone, Debug)] -pub enum OperationKind { - Send { packet: Packet }, - Receive, +pub enum Operation { + Send { + performing_fiber: Option, + packet: Packet, + }, + Receive { + performing_fiber: Option, + }, + Drop, } #[derive(Clone, Debug)] @@ -126,36 +124,49 @@ pub enum Status { } impl Vm { - fn new_with_fiber(fiber: Fiber) -> Self { - let fiber = FiberTree::Single(Single { - fiber, - parent: None, - }); - let mut fiber_id_generator = IdGenerator::start_at(0); - let root_fiber_id = fiber_id_generator.generate(); + pub fn new() -> Self { Self { channels: Default::default(), - fibers: [(root_fiber_id, fiber)].into_iter().collect(), - root_fiber: root_fiber_id, + fibers: HashMap::new(), + root_fiber: None, external_operations: Default::default(), channel_id_generator: IdGenerator::start_at(0), - fiber_id_generator, + fiber_id_generator: IdGenerator::start_at(0), } } - pub fn new_for_running_closure(heap: Heap, closure: Pointer, arguments: &[Pointer]) -> Self { - Self::new_with_fiber(Fiber::new_for_running_closure(heap, closure, arguments)) + + fn set_up_with_fiber(&mut self, fiber: Fiber) { + assert!(self.root_fiber.is_none(), "VM already set up"); + let root_fiber_id = self.fiber_id_generator.generate(); + self.fibers.insert( + root_fiber_id, + FiberTree::Single(Single { + fiber, + parent: None, + }), + ); + self.root_fiber = Some(root_fiber_id); + } + pub fn set_up_for_running_closure( + &mut self, + heap: Heap, + closure: Pointer, + arguments: &[Pointer], + ) { + self.set_up_with_fiber(Fiber::new_for_running_closure(heap, closure, arguments)) } - pub fn new_for_running_module_closure(closure: Closure) -> Self { - Self::new_with_fiber(Fiber::new_for_running_module_closure(closure)) + pub fn set_up_for_running_module_closure(&mut self, closure: Closure) { + self.set_up_with_fiber(Fiber::new_for_running_module_closure(closure)) } + pub fn tear_down(mut self) -> TearDownResult { - let tree = self.fibers.remove(&self.root_fiber).unwrap(); + let tree = self.fibers.remove(&self.root_fiber.unwrap()).unwrap(); let single = tree.into_single().unwrap(); single.fiber.tear_down() } pub fn status(&self) -> Status { - self.status_of(self.root_fiber) + self.status_of(self.root_fiber.expect("VM not set up yet")) } fn status_of(&self, fiber: FiberId) -> Status { match &self.fibers[&fiber] { @@ -200,22 +211,46 @@ impl Vm { todo!() } pub fn cloned_tracer(&self) -> Tracer { + // TODO: Remove self.fiber().tracer.clone() } + /// Can be called at any time from outside the VM to create a channel that + /// can be used to communicate with the outside world. pub fn create_channel(&mut self) -> ChannelId { let id = self.channel_id_generator.generate(); self.channels.insert(id, Channel::External); + self.external_operations.insert(id, vec![]); id } + pub fn complete_send(&mut self, performing_fiber: Option) { + if let Some(fiber) = performing_fiber { + let tree = self.fibers.get_mut(&fiber).unwrap(); + tree.as_single_mut().unwrap().fiber.complete_send(); + } + } + + pub fn complete_receive(&mut self, performing_fiber: Option, packet: Packet) { + if let Some(fiber) = performing_fiber { + let tree = self.fibers.get_mut(&fiber).unwrap(); + tree.as_single_mut().unwrap().fiber.complete_receive(packet); + } + } + + /// May only be called if a drop operation was emitted for that channel. + pub fn free_channel(&mut self, channel: ChannelId) { + self.channels.remove(&channel); + self.external_operations.remove(&channel); + } + pub fn run(&mut self, context: &mut C) { assert!( self.is_running(), "Called Vm::run on a VM that is not ready to run." ); - let mut fiber_id = self.root_fiber; + let mut fiber_id = self.root_fiber.unwrap(); let fiber = loop { match self.fibers.get_mut(&fiber_id).unwrap() { FiberTree::Single(Single { fiber, .. }) => break fiber, @@ -326,7 +361,7 @@ impl Vm { } }; - if is_finished && fiber_id != self.root_fiber { + if is_finished && fiber_id != self.root_fiber.unwrap() { let tree = self.fibers.remove(&fiber_id).unwrap(); let single = tree.into_single().unwrap(); let TearDownResult { heap, result, .. } = single.fiber.tear_down(); @@ -388,6 +423,34 @@ impl Vm { } } } + + let all_channels = self.channels.keys().copied().collect::>(); + let mut known_channels = HashSet::new(); + for fiber in self.fibers.values() { + if let Some(single) = fiber.as_single() { + known_channels.extend(single.fiber.heap.known_channels().into_iter()); + } + } + let forgotten_channels = all_channels.difference(&known_channels); + for channel in forgotten_channels { + match self.channels.get(channel).unwrap() { + // If an internal channel is not referenced anymore by any + // fiber, no future reference to it can be obtained in the + // future. Thus, it's safe to remove such channels. + Channel::Internal(_) => { + self.channels.remove(channel); + } + // External channels may be re-sent into the VM from the outside + // even after no fibers remember them. Rather than removing them + // directly, we communicate to the outside that no fiber + // references them anymore. + Channel::External => { + self.push_external_operation(*channel, Operation::Drop); + } + // Nurseries are automatically removed when they are exited. + Channel::Nursery(_) => {} + } + } } fn cancel(&mut self, fiber: FiberId) { match self.fibers.remove(&fiber).unwrap() { @@ -420,9 +483,9 @@ impl Vm { } Channel::External => self.push_external_operation( channel, - Operation { + Operation::Send { performing_fiber, - kind: OperationKind::Send { packet }, + packet, }, ), Channel::Nursery(parent_id) => { @@ -487,13 +550,7 @@ impl Vm { channel.receive(&mut self.fibers, performing_fiber); } Channel::External => { - self.push_external_operation( - channel, - Operation { - performing_fiber, - kind: OperationKind::Receive, - }, - ); + self.push_external_operation(channel, Operation::Receive { performing_fiber }); } Channel::Nursery { .. } => unreachable!("nurseries are only sent stuff"), } @@ -591,6 +648,12 @@ impl FiberTree { _ => None, } } + fn as_single(&self) -> Option<&Single> { + match self { + FiberTree::Single(single) => Some(single), + _ => None, + } + } fn as_single_mut(&mut self) -> Option<&mut Single> { match self { FiberTree::Single(single) => Some(single), @@ -674,15 +737,12 @@ impl fmt::Debug for Channel { "operations", &pending_sends .iter() - .map(|(fiber, packet)| Operation { + .map(|(fiber, packet)| Operation::Send { performing_fiber: *fiber, - kind: OperationKind::Send { - packet: packet.clone(), - }, + packet: packet.clone(), }) - .chain(pending_receives.iter().map(|fiber| Operation { + .chain(pending_receives.iter().map(|fiber| Operation::Receive { performing_fiber: *fiber, - kind: OperationKind::Receive, })) .collect_vec(), ) @@ -694,12 +754,23 @@ impl fmt::Debug for Channel { } impl fmt::Debug for Operation { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if let Some(fiber) = self.performing_fiber { - write!(f, "{:?} ", fiber)?; - } - match &self.kind { - OperationKind::Send { packet } => write!(f, "sending {:?}", packet), - OperationKind::Receive => write!(f, "receiving"), + match &self { + Operation::Send { + performing_fiber, + packet, + } => { + if let Some(fiber) = performing_fiber { + write!(f, "{:?} ", fiber)?; + } + write!(f, "sending {:?}", packet) + } + Operation::Receive { performing_fiber } => { + if let Some(fiber) = performing_fiber { + write!(f, "{:?} ", fiber)?; + } + write!(f, "receiving") + } + Operation::Drop => write!(f, "dropping"), } } } From 030f805816a09b69d5b7b503d69cc49ebca0ba2b Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 12 Sep 2022 23:04:44 +0200 Subject: [PATCH 41/59] Don't panic when sending to a dead nursery --- compiler/src/vm/mod.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 71464fb63..d3b65f69a 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -474,15 +474,23 @@ impl Vm { fn send_to_channel( &mut self, performing_fiber: Option, - channel: ChannelId, + channel_id: ChannelId, packet: Packet, ) { - match self.channels.get_mut(&channel).unwrap() { + let channel = match self.channels.get_mut(&channel_id) { + Some(channel) => channel, + None => { + // The channel was a nursery which is now dead. + InternalChannel::complete_send(&mut self.fibers, performing_fiber); + return; + } + }; + match channel { Channel::Internal(channel) => { channel.send(&mut self.fibers, performing_fiber, packet); } Channel::External => self.push_external_operation( - channel, + channel_id, Operation::Send { performing_fiber, packet, From 26d6daadd0c668f45666ff47d72f5d9b487fbd55 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 22 Sep 2022 22:03:15 +0200 Subject: [PATCH 42/59] Make it an error to sent to a dead nursery --- compiler/src/vm/mod.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index d3b65f69a..894b21372 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -480,8 +480,14 @@ impl Vm { let channel = match self.channels.get_mut(&channel_id) { Some(channel) => channel, None => { - // The channel was a nursery which is now dead. - InternalChannel::complete_send(&mut self.fibers, performing_fiber); + // The channel was a nursery that died. + if let Some(fiber) = performing_fiber { + let tree = self.fibers.get_mut(&fiber).unwrap(); + tree.as_single_mut().unwrap().fiber.panic( + "the nursery is already dead because the parallel section ended" + .to_string(), + ); + } return; } }; From 0db9b4d2bae65d04e0d3004e362757cd20e688aa Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sat, 24 Sep 2022 16:44:34 +0200 Subject: [PATCH 43/59] Clean up tracer --- compiler/src/fuzzer/mod.rs | 2 +- compiler/src/fuzzer/utils.rs | 6 +- .../hints/constant_evaluator.rs | 77 ++-- compiler/src/language_server/hints/fuzzer.rs | 16 +- compiler/src/main.rs | 2 +- compiler/src/vm/fiber.rs | 24 +- compiler/src/vm/tracer.rs | 428 ++++++++++++++---- 7 files changed, 398 insertions(+), 157 deletions(-) diff --git a/compiler/src/fuzzer/mod.rs b/compiler/src/fuzzer/mod.rs index ded5cbb9e..e2042d072 100644 --- a/compiler/src/fuzzer/mod.rs +++ b/compiler/src/fuzzer/mod.rs @@ -59,7 +59,7 @@ pub async fn fuzz(db: &Database, module: Module) { info!("This was the stack trace:"); tracer.dump_stack_trace(db, heap); - module.dump_associated_debug_file("trace", &tracer.format_call_tree(heap)); + module.dump_associated_debug_file("trace", &tracer.full_trace().format(&heap)); } } } diff --git a/compiler/src/fuzzer/utils.rs b/compiler/src/fuzzer/utils.rs index f2247c6ae..a25bbfd22 100644 --- a/compiler/src/fuzzer/utils.rs +++ b/compiler/src/fuzzer/utils.rs @@ -1,7 +1,7 @@ use crate::{ compiler::hir::{self, Expression, HirDb, Lambda}, database::Database, - vm::tracer::{TraceEntry, Tracer}, + vm::tracer::{EventData, Tracer}, }; pub fn did_need_in_closure_cause_panic( @@ -9,7 +9,7 @@ pub fn did_need_in_closure_cause_panic( closure_id: &hir::Id, tracer: &Tracer, ) -> bool { - let entry = if let Some(entry) = tracer.log().last() { + let entry = if let Some(entry) = tracer.events.last() { entry } else { // The only way there's no trace log before the panic is when there's an @@ -17,7 +17,7 @@ pub fn did_need_in_closure_cause_panic( // LIR. That's also definitely the fault of the function. return false; }; - if let TraceEntry::NeedsStarted { id, .. } = entry { + if let EventData::NeedsStarted { id, .. } = &entry.data { let mut id = id.parent().unwrap(); loop { if &id == closure_id { diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index 8bfeea0d9..93d57156a 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -12,7 +12,7 @@ use crate::{ vm::{ self, context::{DbUseProvider, ModularContext, RunLimitedNumberOfInstructions}, - tracer::TraceEntry, + tracer::{EventData, StackEntry}, Closure, Heap, Pointer, Vm, }, }; @@ -87,43 +87,38 @@ impl ConstantEvaluator { } }; if module.to_possible_paths().is_some() { - module.dump_associated_debug_file( - "trace", - &vm.fiber().tracer.format_call_tree(&vm.fiber().heap), - ); + let fiber = vm.fiber(); + module + .dump_associated_debug_file("trace", &fiber.tracer.format_full_trace(&fiber.heap)); } - for entry in vm.cloned_tracer().log() { - let (id, value) = match entry { - TraceEntry::ValueEvaluated { id, value } => { - if &id.module != module { - continue; - } - let ast_id = match db.hir_to_ast_id(id.clone()) { - Some(ast_id) => ast_id, - None => continue, - }; - let ast = match db.ast(module.clone()) { - Some((ast, _)) => (*ast).clone(), - None => continue, - }; - let ast = match ast.find(&ast_id) { - Some(ast) => ast, - None => continue, - }; - if !matches!(ast.kind, AstKind::Assignment(_)) { - continue; - } - (id.clone(), value) + for entry in &vm.cloned_tracer().events { + if let EventData::ValueEvaluated { id, value } = &entry.data { + if &id.module != module { + continue; + } + let ast_id = match db.hir_to_ast_id(id.clone()) { + Some(ast_id) => ast_id, + None => continue, + }; + let ast = match db.ast(module.clone()) { + Some((ast, _)) => (*ast).clone(), + None => continue, + }; + let ast = match ast.find(&ast_id) { + Some(ast) => ast, + None => continue, + }; + if !matches!(ast.kind, AstKind::Assignment(_)) { + continue; } - _ => continue, - }; - hints.push(Hint { - kind: HintKind::Value, - text: value.format(&vm.fiber().heap), - position: id_to_end_of_line(db, id).unwrap(), - }); + hints.push(Hint { + kind: HintKind::Value, + text: value.format(&vm.fiber().heap), + position: id_to_end_of_line(db, id.clone()).unwrap(), + }); + } } hints @@ -134,17 +129,17 @@ fn panic_hint(db: &Database, module: Module, vm: &Vm, reason: String) -> Option< // We want to show the hint at the last call site still inside the current // module. If there is no call site in this module, then the panic results // from a compiler error in a previous stage which is already reported. - let stack = vm.fiber().tracer.stack(); + let stack = vm.fiber().tracer.stack_trace(); if stack.len() == 1 { - // The stack only contains a `ModuleStarted` entry. This indicates an - // error during compilation resulting in a top-level error instruction. + // The stack only contains an `InModule` entry. This indicates an error + // during compilation resulting in a top-level error instruction. return None; } let last_call_in_this_module = stack.iter().find(|entry| { let id = match entry { - TraceEntry::CallStarted { id, .. } => id, - TraceEntry::NeedsStarted { id, .. } => id, + StackEntry::Call { id, .. } => id, + StackEntry::Needs { id, .. } => id, _ => return false, }; // Make sure the entry comes from the same file and is not generated @@ -153,7 +148,7 @@ fn panic_hint(db: &Database, module: Module, vm: &Vm, reason: String) -> Option< })?; let (id, call_info) = match last_call_in_this_module { - TraceEntry::CallStarted { id, closure, args } => ( + StackEntry::Call { id, closure, args } => ( id, format!( "{} {}", @@ -163,7 +158,7 @@ fn panic_hint(db: &Database, module: Module, vm: &Vm, reason: String) -> Option< .join(" ") ), ), - TraceEntry::NeedsStarted { + StackEntry::Needs { id, condition, reason, diff --git a/compiler/src/language_server/hints/fuzzer.rs b/compiler/src/language_server/hints/fuzzer.rs index 0c1b10e82..8193338f8 100644 --- a/compiler/src/language_server/hints/fuzzer.rs +++ b/compiler/src/language_server/hints/fuzzer.rs @@ -9,7 +9,7 @@ use crate::{ module::Module, vm::{ context::{DbUseProvider, ModularContext, RunLimitedNumberOfInstructions}, - tracer::TraceEntry, + tracer::EventData, Heap, Pointer, }, }; @@ -112,15 +112,15 @@ impl FuzzerManager { let second_hint = { let panicking_inner_call = tracer - .log() + .events .iter() .rev() // Find the innermost panicking call that is in the // function. .find(|entry| { - let innermost_panicking_call_id = match entry { - TraceEntry::CallStarted { id, .. } => id, - TraceEntry::NeedsStarted { id, .. } => id, + let innermost_panicking_call_id = match &entry.data { + EventData::CallStarted { id, .. } => id, + EventData::NeedsStarted { id, .. } => id, _ => return false, }; id.is_same_module_and_any_parent_of(innermost_panicking_call_id) @@ -135,11 +135,11 @@ impl FuzzerManager { continue; } }; - let (call_id, name, arguments) = match panicking_inner_call { - TraceEntry::CallStarted { id, closure, args } => { + let (call_id, name, arguments) = match &panicking_inner_call.data { + EventData::CallStarted { id, closure, args } => { (id.clone(), closure.format(heap), args.clone()) } - TraceEntry::NeedsStarted { + EventData::NeedsStarted { id, condition, reason, diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 8234f9aa6..89a8e317c 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -233,7 +233,7 @@ fn run(options: CandyRunOptions) { } = vm.tear_down(); if options.debug { - module.dump_associated_debug_file("trace", &tracer.format_call_tree(&heap)); + module.dump_associated_debug_file("trace", &tracer.full_trace().format(&heap)); } let exported_definitions: Struct = match result { diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index 34fc5a2c5..4ab5ffd6e 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -2,7 +2,7 @@ use super::{ channel::{Capacity, Packet}, context::Context, heap::{Builtin, ChannelId, Closure, Data, Heap, Pointer}, - tracer::{TraceEntry, Tracer}, + tracer::{EventData, Tracer}, }; use crate::{ compiler::{hir::Id, lir::Instruction}, @@ -86,7 +86,7 @@ impl Fiber { call_stack: vec![], import_stack: vec![], heap, - tracer: Tracer::default(), + tracer: Tracer::new(), fuzzable_closures: vec![], } } @@ -191,7 +191,7 @@ impl Fiber { fn get_from_data_stack(&self, offset: usize) -> Pointer { self.data_stack[self.data_stack.len() - 1 - offset as usize] } - fn panic(&mut self, reason: String) { + pub fn panic(&mut self, reason: String) { self.status = Status::Panicked { reason }; } @@ -400,7 +400,7 @@ impl Fiber { Instruction::TraceValueEvaluated(id) => { let value = *self.data_stack.last().unwrap(); self.heap.dup(value); - self.tracer.push(TraceEntry::ValueEvaluated { id, value }); + self.tracer.push(EventData::ValueEvaluated { id, value }); } Instruction::TraceCallStarts { id, num_args } => { let closure = *self.data_stack.last().unwrap(); @@ -416,25 +416,29 @@ impl Fiber { args.reverse(); self.tracer - .push(TraceEntry::CallStarted { id, closure, args }); + .push(EventData::CallStarted { id, closure, args }); } Instruction::TraceCallEnds => { let return_value = *self.data_stack.last().unwrap(); self.heap.dup(return_value); - self.tracer.push(TraceEntry::CallEnded { return_value }); + self.tracer.push(EventData::CallEnded { return_value }); } Instruction::TraceNeedsStarts { id } => { let condition = self.data_stack[self.data_stack.len() - 1]; let reason = self.data_stack[self.data_stack.len() - 2]; self.heap.dup(condition); self.heap.dup(reason); - self.tracer.push(TraceEntry::NeedsStarted { + self.tracer.push(EventData::NeedsStarted { id, condition, reason, }); } - Instruction::TraceNeedsEnds => self.tracer.push(TraceEntry::NeedsEnded), + Instruction::TraceNeedsEnds => { + let nothing = *self.data_stack.last().unwrap(); + self.heap.dup(nothing); + self.tracer.push(EventData::NeedsEnded { nothing }); + } Instruction::TraceModuleStarts { module } => { if self.import_stack.contains(&module) { self.panic(format!( @@ -448,13 +452,13 @@ impl Fiber { )); } self.import_stack.push(module.clone()); - self.tracer.push(TraceEntry::ModuleStarted { module }); + self.tracer.push(EventData::ModuleStarted { module }); } Instruction::TraceModuleEnds => { self.import_stack.pop().unwrap(); let export_map = *self.data_stack.last().unwrap(); self.heap.dup(export_map); - self.tracer.push(TraceEntry::ModuleEnded { export_map }) + self.tracer.push(EventData::ModuleEnded { export_map }) } Instruction::Error { id, errors } => { self.panic(format!( diff --git a/compiler/src/vm/tracer.rs b/compiler/src/vm/tracer.rs index f94e55a7b..4db3f1d2c 100644 --- a/compiler/src/vm/tracer.rs +++ b/compiler/src/vm/tracer.rs @@ -6,15 +6,22 @@ use crate::{ module::Module, }; use itertools::Itertools; +use std::time::Instant; use tracing::error; -#[derive(Default, Clone)] +#[derive(Clone)] pub struct Tracer { - log: Vec, - stack: Vec, + pub events: Vec, } + #[derive(Clone)] -pub enum TraceEntry { +pub struct Event { + pub when: Instant, + pub data: EventData, +} + +#[derive(Clone)] +pub enum EventData { ValueEvaluated { id: Id, value: Pointer, @@ -32,7 +39,17 @@ pub enum TraceEntry { condition: Pointer, reason: Pointer, }, - NeedsEnded, + NeedsEnded { + nothing: Pointer, + }, + ParallelStarted, + ParallelChildFinished { + tracer: Tracer, + heap: Heap, + }, + ParallelEnded { + return_value: Pointer, + }, ModuleStarted { module: Module, }, @@ -42,56 +59,100 @@ pub enum TraceEntry { } impl Tracer { - pub fn push(&mut self, entry: TraceEntry) { - self.log.push(entry.clone()); - match entry { - TraceEntry::CallStarted { .. } => { - self.stack.push(entry); - } - TraceEntry::CallEnded { .. } => { - self.stack.pop().unwrap(); - } - TraceEntry::NeedsStarted { .. } => { - self.stack.push(entry); - } - TraceEntry::NeedsEnded => { - self.stack.pop().unwrap(); - } - TraceEntry::ModuleStarted { .. } => { - self.stack.push(entry); - } - TraceEntry::ModuleEnded { .. } => { - self.stack.pop().unwrap(); - } - _ => {} - } - } - pub fn log(&self) -> &[TraceEntry] { - &self.log + pub fn push(&mut self, data: EventData) { + self.events.push(Event { + when: Instant::now(), + data, + }); } +} + +// Stack traces are a reduced view of the tracing state that represent the stack +// trace at a given moment in time. + +#[derive(Clone)] +pub enum StackEntry { + Call { + id: Id, + closure: Pointer, + args: Vec, + }, + Needs { + id: Id, + condition: Pointer, + reason: Pointer, + }, + Parallel { + children: Vec<(Tracer, Heap)>, + }, + Module { + module: Module, + }, +} - pub fn stack(&self) -> &[TraceEntry] { - &self.stack +impl Tracer { + pub fn new() -> Self { + Self { events: vec![] } } - pub fn dump_stack_trace(&self, db: &Database, heap: &Heap) { - for line in self.format_stack_trace(db, heap).lines() { - error!("{}", line); + pub fn stack_trace(&self) -> Vec { + let mut stack = vec![]; + for Event { data, .. } in &self.events { + match data.clone() { + EventData::ValueEvaluated { .. } => {} + EventData::CallStarted { id, closure, args } => { + stack.push(StackEntry::Call { id, closure, args }) + } + EventData::CallEnded { .. } => { + stack.pop().unwrap(); + } + EventData::NeedsStarted { + id, + condition, + reason, + } => stack.push(StackEntry::Needs { + id, + condition, + reason, + }), + EventData::NeedsEnded { .. } => { + stack.pop().unwrap(); + } + EventData::ParallelStarted => { + stack.push(StackEntry::Parallel { children: vec![] }); + } + EventData::ParallelChildFinished { tracer, heap } => { + let entry = stack.last_mut().unwrap(); + let children = match entry { + StackEntry::Parallel { children } => children, + _ => unreachable!(), + }; + children.push((tracer, heap)); + } + EventData::ParallelEnded { .. } => { + stack.pop().unwrap(); + } + EventData::ModuleStarted { module } => stack.push(StackEntry::Module { module }), + EventData::ModuleEnded { .. } => { + stack.pop().unwrap(); + } + } } + stack } pub fn format_stack_trace(&self, db: &Database, heap: &Heap) -> String { - self.stack + self.stack_trace() .iter() .rev() .map(|entry| { let (call_string, hir_id) = match entry { - TraceEntry::CallStarted { id, closure, args } => ( + StackEntry::Call { id, closure, args } => ( format!( "{closure} {}", args.iter().map(|arg| arg.format(heap)).join(" ") ), Some(id), ), - TraceEntry::NeedsStarted { + StackEntry::Needs { id, condition, reason, @@ -99,8 +160,11 @@ impl Tracer { format!("needs {} {}", condition.format(heap), reason.format(heap)), Some(id), ), - TraceEntry::ModuleStarted { module } => (format!("use {module}"), None), - _ => unreachable!(), + StackEntry::Parallel { .. } => { + (format!("parallel section (todo: format children)"), None) + // TODO: format + } + StackEntry::Module { module } => (format!("use {module}"), None), }; let caller_location_string = { let (hir_id, ast_id, cst_id, span) = if let Some(hir_id) = hir_id { @@ -140,67 +204,245 @@ impl Tracer { }) .join("\n") } + pub fn dump_stack_trace(&self, db: &Database, heap: &Heap) { + for line in self.format_stack_trace(db, heap).lines() { + error!("{}", line); + } + } +} - pub fn format_call_tree(&self, heap: &Heap) -> String { - let actions = self - .log - .iter() - .map(|entry| match entry { - TraceEntry::ValueEvaluated { id, value } => { - Action::Stay(format!("{id} = {}", value.format(heap))) - } - TraceEntry::CallStarted { id, closure, args } => Action::Start(format!( - "{id} {} {}", - closure.format(heap), - args.iter().map(|arg| arg.format(heap)).join(" ") - )), - TraceEntry::CallEnded { return_value } => { - Action::End(format!(" = {}", return_value.format(heap))) - } - TraceEntry::NeedsStarted { +// Full traces are a computed tree view of the whole execution. + +pub struct Trace { + start: Instant, + end: Instant, + data: TraceData, +} +pub enum TraceData { + Call { + id: Id, + closure: Pointer, + args: Vec, + inner: Vec, + result: TraceResult, + }, + Needs { + id: Id, + condition: Pointer, + reason: Pointer, + result: TraceResult, + }, + Parallel { + id: Id, + children: Vec, + result: TraceResult, + }, + Module { + module: Module, + inner: Vec, + result: TraceResult, + }, +} +pub enum TraceResult { + Returned(Pointer), + Panicked(Pointer), + Canceled, +} + +impl Tracer { + pub fn full_trace(&self) -> Trace { + let mut stack = vec![Span { + start: self + .events + .first() + .map(|event| event.when) + .unwrap_or_else(Instant::now), + data: None, + inner: vec![], + }]; + for event in &self.events { + match &event.data { + EventData::ValueEvaluated { .. } => {} + EventData::CallStarted { id, closure, args } => { + stack.push(Span { + start: event.when, + data: Some(StackEntry::Call { + id: id.clone(), + closure: *closure, + args: args.clone(), + }), + inner: vec![], + }); + } + EventData::CallEnded { return_value } => { + let span = stack.pop().unwrap(); + let (id, closure, args) = match span.data.unwrap() { + StackEntry::Call { id, closure, args } => (id, closure, args), + StackEntry::Needs { .. } => unreachable!(), + StackEntry::Module { .. } => unreachable!(), + }; + stack.last_mut().unwrap().inner.push(Trace { + start: span.start, + end: event.when, + data: TraceData::Call { + id, + closure, + args, + inner: span.inner, + result: TraceResult::Returned(*return_value), + }, + }); + } + EventData::NeedsStarted { id, condition, reason, - } => Action::Start(format!( - "{id} needs {} {}", - condition.format(heap), - reason.format(heap) - )), - TraceEntry::NeedsEnded => Action::End(" = Nothing".to_string()), - TraceEntry::ModuleStarted { module } => Action::Start(format!("module {module}")), - TraceEntry::ModuleEnded { export_map } => Action::End(export_map.format(heap)), - }) - .collect_vec(); - - let mut lines = vec![]; - let mut stack = vec![]; - let mut indentation = 0; - - for action in actions { - let indent = " ".repeat(indentation); - match action { - Action::Start(line) => { - stack.push(lines.len()); - lines.push(format!("{indent}{line}")); - indentation += 1; - } - Action::End(addendum) => { - let start = stack.pop().unwrap(); - lines[start].push_str(&addendum); - indentation -= 1; - } - Action::Stay(line) => { - lines.push(format!("{indent}{line}")); + } => { + stack.push(Span { + start: event.when, + data: Some(StackEntry::Needs { + id: id.clone(), + condition: *condition, + reason: *reason, + }), + inner: vec![], + }); + } + EventData::NeedsEnded { nothing } => { + let span = stack.pop().unwrap(); + let (id, condition, reason) = match span.data.unwrap() { + StackEntry::Needs { + id, + condition, + reason, + } => (id, condition, reason), + _ => unreachable!(), + }; + stack.last_mut().unwrap().inner.push(Trace { + start: span.start, + end: event.when, + data: TraceData::Needs { + id, + condition, + reason, + result: TraceResult::Returned(*nothing), + }, + }); + } + EventData::ParallelStarted => {} + EventData::ParallelEnded { return_value } => { + let span = stack.pop().unwrap(); + let children = match span.data.unwrap() { + StackEntry::Parallel { children } => children, + _ => unreachable!(), + }; + stack.last_mut().unwrap().inner.push(Trace { + start: span.start, + end: event.when, + data: TraceData::Parallel { + id, + children, + result: TraceResult::Returned(return_value), + }, + }); + } + EventData::ModuleStarted { module } => { + stack.push(Span { + start: event.when, + data: Some(StackEntry::Module { + module: module.clone(), + }), + inner: vec![], + }); + } + EventData::ModuleEnded { export_map } => { + let span = stack.pop().unwrap(); + let module = match span.data.unwrap() { + StackEntry::Call { .. } => unreachable!(), + StackEntry::Needs { .. } => unreachable!(), + StackEntry::Module { module } => module, + }; + stack.last_mut().unwrap().inner.push(Trace { + start: span.start, + end: event.when, + data: TraceData::Module { + module, + inner: span.inner, + result: TraceResult::Returned(*export_map), + }, + }); } } } + stack.pop().unwrap().inner.pop().unwrap() // TODO: handle multiple traces + } + pub fn format_full_trace(&self, heap: &Heap) -> String { + self.full_trace().format(heap) + } +} - lines.join("\n") +struct Span { + start: Instant, + data: Option, + inner: Vec, +} + +impl TraceResult { + fn format(&self, heap: &Heap) -> String { + match self { + TraceResult::Returned(return_value) => return_value.format(heap), + TraceResult::Panicked(panic_value) => { + format!("panicked with {}", panic_value.format(heap)) + } + TraceResult::Canceled => "canceled".to_string(), + } } } -enum Action { - Start(String), - End(String), - Stay(String), +impl Trace { + pub fn format(&self, heap: &Heap) -> String { + let mut lines = vec![]; + match &self.data { + TraceData::Call { + id, + args, + inner, + result, + .. + } => { + lines.push(format!( + "call {id} {} = {}", + args.iter().map(|arg| arg.format(heap)).join(" "), + result.format(heap), + )); + for trace in inner { + lines.extend(trace.format(heap).lines().map(|line| format!(" {line}"))); + } + } + TraceData::Needs { + condition, + reason, + result, + .. + } => { + lines.push(format!( + "needs {} {} = {}", + result.format(heap), + condition.format(heap), + reason.format(heap), + )); + } + TraceData::Module { + module, + inner, + result, + } => { + lines.push(format!("{module} = {}", result.format(heap))); + for trace in inner { + lines.extend(trace.format(heap).lines().map(|line| format!(" {line}"))); + } + } + } + lines.join("\n") + } } From 56707163364e60105093d4ef51601d8c001dc5f3 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sat, 24 Sep 2022 16:57:35 +0200 Subject: [PATCH 44/59] Fix tracer --- compiler/src/vm/fiber.rs | 4 ++-- compiler/src/vm/tracer.rs | 42 +++++++++++---------------------------- 2 files changed, 14 insertions(+), 32 deletions(-) diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index 4ab5ffd6e..5e313b4d1 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -424,8 +424,8 @@ impl Fiber { self.tracer.push(EventData::CallEnded { return_value }); } Instruction::TraceNeedsStarts { id } => { - let condition = self.data_stack[self.data_stack.len() - 1]; - let reason = self.data_stack[self.data_stack.len() - 2]; + let condition = self.data_stack[self.data_stack.len() - 2]; + let reason = self.data_stack[self.data_stack.len() - 1]; self.heap.dup(condition); self.heap.dup(reason); self.tracer.push(EventData::NeedsStarted { diff --git a/compiler/src/vm/tracer.rs b/compiler/src/vm/tracer.rs index 4db3f1d2c..9515ed46b 100644 --- a/compiler/src/vm/tracer.rs +++ b/compiler/src/vm/tracer.rs @@ -43,10 +43,6 @@ pub enum EventData { nothing: Pointer, }, ParallelStarted, - ParallelChildFinished { - tracer: Tracer, - heap: Heap, - }, ParallelEnded { return_value: Pointer, }, @@ -82,9 +78,7 @@ pub enum StackEntry { condition: Pointer, reason: Pointer, }, - Parallel { - children: Vec<(Tracer, Heap)>, - }, + Parallel, Module { module: Module, }, @@ -118,15 +112,7 @@ impl Tracer { stack.pop().unwrap(); } EventData::ParallelStarted => { - stack.push(StackEntry::Parallel { children: vec![] }); - } - EventData::ParallelChildFinished { tracer, heap } => { - let entry = stack.last_mut().unwrap(); - let children = match entry { - StackEntry::Parallel { children } => children, - _ => unreachable!(), - }; - children.push((tracer, heap)); + stack.push(StackEntry::Parallel); } EventData::ParallelEnded { .. } => { stack.pop().unwrap(); @@ -233,8 +219,6 @@ pub enum TraceData { result: TraceResult, }, Parallel { - id: Id, - children: Vec, result: TraceResult, }, Module { @@ -278,8 +262,7 @@ impl Tracer { let span = stack.pop().unwrap(); let (id, closure, args) = match span.data.unwrap() { StackEntry::Call { id, closure, args } => (id, closure, args), - StackEntry::Needs { .. } => unreachable!(), - StackEntry::Module { .. } => unreachable!(), + _ => unreachable!(), }; stack.last_mut().unwrap().inner.push(Trace { start: span.start, @@ -332,17 +315,11 @@ impl Tracer { EventData::ParallelStarted => {} EventData::ParallelEnded { return_value } => { let span = stack.pop().unwrap(); - let children = match span.data.unwrap() { - StackEntry::Parallel { children } => children, - _ => unreachable!(), - }; stack.last_mut().unwrap().inner.push(Trace { start: span.start, end: event.when, data: TraceData::Parallel { - id, - children, - result: TraceResult::Returned(return_value), + result: TraceResult::Returned(*return_value), }, }); } @@ -358,9 +335,8 @@ impl Tracer { EventData::ModuleEnded { export_map } => { let span = stack.pop().unwrap(); let module = match span.data.unwrap() { - StackEntry::Call { .. } => unreachable!(), - StackEntry::Needs { .. } => unreachable!(), StackEntry::Module { module } => module, + _ => unreachable!(), }; stack.last_mut().unwrap().inner.push(Trace { start: span.start, @@ -427,9 +403,15 @@ impl Trace { } => { lines.push(format!( "needs {} {} = {}", - result.format(heap), condition.format(heap), reason.format(heap), + result.format(heap), + )); + } + TraceData::Parallel { result } => { + lines.push(format!( + "parallel section that completed with {}", + result.format(heap), )); } TraceData::Module { From b3706215df3ebb16d9959e001a11de2593f30dbf Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sat, 24 Sep 2022 17:42:44 +0200 Subject: [PATCH 45/59] Make code more readable --- compiler/src/vm/mod.rs | 136 ++++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 64 deletions(-) diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 894b21372..80fd44014 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -199,12 +199,9 @@ impl Vm { FiberTree::Try(Try { child, .. }) => self.status_of(*child), } } - fn is_running(&self) -> bool { + fn can_run(&self) -> bool { matches!(self.status(), Status::CanRun) } - fn is_finished(&self) -> bool { - matches!(self.status(), Status::Done | Status::Panicked { .. }) - } pub fn fiber(&self) -> &Fiber { // TODO: Remove before merging the PR @@ -246,10 +243,11 @@ impl Vm { pub fn run(&mut self, context: &mut C) { assert!( - self.is_running(), + self.can_run(), "Called Vm::run on a VM that is not ready to run." ); + // Choose a random fiber to run. let mut fiber_id = self.root_fiber.unwrap(); let fiber = loop { match self.fibers.get_mut(&fiber_id).unwrap() { @@ -262,9 +260,8 @@ impl Vm { FiberTree::Try(Try { child, .. }) => fiber_id = *child, } }; - if !matches!(fiber.status(), fiber::Status::Running) { - return; + return; // TODO: handle } // TODO: Limit context. @@ -362,20 +359,23 @@ impl Vm { }; if is_finished && fiber_id != self.root_fiber.unwrap() { - let tree = self.fibers.remove(&fiber_id).unwrap(); - let single = tree.into_single().unwrap(); + let single = self + .fibers + .remove(&fiber_id) + .unwrap() + .into_single() + .unwrap(); let TearDownResult { heap, result, .. } = single.fiber.tear_down(); - - let parent_id = single + let parent = single .parent .expect("we already checked we're not the root fiber"); - match self.fibers.get_mut(&parent_id).unwrap() { - FiberTree::Single(_) => unreachable!(), + + match self.fibers.get_mut(&parent).unwrap() { + FiberTree::Single(_) => unreachable!("single fibers can't have children"), FiberTree::Parallel(parallel) => { let child = parallel.children.remove(&fiber_id).unwrap(); - let nursery = parallel.nursery; - let result_of_parallel = match result { + match result { Ok(return_value) => { let is_finished = parallel.children.is_empty(); let packet = Packet { @@ -390,30 +390,14 @@ impl Vm { } if is_finished { - Some(Ok(())) - } else { - None - } - } - Err(panic_reason) => { - for fiber_id in parallel.children.clone().into_keys() { - self.cancel(fiber_id); + self.finish_parallel(parent, Ok(())) } - Some(Err(panic_reason)) } - }; - - if let Some(result) = result_of_parallel { - self.channels.remove(&nursery).unwrap(); - self.fibers.replace(parent_id, |tree| { - let mut paused_fiber = tree.into_parallel().unwrap().paused_fiber; - paused_fiber.fiber.complete_parallel_scope(result); - FiberTree::Single(paused_fiber) - }); + Err(panic_reason) => self.finish_parallel(parent, Err(panic_reason)), } } FiberTree::Try(Try { .. }) => { - self.fibers.replace(parent_id, |tree| { + self.fibers.replace(parent, |tree| { let mut paused_fiber = tree.into_try().unwrap().paused_fiber; paused_fiber .fiber @@ -434,16 +418,17 @@ impl Vm { let forgotten_channels = all_channels.difference(&known_channels); for channel in forgotten_channels { match self.channels.get(channel).unwrap() { - // If an internal channel is not referenced anymore by any - // fiber, no future reference to it can be obtained in the - // future. Thus, it's safe to remove such channels. + // If an internal channel is not referenced by any fiber, no + // reference to it can be obtained in the future. Thus, it's + // safe to remove such channels. Channel::Internal(_) => { self.channels.remove(channel); } // External channels may be re-sent into the VM from the outside // even after no fibers remember them. Rather than removing them // directly, we communicate to the outside that no fiber - // references them anymore. + // references them anymore. The outside can then call + // `free_channel` when it doesn't intend to re-use the channel. Channel::External => { self.push_external_operation(*channel, Operation::Drop); } @@ -452,6 +437,29 @@ impl Vm { } } } + fn finish_parallel(&mut self, parallel_id: FiberId, result: Result<(), String>) { + let parallel = self + .fibers + .get_mut(¶llel_id) + .unwrap() + .as_parallel_mut() + .unwrap(); + + for child_id in parallel.children.clone().into_keys() { + self.cancel(child_id); + } + + self.fibers.replace(parallel_id, |tree| { + let Parallel { + mut paused_fiber, + nursery, + .. + } = tree.into_parallel().unwrap(); + self.channels.remove(&nursery).unwrap(); + paused_fiber.fiber.complete_parallel_scope(result); + FiberTree::Single(paused_fiber) + }); + } fn cancel(&mut self, fiber: FiberId) { match self.fibers.remove(&fiber).unwrap() { FiberTree::Single(_) => {} @@ -504,32 +512,32 @@ impl Vm { ), Channel::Nursery(parent_id) => { info!("Nursery received packet {:?}", packet); - let (heap, closure_to_spawn, return_channel) = - match Self::parse_spawn_packet(packet) { - Some(it) => it, - None => { - // The nursery received an invalid message. TODO: Handle this. - panic!("A nursery received an invalid message."); - } - }; - let child_id = self.fiber_id_generator.generate(); - self.fibers.insert( - child_id, - FiberTree::Single(Single { - fiber: Fiber::new_for_running_closure(heap, closure_to_spawn, &[]), - parent: Some(*parent_id), - }), - ); - - let parent = self - .fibers - .get_mut(parent_id) - .unwrap() - .as_parallel_mut() - .unwrap(); - parent - .children - .insert(child_id, ChildKind::SpawnedChild(return_channel)); + let parent_id = *parent_id; + + match Self::parse_spawn_packet(packet) { + Some((heap, closure_to_spawn, return_channel)) => { + let child_id = self.fiber_id_generator.generate(); + self.fibers.insert( + child_id, + FiberTree::Single(Single { + fiber: Fiber::new_for_running_closure(heap, closure_to_spawn, &[]), + parent: Some(parent_id), + }), + ); + + self.fibers + .get_mut(&parent_id) + .unwrap() + .as_parallel_mut() + .unwrap() + .children + .insert(child_id, ChildKind::SpawnedChild(return_channel)); + } + None => self.finish_parallel( + parent_id, + Err("a nursery received an invalid message".to_string()), + ), + } InternalChannel::complete_send(&mut self.fibers, performing_fiber); } From 9e5a4d89f5f6759d68c12f39fff7b45e8d95d2d3 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 3 Oct 2022 14:50:06 +0200 Subject: [PATCH 46/59] Refactor context --- compiler/src/fuzzer/fuzzer.rs | 30 ++++- compiler/src/fuzzer/mod.rs | 13 +-- .../hints/constant_evaluator.rs | 10 +- compiler/src/language_server/hints/fuzzer.rs | 8 +- compiler/src/main.rs | 13 +-- compiler/src/vm/builtin_functions.rs | 7 +- compiler/src/vm/context.rs | 108 ++++++++---------- compiler/src/vm/fiber.rs | 26 +++-- compiler/src/vm/mod.rs | 32 +++++- compiler/src/vm/tracer.rs | 3 +- compiler/src/vm/use_module.rs | 10 +- 11 files changed, 139 insertions(+), 121 deletions(-) diff --git a/compiler/src/fuzzer/fuzzer.rs b/compiler/src/fuzzer/fuzzer.rs index ffcb41ac1..44ba8620a 100644 --- a/compiler/src/fuzzer/fuzzer.rs +++ b/compiler/src/fuzzer/fuzzer.rs @@ -2,7 +2,12 @@ use super::{generator::generate_n_values, utils::did_need_in_closure_cause_panic use crate::{ compiler::hir, database::Database, - vm::{self, context::Context, tracer::Tracer, Closure, Heap, Pointer, Vm}, + vm::{ + self, + context::{ExecutionController, UseProvider}, + tracer::Tracer, + Closure, Heap, Pointer, Vm, + }, }; use std::mem; @@ -67,18 +72,31 @@ impl Fuzzer { self.status.as_ref().unwrap() } - pub fn run(&mut self, db: &Database, context: &mut C) { + pub fn run( + &mut self, + db: &Database, + use_provider: &mut U, + execution_controller: &mut E, + ) { let mut status = mem::replace(&mut self.status, None).unwrap(); - while matches!(status, Status::StillFuzzing { .. }) && context.should_continue_running() { - status = self.map_status(status, db, context); + while matches!(status, Status::StillFuzzing { .. }) + && execution_controller.should_continue_running() + { + status = self.map_status(status, db, use_provider, execution_controller); } self.status = Some(status); } - fn map_status(&self, status: Status, db: &Database, context: &mut C) -> Status { + fn map_status( + &self, + status: Status, + db: &Database, + use_provider: &mut U, + execution_controller: &mut E, + ) -> Status { match status { Status::StillFuzzing { mut vm, arguments } => match vm.status() { vm::Status::CanRun => { - vm.run(context); + vm.run(use_provider, execution_controller); Status::StillFuzzing { vm, arguments } } vm::Status::WaitingForOperations => panic!("Fuzzing should not have to wait on channel operations because arguments were not channels."), diff --git a/compiler/src/fuzzer/mod.rs b/compiler/src/fuzzer/mod.rs index e2042d072..40d1bf35c 100644 --- a/compiler/src/fuzzer/mod.rs +++ b/compiler/src/fuzzer/mod.rs @@ -7,7 +7,7 @@ use crate::{ database::Database, module::Module, vm::{ - context::{DbUseProvider, ModularContext, RunForever, RunLimitedNumberOfInstructions}, + context::{DbUseProvider, RunForever, RunLimitedNumberOfInstructions}, Closure, Vm, }, }; @@ -18,10 +18,7 @@ pub async fn fuzz(db: &Database, module: Module) { let (fuzzables_heap, fuzzables) = { let mut vm = Vm::new(); vm.set_up_for_running_module_closure(Closure::of_module(db, module.clone()).unwrap()); - vm.run(&mut ModularContext { - use_provider: DbUseProvider { db }, - execution_controller: RunForever, - }); + vm.run(&mut DbUseProvider { db }, &mut RunForever); let result = vm.tear_down(); (result.heap, result.fuzzable_closures) }; @@ -35,10 +32,8 @@ pub async fn fuzz(db: &Database, module: Module) { let mut fuzzer = Fuzzer::new(&fuzzables_heap, closure, id.clone()); fuzzer.run( db, - &mut ModularContext { - use_provider: DbUseProvider { db }, - execution_controller: RunLimitedNumberOfInstructions::new(1000), - }, + &mut DbUseProvider { db }, + &mut RunLimitedNumberOfInstructions::new(1000), ); match fuzzer.status() { Status::StillFuzzing { .. } => {} diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index 93d57156a..9bae4a44c 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -11,7 +11,7 @@ use crate::{ module::Module, vm::{ self, - context::{DbUseProvider, ModularContext, RunLimitedNumberOfInstructions}, + context::{DbUseProvider, RunLimitedNumberOfInstructions}, tracer::{EventData, StackEntry}, Closure, Heap, Pointer, Vm, }, @@ -51,10 +51,10 @@ impl ConstantEvaluator { ); if let Some((module, vm)) = running_vms.choose_mut(&mut thread_rng()) { - vm.run(&mut ModularContext { - use_provider: DbUseProvider { db }, - execution_controller: RunLimitedNumberOfInstructions::new(500), - }); + vm.run( + &mut DbUseProvider { db }, + &mut RunLimitedNumberOfInstructions::new(500), + ); Some(module.clone()) } else { None diff --git a/compiler/src/language_server/hints/fuzzer.rs b/compiler/src/language_server/hints/fuzzer.rs index 8193338f8..b4e33d88b 100644 --- a/compiler/src/language_server/hints/fuzzer.rs +++ b/compiler/src/language_server/hints/fuzzer.rs @@ -8,7 +8,7 @@ use crate::{ fuzzer::{Fuzzer, Status}, module::Module, vm::{ - context::{DbUseProvider, ModularContext, RunLimitedNumberOfInstructions}, + context::{DbUseProvider, RunLimitedNumberOfInstructions}, tracer::EventData, Heap, Pointer, }, @@ -56,10 +56,8 @@ impl FuzzerManager { let fuzzer = running_fuzzers.choose_mut(&mut thread_rng())?; fuzzer.run( db, - &mut ModularContext { - use_provider: DbUseProvider { db }, - execution_controller: RunLimitedNumberOfInstructions::new(100), - }, + &mut DbUseProvider { db }, + &mut RunLimitedNumberOfInstructions::new(100), ); match &fuzzer.status() { diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 89a8e317c..25a456814 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -27,7 +27,7 @@ use crate::{ language_server::utils::LspPositionConversion, module::{Module, ModuleKind}, vm::{ - context::{DbUseProvider, ModularContext, RunForever}, + context::{DbUseProvider, RunForever}, Closure, Status, Struct, TearDownResult, Vm, }, }; @@ -212,11 +212,7 @@ fn run(options: CandyRunOptions) { match vm.status() { Status::CanRun => { debug!("VM still running."); - vm.run(&mut ModularContext { - use_provider: DbUseProvider { db: &db }, - execution_controller: RunForever, - }); - // TODO: handle operations + vm.run(&mut DbUseProvider { db: &db }, &mut RunForever); } Status::WaitingForOperations => { todo!("VM can't proceed until some operations complete."); @@ -276,10 +272,7 @@ fn run(options: CandyRunOptions) { match vm.status() { Status::CanRun => { debug!("VM still running."); - vm.run(&mut ModularContext { - use_provider: DbUseProvider { db: &db }, - execution_controller: RunForever, - }); + vm.run(&mut DbUseProvider { db: &db }, &mut RunForever); // TODO: handle operations } Status::WaitingForOperations => { diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index dc51cce76..50e263a17 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -1,6 +1,6 @@ use super::{ channel::{Capacity, Packet}, - context::Context, + context::PanickingUseProvider, fiber::{Fiber, Status}, heap::{ChannelId, Closure, Data, Int, Pointer, ReceivePort, SendPort, Struct, Symbol, Text}, Heap, @@ -15,9 +15,8 @@ use tracing::{info, span, Level}; use unicode_segmentation::UnicodeSegmentation; impl Fiber { - pub(super) fn run_builtin_function( + pub(super) fn run_builtin_function( &mut self, - context: &C, builtin_function: &BuiltinFunction, args: &[Pointer], ) { @@ -65,7 +64,7 @@ impl Fiber { Ok(Return(value)) => self.data_stack.push(value), Ok(DivergeControlFlow { closure }) => { self.data_stack.push(closure); - self.run_instruction(context, Instruction::Call { num_args: 0 }); + self.run_instruction(&PanickingUseProvider, Instruction::Call { num_args: 0 }); } Ok(CreateChannel { capacity }) => self.status = Status::CreatingChannel { capacity }, Ok(Send { channel, packet }) => self.status = Status::Sending { channel, packet }, diff --git a/compiler/src/vm/context.rs b/compiler/src/vm/context.rs index 0b412b311..971b6c6ec 100644 --- a/compiler/src/vm/context.rs +++ b/compiler/src/vm/context.rs @@ -4,64 +4,26 @@ use crate::{ module::{Module, ModuleDb, ModuleKind}, }; -/// Fibers need a context whenever they want to run some expressions. It's used -/// to parameterize the running of the code over the outside world and effects -/// without bleeding implementation details (like salsa) into the code of the -/// VM itself. -pub trait Context { +// VMs and fibers need some of these traits when they run some expressions. This +// allows parameterizing the running of the code over the outside world and +// effects without bleeding implementation details (such as salsa) into the code +// of the VM. + +pub trait UseProvider { fn use_module(&self, module: Module) -> Result; - fn should_continue_running(&self) -> bool; - fn instruction_executed(&mut self); } pub enum UseResult { Asset(Vec), Code(Lir), } -/// Context that can be used when you want to execute some known instructions -/// that are guaranteed not to import other modules. -pub struct DummyContext; -impl Context for DummyContext { +pub struct PanickingUseProvider; +impl UseProvider for PanickingUseProvider { fn use_module(&self, _: Module) -> Result { - panic!("A dummy context was used for importing a module") - } - - fn should_continue_running(&self) -> bool { - true + panic!() } - - fn instruction_executed(&mut self) {} } -/// The modular context is a version of the `Context` where several sub-tasks -/// are handled in isolation. -pub struct ModularContext { - pub use_provider: U, - pub execution_controller: E, -} -pub trait UseProvider { - fn use_module(&self, module: Module) -> Result; -} -pub trait ExecutionController { - fn should_continue_running(&self) -> bool; - fn instruction_executed(&mut self); -} - -impl Context for ModularContext { - fn use_module(&self, module: Module) -> Result { - self.use_provider.use_module(module) - } - - fn should_continue_running(&self) -> bool { - self.execution_controller.should_continue_running() - } - - fn instruction_executed(&mut self) { - self.execution_controller.instruction_executed() - } -} - -/// Uses a salsa database to import modules. pub struct DbUseProvider<'a> { pub db: &'a Database, } @@ -80,35 +42,63 @@ impl<'a> UseProvider for DbUseProvider<'a> { } } -/// Limits the execution by the number of executed instructions. +pub trait ExecutionController { + fn should_continue_running(&self) -> bool; + fn instruction_executed(&mut self); +} + +pub struct RunForever; +impl ExecutionController for RunForever { + fn should_continue_running(&self) -> bool { + true + } + + fn instruction_executed(&mut self) {} +} + pub struct RunLimitedNumberOfInstructions { - max_instructions: usize, - instructions_executed: usize, + instructions_left: usize, } impl RunLimitedNumberOfInstructions { pub fn new(max_instructions: usize) -> Self { Self { - max_instructions, - instructions_executed: 0, + instructions_left: max_instructions, } } } impl ExecutionController for RunLimitedNumberOfInstructions { fn should_continue_running(&self) -> bool { - self.instructions_executed < self.max_instructions + self.instructions_left > 0 } fn instruction_executed(&mut self) { - self.instructions_executed += 1; + if self.instructions_left == 0 { + panic!(); + } + self.instructions_left -= 1; } } -/// Runs forever. -pub struct RunForever; -impl ExecutionController for RunForever { +pub struct CombiningExecutionController<'a, 'b, A: ExecutionController, B: ExecutionController> { + a: &'a mut A, + b: &'b mut B, +} +impl<'a, 'b, A: ExecutionController, B: ExecutionController> + CombiningExecutionController<'a, 'b, A, B> +{ + pub fn new(a: &'a mut A, b: &'b mut B) -> Self { + CombiningExecutionController { a, b } + } +} +impl<'a, 'b, A: ExecutionController, B: ExecutionController> ExecutionController + for CombiningExecutionController<'a, 'b, A, B> +{ fn should_continue_running(&self) -> bool { - true + self.a.should_continue_running() && self.b.should_continue_running() } - fn instruction_executed(&mut self) {} + fn instruction_executed(&mut self) { + self.a.instruction_executed(); + self.b.instruction_executed(); + } } diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index 5e313b4d1..a89334033 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -1,13 +1,13 @@ use super::{ channel::{Capacity, Packet}, - context::Context, + context::{ExecutionController, UseProvider}, heap::{Builtin, ChannelId, Closure, Data, Heap, Pointer}, tracer::{EventData, Tracer}, }; use crate::{ compiler::{hir::Id, lir::Instruction}, module::Module, - vm::context::DummyContext, + vm::context::PanickingUseProvider, }; use itertools::Itertools; use std::collections::HashMap; @@ -103,7 +103,7 @@ impl Fiber { fiber.status = Status::Running; fiber.run_instruction( - &DummyContext, + &PanickingUseProvider, Instruction::Call { num_args: arguments.len(), }, @@ -195,12 +195,18 @@ impl Fiber { self.status = Status::Panicked { reason }; } - pub fn run(&mut self, context: &mut C) { + pub fn run( + &mut self, + use_provider: &mut U, + execution_controller: &mut E, + ) { assert!( matches!(self.status, Status::Running), "Called Fiber::run on a fiber that is not ready to run." ); - while matches!(self.status, Status::Running) && context.should_continue_running() { + while matches!(self.status, Status::Running) + && execution_controller.should_continue_running() + { let current_closure = self.heap.get(self.next_instruction.closure); let current_body = if let Data::Closure(Closure { body, .. }) = ¤t_closure.data { body @@ -234,15 +240,15 @@ impl Fiber { } self.next_instruction.instruction += 1; - self.run_instruction(context, instruction); - context.instruction_executed(); + self.run_instruction(use_provider, instruction); + execution_controller.instruction_executed(); if self.next_instruction == InstructionPointer::null_pointer() { self.status = Status::Done; } } } - pub fn run_instruction(&mut self, context: &C, instruction: Instruction) { + pub fn run_instruction(&mut self, use_provider: &U, instruction: Instruction) { match instruction { Instruction::CreateInt(int) => { let address = self.heap.create_int(int.into()); @@ -338,7 +344,7 @@ impl Fiber { } Data::Builtin(Builtin { function: builtin }) => { self.heap.drop(closure_address); - self.run_builtin_function(context, &builtin, &args); + self.run_builtin_function(&builtin, &args); } _ => { self.panic(format!( @@ -355,7 +361,7 @@ impl Fiber { } Instruction::UseModule { current_module } => { let relative_path = self.data_stack.pop().unwrap(); - match self.use_module(context, current_module, relative_path) { + match self.use_module(use_provider, current_module, relative_path) { Ok(()) => {} Err(reason) => { self.panic(reason); diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 80fd44014..db8eaef6c 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -8,7 +8,10 @@ mod use_module; use self::{ channel::{ChannelBuf, Packet}, - context::Context, + context::{ + CombiningExecutionController, ExecutionController, RunLimitedNumberOfInstructions, + UseProvider, + }, heap::{ChannelId, SendPort}, tracer::Tracer, }; @@ -32,7 +35,6 @@ pub struct Vm { fibers: HashMap, root_fiber: Option, // only None when no fiber is created yet - // TODO: Drop channels channels: HashMap, pub external_operations: HashMap>, @@ -241,7 +243,26 @@ impl Vm { self.external_operations.remove(&channel); } - pub fn run(&mut self, context: &mut C) { + pub fn run( + &mut self, + use_provider: &mut U, + execution_controller: &mut E, + ) { + while self.can_run() && execution_controller.should_continue_running() { + self.run_raw( + use_provider, + &mut CombiningExecutionController::new( + execution_controller, + &mut RunLimitedNumberOfInstructions::new(100), + ), + ); + } + } + fn run_raw( + &mut self, + use_provider: &mut U, + execution_controller: &mut E, + ) { assert!( self.can_run(), "Called Vm::run on a VM that is not ready to run." @@ -261,11 +282,10 @@ impl Vm { } }; if !matches!(fiber.status(), fiber::Status::Running) { - return; // TODO: handle + return; } - // TODO: Limit context. - fiber.run(context); + fiber.run(use_provider, execution_controller); let is_finished = match fiber.status() { fiber::Status::Running => false, diff --git a/compiler/src/vm/tracer.rs b/compiler/src/vm/tracer.rs index 9515ed46b..2aa64dd19 100644 --- a/compiler/src/vm/tracer.rs +++ b/compiler/src/vm/tracer.rs @@ -147,8 +147,7 @@ impl Tracer { Some(id), ), StackEntry::Parallel { .. } => { - (format!("parallel section (todo: format children)"), None) - // TODO: format + ("parallel section (todo: format children)".to_string(), None) } StackEntry::Module { module } => (format!("use {module}"), None), }; diff --git a/compiler/src/vm/use_module.rs b/compiler/src/vm/use_module.rs index e82036af1..339e82bc1 100644 --- a/compiler/src/vm/use_module.rs +++ b/compiler/src/vm/use_module.rs @@ -1,5 +1,5 @@ use super::{ - context::{Context, UseResult}, + context::{PanickingUseProvider, UseProvider, UseResult}, heap::{Closure, Heap, Pointer, Text}, Fiber, }; @@ -10,16 +10,16 @@ use crate::{ use itertools::Itertools; impl Fiber { - pub fn use_module( + pub fn use_module( &mut self, - context: &C, + use_provider: &U, current_module: Module, relative_path: Pointer, ) -> Result<(), String> { let target = UsePath::parse(&self.heap, relative_path)?; let module = target.resolve_relative_to(current_module)?; - match context.use_module(module.clone())? { + match use_provider.use_module(module.clone())? { UseResult::Asset(bytes) => { let bytes = bytes .iter() @@ -32,7 +32,7 @@ impl Fiber { let module_closure = Closure::of_module_lir(module, lir); let address = self.heap.create_closure(module_closure); self.data_stack.push(address); - self.run_instruction(context, Instruction::Call { num_args: 0 }); + self.run_instruction(&PanickingUseProvider, Instruction::Call { num_args: 0 }); } } From 0acb7adbd0566f8b4f2d7298af3ca085c329a812 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 3 Oct 2022 15:45:17 +0200 Subject: [PATCH 47/59] Use tear down when appropriate --- compiler/src/fuzzer/fuzzer.rs | 9 ++-- .../hints/constant_evaluator.rs | 44 +++++++------------ compiler/src/language_server/hints/mod.rs | 2 +- compiler/src/vm/mod.rs | 17 ++++--- compiler/src/vm/tracer.rs | 37 ---------------- 5 files changed, 30 insertions(+), 79 deletions(-) diff --git a/compiler/src/fuzzer/fuzzer.rs b/compiler/src/fuzzer/fuzzer.rs index 44ba8620a..64d8df2a7 100644 --- a/compiler/src/fuzzer/fuzzer.rs +++ b/compiler/src/fuzzer/fuzzer.rs @@ -6,7 +6,7 @@ use crate::{ self, context::{ExecutionController, UseProvider}, tracer::Tracer, - Closure, Heap, Pointer, Vm, + Closure, Heap, Pointer, TearDownResult, Vm, }, }; use std::mem; @@ -106,16 +106,17 @@ impl Fuzzer { // If a `needs` directly inside the tested closure was not // satisfied, then the panic is not closure's fault, but our // fault. + let TearDownResult { heap, tracer, .. } = vm.tear_down(); let is_our_fault = - did_need_in_closure_cause_panic(db, &self.closure_id, &vm.cloned_tracer()); + did_need_in_closure_cause_panic(db, &self.closure_id, &tracer); if is_our_fault { Status::new_fuzzing_attempt(&self.closure_heap, self.closure) } else { Status::PanickedForArguments { - heap: vm.fiber().heap.clone(), + heap, arguments, reason, - tracer: vm.fiber().tracer.clone(), + tracer, } } } diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index 9bae4a44c..f8aea9e7a 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -13,7 +13,7 @@ use crate::{ self, context::{DbUseProvider, RunLimitedNumberOfInstructions}, tracer::{EventData, StackEntry}, - Closure, Heap, Pointer, Vm, + Closure, Heap, Pointer, TearDownResult, Vm, }, }; use itertools::Itertools; @@ -61,17 +61,14 @@ impl ConstantEvaluator { } } - pub fn get_fuzzable_closures(&self, module: &Module) -> (&Heap, Vec<(Id, Pointer)>) { + pub fn get_fuzzable_closures(&self, module: &Module) -> (Heap, Vec<(Id, Pointer)>) { let vm = &self.vms[module]; - ( - &vm.fiber().heap, - vm.fiber() - .fuzzable_closures - .iter() - .filter(|(id, _)| &id.module == module) - .cloned() - .collect_vec(), - ) + let TearDownResult { + heap, + fuzzable_closures, + .. + } = vm.clone().tear_down(); + (heap, fuzzable_closures) } pub fn get_hints(&self, db: &Database, module: &Module) -> Vec { @@ -86,13 +83,9 @@ impl ConstantEvaluator { hints.push(hint); } }; - if module.to_possible_paths().is_some() { - let fiber = vm.fiber(); - module - .dump_associated_debug_file("trace", &fiber.tracer.format_full_trace(&fiber.heap)); - } - for entry in &vm.cloned_tracer().events { + let TearDownResult { heap, tracer, .. } = vm.clone().tear_down(); + for entry in &tracer.events { if let EventData::ValueEvaluated { id, value } = &entry.data { if &id.module != module { continue; @@ -115,7 +108,7 @@ impl ConstantEvaluator { hints.push(Hint { kind: HintKind::Value, - text: value.format(&vm.fiber().heap), + text: value.format(&heap), position: id_to_end_of_line(db, id.clone()).unwrap(), }); } @@ -129,7 +122,8 @@ fn panic_hint(db: &Database, module: Module, vm: &Vm, reason: String) -> Option< // We want to show the hint at the last call site still inside the current // module. If there is no call site in this module, then the panic results // from a compiler error in a previous stage which is already reported. - let stack = vm.fiber().tracer.stack_trace(); + let TearDownResult { heap, tracer, .. } = vm.clone().tear_down(); + let stack = tracer.stack_trace(); if stack.len() == 1 { // The stack only contains an `InModule` entry. This indicates an error // during compilation resulting in a top-level error instruction. @@ -152,10 +146,8 @@ fn panic_hint(db: &Database, module: Module, vm: &Vm, reason: String) -> Option< id, format!( "{} {}", - closure.format(&vm.fiber().heap), - args.iter() - .map(|arg| arg.format(&vm.fiber().heap)) - .join(" ") + closure.format(&heap), + args.iter().map(|arg| arg.format(&heap)).join(" ") ), ), StackEntry::Needs { @@ -164,11 +156,7 @@ fn panic_hint(db: &Database, module: Module, vm: &Vm, reason: String) -> Option< reason, } => ( id, - format!( - "needs {} {}", - condition.format(&vm.fiber().heap), - reason.format(&vm.fiber().heap) - ), + format!("needs {} {}", condition.format(&heap), reason.format(&heap)), ), _ => unreachable!(), }; diff --git a/compiler/src/language_server/hints/mod.rs b/compiler/src/language_server/hints/mod.rs index dec9b3fd0..20e3d1058 100644 --- a/compiler/src/language_server/hints/mod.rs +++ b/compiler/src/language_server/hints/mod.rs @@ -99,7 +99,7 @@ pub async fn run_server( let module_with_new_insight = 'new_insight: { if let Some(module) = constant_evaluator.run(&db) { let (heap, closures) = constant_evaluator.get_fuzzable_closures(&module); - fuzzer.update_module(module.clone(), heap, &closures); + fuzzer.update_module(module.clone(), &heap, &closures); break 'new_insight Some(module); } if let Some(module) = fuzzer.run(&db) { diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index db8eaef6c..a6b34f30b 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -13,7 +13,6 @@ use self::{ UseProvider, }, heap::{ChannelId, SendPort}, - tracer::Tracer, }; pub use fiber::{Fiber, TearDownResult}; pub use heap::{Closure, Heap, Object, Pointer, Struct}; @@ -205,14 +204,14 @@ impl Vm { matches!(self.status(), Status::CanRun) } - pub fn fiber(&self) -> &Fiber { - // TODO: Remove before merging the PR - todo!() - } - pub fn cloned_tracer(&self) -> Tracer { - // TODO: Remove - self.fiber().tracer.clone() - } + // pub fn fiber(&self) -> &Fiber { + // // TODO: Remove before merging the PR + // todo!() + // } + // pub fn cloned_tracer(&self) -> Tracer { + // // TODO: Remove + // self.fiber().tracer.clone() + // } /// Can be called at any time from outside the VM to create a channel that /// can be used to communicate with the outside world. diff --git a/compiler/src/vm/tracer.rs b/compiler/src/vm/tracer.rs index 2aa64dd19..6f182a877 100644 --- a/compiler/src/vm/tracer.rs +++ b/compiler/src/vm/tracer.rs @@ -42,10 +42,6 @@ pub enum EventData { NeedsEnded { nothing: Pointer, }, - ParallelStarted, - ParallelEnded { - return_value: Pointer, - }, ModuleStarted { module: Module, }, @@ -78,7 +74,6 @@ pub enum StackEntry { condition: Pointer, reason: Pointer, }, - Parallel, Module { module: Module, }, @@ -111,12 +106,6 @@ impl Tracer { EventData::NeedsEnded { .. } => { stack.pop().unwrap(); } - EventData::ParallelStarted => { - stack.push(StackEntry::Parallel); - } - EventData::ParallelEnded { .. } => { - stack.pop().unwrap(); - } EventData::ModuleStarted { module } => stack.push(StackEntry::Module { module }), EventData::ModuleEnded { .. } => { stack.pop().unwrap(); @@ -146,9 +135,6 @@ impl Tracer { format!("needs {} {}", condition.format(heap), reason.format(heap)), Some(id), ), - StackEntry::Parallel { .. } => { - ("parallel section (todo: format children)".to_string(), None) - } StackEntry::Module { module } => (format!("use {module}"), None), }; let caller_location_string = { @@ -217,9 +203,6 @@ pub enum TraceData { reason: Pointer, result: TraceResult, }, - Parallel { - result: TraceResult, - }, Module { module: Module, inner: Vec, @@ -311,17 +294,6 @@ impl Tracer { }, }); } - EventData::ParallelStarted => {} - EventData::ParallelEnded { return_value } => { - let span = stack.pop().unwrap(); - stack.last_mut().unwrap().inner.push(Trace { - start: span.start, - end: event.when, - data: TraceData::Parallel { - result: TraceResult::Returned(*return_value), - }, - }); - } EventData::ModuleStarted { module } => { stack.push(Span { start: event.when, @@ -351,9 +323,6 @@ impl Tracer { } stack.pop().unwrap().inner.pop().unwrap() // TODO: handle multiple traces } - pub fn format_full_trace(&self, heap: &Heap) -> String { - self.full_trace().format(heap) - } } struct Span { @@ -407,12 +376,6 @@ impl Trace { result.format(heap), )); } - TraceData::Parallel { result } => { - lines.push(format!( - "parallel section that completed with {}", - result.format(heap), - )); - } TraceData::Module { module, inner, From 989541ebbc3f1bd09599ee745bbd50a53b160963 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 3 Oct 2022 22:12:56 +0200 Subject: [PATCH 48/59] Ignore some lints --- compiler/src/fuzzer/mod.rs | 2 +- compiler/src/language_server/semantic_tokens.rs | 1 + compiler/src/vm/tracer.rs | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/compiler/src/fuzzer/mod.rs b/compiler/src/fuzzer/mod.rs index 40d1bf35c..03f902f69 100644 --- a/compiler/src/fuzzer/mod.rs +++ b/compiler/src/fuzzer/mod.rs @@ -54,7 +54,7 @@ pub async fn fuzz(db: &Database, module: Module) { info!("This was the stack trace:"); tracer.dump_stack_trace(db, heap); - module.dump_associated_debug_file("trace", &tracer.full_trace().format(&heap)); + module.dump_associated_debug_file("trace", &tracer.full_trace().format(heap)); } } } diff --git a/compiler/src/language_server/semantic_tokens.rs b/compiler/src/language_server/semantic_tokens.rs index 999be094e..bc57a9434 100644 --- a/compiler/src/language_server/semantic_tokens.rs +++ b/compiler/src/language_server/semantic_tokens.rs @@ -121,6 +121,7 @@ impl<'a> Context<'a> { self.cursor, ); + #[allow(clippy::bool_to_int_with_if)] let definition_modifier = if type_ == SemanticTokenType::Assignment { 0b1 } else { diff --git a/compiler/src/vm/tracer.rs b/compiler/src/vm/tracer.rs index 6f182a877..12a4ad666 100644 --- a/compiler/src/vm/tracer.rs +++ b/compiler/src/vm/tracer.rs @@ -185,8 +185,12 @@ impl Tracer { // Full traces are a computed tree view of the whole execution. pub struct Trace { + #[allow(dead_code)] start: Instant, + + #[allow(dead_code)] end: Instant, + data: TraceData, } pub enum TraceData { @@ -211,7 +215,11 @@ pub enum TraceData { } pub enum TraceResult { Returned(Pointer), + + #[allow(dead_code)] Panicked(Pointer), + + #[allow(dead_code)] Canceled, } From 433cea706dde2a8e2a280f119549e30a547b9615 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 3 Oct 2022 22:18:28 +0200 Subject: [PATCH 49/59] Update todos --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 573a8acb0..7ced79e64 100644 --- a/README.md +++ b/README.md @@ -117,9 +117,12 @@ Major milestones: ## Short-term TODOs -- implement fibers, channels, and nurseries -- remove builtinPrint +- casing of module names +- fix fault attribution +- rearchitect tracing +- new name? - add caching while compile-time evaluating code +- tags? - pattern matching - pipe operator - text interpolation @@ -127,13 +130,14 @@ Major milestones: - inline functions - minimize inputs found through fuzzing - fuzz parser +- remove builtinPrint - tail call optimization -- new name? - parse function declaration with doc comment but no code +- tracing visualization +- distinguish packages from normal modules - complain about comment lines with too much indentation - develop guidelines about how to format reasons - module files: `.candy` vs `_.candy`? -- casing of module names? ## How to use Candy From 56d8b57d4b7decc4979429f83809caf14a81ab36 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 3 Oct 2022 22:19:07 +0200 Subject: [PATCH 50/59] Ignore more lints --- compiler/src/vm/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index a6b34f30b..699006d7c 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -229,6 +229,7 @@ impl Vm { } } + #[allow(dead_code)] pub fn complete_receive(&mut self, performing_fiber: Option, packet: Packet) { if let Some(fiber) = performing_fiber { let tree = self.fibers.get_mut(&fiber).unwrap(); From c8267cab6943856613297036f12e48b682efa434 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 10 Oct 2022 22:54:19 +0200 Subject: [PATCH 51/59] Apply minor suggestions from code review Co-Authored-By: Jonas Wanke --- .../hints/constant_evaluator.rs | 50 +++++++++---------- compiler/src/main.rs | 5 +- compiler/src/vm/channel.rs | 6 +-- compiler/src/vm/fiber.rs | 33 ++++++------ compiler/src/vm/heap/mod.rs | 11 ++-- compiler/src/vm/heap/object.rs | 5 +- compiler/src/vm/mod.rs | 35 +++++-------- compiler/src/vm/tracer.rs | 19 +++---- 8 files changed, 75 insertions(+), 89 deletions(-) diff --git a/compiler/src/language_server/hints/constant_evaluator.rs b/compiler/src/language_server/hints/constant_evaluator.rs index f8aea9e7a..74d1efd24 100644 --- a/compiler/src/language_server/hints/constant_evaluator.rs +++ b/compiler/src/language_server/hints/constant_evaluator.rs @@ -86,32 +86,32 @@ impl ConstantEvaluator { let TearDownResult { heap, tracer, .. } = vm.clone().tear_down(); for entry in &tracer.events { - if let EventData::ValueEvaluated { id, value } = &entry.data { - if &id.module != module { - continue; - } - let ast_id = match db.hir_to_ast_id(id.clone()) { - Some(ast_id) => ast_id, - None => continue, - }; - let ast = match db.ast(module.clone()) { - Some((ast, _)) => (*ast).clone(), - None => continue, - }; - let ast = match ast.find(&ast_id) { - Some(ast) => ast, - None => continue, - }; - if !matches!(ast.kind, AstKind::Assignment(_)) { - continue; - } - - hints.push(Hint { - kind: HintKind::Value, - text: value.format(&heap), - position: id_to_end_of_line(db, id.clone()).unwrap(), - }); + let EventData::ValueEvaluated { id, value } = &entry.data else { continue; }; + + if &id.module != module { + continue; + } + let ast_id = match db.hir_to_ast_id(id.clone()) { + Some(ast_id) => ast_id, + None => continue, + }; + let ast = match db.ast(module.clone()) { + Some((ast, _)) => (*ast).clone(), + None => continue, + }; + let ast = match ast.find(&ast_id) { + Some(ast) => ast, + None => continue, + }; + if !matches!(ast.kind, AstKind::Assignment(_)) { + continue; } + + hints.push(Hint { + kind: HintKind::Value, + text: value.format(&heap), + position: id_to_end_of_line(db, id.clone()).unwrap(), + }); } hints diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 25a456814..1e6976d9f 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -1,6 +1,7 @@ #![feature(async_closure)] #![feature(box_patterns)] #![feature(let_chains)] +#![feature(let_else)] #![feature(never_type)] #![feature(try_trait_v2)] #![allow(clippy::module_inception)] @@ -211,7 +212,7 @@ fn run(options: CandyRunOptions) { info!("Tree: {:#?}", vm); match vm.status() { Status::CanRun => { - debug!("VM still running."); + debug!("VM is still running."); vm.run(&mut DbUseProvider { db: &db }, &mut RunForever); } Status::WaitingForOperations => { @@ -271,7 +272,7 @@ fn run(options: CandyRunOptions) { info!("Tree: {:#?}", vm); match vm.status() { Status::CanRun => { - debug!("VM still running."); + debug!("VM is still running."); vm.run(&mut DbUseProvider { db: &db }, &mut RunForever); // TODO: handle operations } diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index 4ae8b00c4..ac56dddd1 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -6,7 +6,7 @@ use std::{collections::VecDeque, fmt}; /// ports, you can get packets out again. /// /// Channels always have a maximum capacity of packets that they can hold -/// simultaneously – you can set it to something large, but having no capacity +/// simultaneously – you can set it to something large, but having no limit /// enables buggy code that leaks memory. #[derive(Clone)] pub struct ChannelBuf { @@ -40,7 +40,7 @@ impl ChannelBuf { pub fn send(&mut self, packet: Packet) { if self.is_full() { - panic!("Tried to send on channel that is full."); + panic!("Tried to send on a channel that is full."); } self.packets.push_back(packet); } @@ -48,7 +48,7 @@ impl ChannelBuf { pub fn receive(&mut self) -> Packet { self.packets .pop_front() - .expect("Tried to receive from channel that is empty.") + .expect("Tried to receive from a channel that is empty.") } } diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index a89334033..d50f20c4b 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -29,7 +29,7 @@ pub struct Fiber { pub heap: Heap, // Debug stuff. This is not essential to a correct working of the fiber, but - // enables advanced functionality like stack traces or finding out who's + // enables advanced functionality like stack traces or finding out whose // fault a panic is. pub tracer: Tracer, pub fuzzable_closures: Vec<(Id, Pointer)>, @@ -142,6 +142,8 @@ impl Fiber { // calling `run` again. pub fn complete_channel_create(&mut self, channel: ChannelId) { + assert!(matches!(self.status, Status::CreatingChannel { .. })); + let send_port = self.heap.create_send_port(channel); let receive_port = self.heap.create_receive_port(channel); self.data_stack @@ -149,10 +151,14 @@ impl Fiber { self.status = Status::Running; } pub fn complete_send(&mut self) { + assert!(matches!(self.status, Status::Sending { .. })); + self.data_stack.push(self.heap.create_nothing()); self.status = Status::Running; } pub fn complete_receive(&mut self, packet: Packet) { + assert!(matches!(self.status, Status::Receiving { .. })); + let address = packet .heap .clone_single_to_other_heap(&mut self.heap, packet.value); @@ -160,6 +166,8 @@ impl Fiber { self.status = Status::Running; } pub fn complete_parallel_scope(&mut self, result: Result<(), String>) { + assert!(matches!(self.status, Status::InParallelScope { .. })); + match result { Ok(()) => { self.data_stack.push(self.heap.create_nothing()); @@ -169,22 +177,12 @@ impl Fiber { } } pub fn complete_try(&mut self, result: Result) { - let result = match result { - Ok(Packet { - heap, - value: return_value, - }) => { - let ok = self.heap.create_symbol("Ok".to_string()); - let return_value = heap.clone_single_to_other_heap(&mut self.heap, return_value); - self.heap.create_list(&[ok, return_value]) - } - Err(panic_reason) => { - let err = self.heap.create_symbol("Err".to_string()); - let reason = self.heap.create_text(panic_reason); - self.heap.create_list(&[err, reason]) - } - }; - self.data_stack.push(result); + assert!(matches!(self.status, Status::InTry { .. })); + + let result = result + .map(|Packet { heap, value }| heap.clone_single_to_other_heap(&mut self.heap, value)) + .map_err(|reason| self.heap.create_text(reason)); + self.data_stack.push(self.heap.create_result(result)); self.status = Status::Running; } @@ -192,6 +190,7 @@ impl Fiber { self.data_stack[self.data_stack.len() - 1 - offset as usize] } pub fn panic(&mut self, reason: String) { + assert!(matches!(self.status, Status::Running)); self.status = Status::Panicked { reason }; } diff --git a/compiler/src/vm/heap/mod.rs b/compiler/src/vm/heap/mod.rs index f0269bb85..c30dd32eb 100644 --- a/compiler/src/vm/heap/mod.rs +++ b/compiler/src/vm/heap/mod.rs @@ -242,11 +242,12 @@ impl Heap { self.create_symbol("Nothing".to_string()) } pub fn create_list(&mut self, items: &[Pointer]) -> Pointer { - let mut fields = vec![]; - for (index, item) in items.iter().enumerate() { - fields.push((self.create_int(index.into()), *item)); - } - self.create_struct(fields.into_iter().collect()) + let fields = items + .iter() + .enumerate() + .map(|(index, item)| (self.create_int(index.into()), *item)) + .collect(); + self.create_struct(fields) } pub fn create_bool(&mut self, value: bool) -> Pointer { self.create_symbol(if value { "True" } else { "False" }.to_string()) diff --git a/compiler/src/vm/heap/object.rs b/compiler/src/vm/heap/object.rs index bae25b248..08fabb48f 100644 --- a/compiler/src/vm/heap/object.rs +++ b/compiler/src/vm/heap/object.rs @@ -7,6 +7,7 @@ use crate::{ }, database::Database, module::Module, + vm::CountableId, }; use itertools::Itertools; use num_bigint::BigInt; @@ -166,8 +167,8 @@ pub struct ReceivePort { #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct ChannelId(usize); -impl From for ChannelId { - fn from(id: usize) -> Self { +impl CountableId for ChannelId { + fn from_usize(id: usize) -> Self { Self(id) } } diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 699006d7c..7b878565f 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -43,6 +43,11 @@ pub struct Vm { #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct FiberId(usize); +impl CountableId for FiberId { + fn from_usize(id: usize) -> Self { + Self(id) + } +} #[derive(Clone)] enum FiberTree { @@ -58,7 +63,7 @@ enum FiberTree { Try(Try), } -/// Single fibers are the leafs of the fiber tree. +/// Single fibers are the leaves of the fiber tree. #[derive(Clone)] struct Single { fiber: Fiber, @@ -204,15 +209,6 @@ impl Vm { matches!(self.status(), Status::CanRun) } - // pub fn fiber(&self) -> &Fiber { - // // TODO: Remove before merging the PR - // todo!() - // } - // pub fn cloned_tracer(&self) -> Tracer { - // // TODO: Remove - // self.fiber().tracer.clone() - // } - /// Can be called at any time from outside the VM to create a channel that /// can be used to communicate with the outside world. pub fn create_channel(&mut self) -> ChannelId { @@ -333,9 +329,7 @@ impl Vm { let single = tree.into_single().unwrap(); FiberTree::Parallel(Parallel { paused_fiber: single, - children: [(first_child_id, ChildKind::InitialChild)] - .into_iter() - .collect(), + children: HashMap::from([(first_child_id, ChildKind::InitialChild)]), return_value: None, nursery: nursery_id, }) @@ -817,18 +811,12 @@ impl fmt::Debug for Operation { } } -impl From for FiberId { - fn from(id: usize) -> Self { - Self(id) - } -} - #[derive(Clone)] -struct IdGenerator> { +struct IdGenerator { next_id: usize, _data: PhantomData, } -impl> IdGenerator { +impl IdGenerator { fn start_at(id: usize) -> Self { Self { next_id: id, @@ -838,9 +826,12 @@ impl> IdGenerator { fn generate(&mut self) -> T { let id = self.next_id; self.next_id += 1; - id.into() + T::from_usize(id) } } +pub trait CountableId { + fn from_usize(id: usize) -> Self; +} trait ReplaceHashMapValue { fn replace V>(&mut self, key: K, replacer: F); diff --git a/compiler/src/vm/tracer.rs b/compiler/src/vm/tracer.rs index 12a4ad666..249ec9aa4 100644 --- a/compiler/src/vm/tracer.rs +++ b/compiler/src/vm/tracer.rs @@ -250,9 +250,8 @@ impl Tracer { } EventData::CallEnded { return_value } => { let span = stack.pop().unwrap(); - let (id, closure, args) = match span.data.unwrap() { - StackEntry::Call { id, closure, args } => (id, closure, args), - _ => unreachable!(), + let StackEntry::Call { id, closure, args } = span.data.unwrap() else { + unreachable!() }; stack.last_mut().unwrap().inner.push(Trace { start: span.start, @@ -283,13 +282,8 @@ impl Tracer { } EventData::NeedsEnded { nothing } => { let span = stack.pop().unwrap(); - let (id, condition, reason) = match span.data.unwrap() { - StackEntry::Needs { - id, - condition, - reason, - } => (id, condition, reason), - _ => unreachable!(), + let StackEntry::Needs { id, condition, reason } = span.data.unwrap() else { + unreachable!() }; stack.last_mut().unwrap().inner.push(Trace { start: span.start, @@ -313,9 +307,8 @@ impl Tracer { } EventData::ModuleEnded { export_map } => { let span = stack.pop().unwrap(); - let module = match span.data.unwrap() { - StackEntry::Module { module } => module, - _ => unreachable!(), + let StackEntry::Module { module } = span.data.unwrap() else { + unreachable!() }; stack.last_mut().unwrap().inner.push(Trace { start: span.start, From 8b0ebb8c8fdc1eef73f80f142708d0ab73562eab Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 10 Oct 2022 22:55:30 +0200 Subject: [PATCH 52/59] Update todos Co-Authored-By: Jonas Wanke --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 7ced79e64..12f25191b 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,6 @@ Major milestones: ## Short-term TODOs -- casing of module names - fix fault attribution - rearchitect tracing - new name? @@ -137,7 +136,6 @@ Major milestones: - distinguish packages from normal modules - complain about comment lines with too much indentation - develop guidelines about how to format reasons -- module files: `.candy` vs `_.candy`? ## How to use Candy From c817906916de670b6ea933af6a11436932ff325d Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 10 Oct 2022 23:27:53 +0200 Subject: [PATCH 53/59] Update Core Candy code Co-Authored-By: Jonas Wanke --- packages/Core/_.candy | 6 ++++++ packages/Core/{Channel.candy => channel.candy} | 6 +++--- packages/Core/{Concurrency.candy => concurrency.candy} | 10 +++++----- 3 files changed, 14 insertions(+), 8 deletions(-) rename packages/Core/{Channel.candy => channel.candy} (80%) rename packages/Core/{Concurrency.candy => concurrency.candy} (63%) diff --git a/packages/Core/_.candy b/packages/Core/_.candy index 49a45bbda..6550121a5 100644 --- a/packages/Core/_.candy +++ b/packages/Core/_.candy @@ -1,6 +1,12 @@ bool := use ".bool" +channel := use ".channel" check := (use ".check").check +concurrency = use ".concurrency" +parallel := concurrency.parallel +async := concurrency.async +await := concurrency.await + conditionals = use ".conditionals" if := conditionals.if ifElse := conditionals.ifElse diff --git a/packages/Core/Channel.candy b/packages/Core/channel.candy similarity index 80% rename from packages/Core/Channel.candy rename to packages/Core/channel.candy index 3b4794558..7f3745f8d 100644 --- a/packages/Core/Channel.candy +++ b/packages/Core/channel.candy @@ -1,6 +1,6 @@ -isInt = (use "..Int").is -equals = (use "..Equality").equals -isType = (use "..Type").is +equals = (use "..equality").equals +isInt = (use "..int").is +isType = (use "..type").is isSendPort value := isType value SendPort isReceivePort value := isType value ReceivePort diff --git a/packages/Core/Concurrency.candy b/packages/Core/concurrency.candy similarity index 63% rename from packages/Core/Concurrency.candy rename to packages/Core/concurrency.candy index fe29d1fed..9bf206b91 100644 --- a/packages/Core/Concurrency.candy +++ b/packages/Core/concurrency.candy @@ -1,6 +1,6 @@ -channel = use "..Channel" -function = use "..Function" -structGet = (use "..Struct").getUnwrap +channel = use "..channel" +function = use "..function" +structGetUnwrap = (use "..struct").getUnwrap parallel body := needs (function.is1 body) @@ -10,8 +10,8 @@ async nursery body := needs (channel.isSendPort nursery) needs (function.is0 body) returnChannel = channel.create 1 - returnSendPort = structGet returnChannel 0 - returnReceivePort = structGet returnChannel 1 + returnSendPort = structGetUnwrap returnChannel 0 + returnReceivePort = structGetUnwrap returnChannel 1 channel.send nursery [Closure: body, ReturnChannel: returnSendPort] returnReceivePort From 3546a2f75cd00ecf8bb78e082a8dd7c97d5299b0 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Wed, 12 Oct 2022 21:28:40 +0200 Subject: [PATCH 54/59] Apply more suggestions Co-Authored-By: Jonas Wanke --- compiler/src/main.rs | 7 +++++-- compiler/src/vm/fiber.rs | 17 ++++++++++++----- compiler/src/vm/mod.rs | 5 ++++- packages/Benchmark.candy | 14 ++++++-------- packages/Core/concurrency.candy | 6 ++---- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 1e6976d9f..13fbe8ce2 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -37,6 +37,7 @@ use itertools::Itertools; use language_server::CandyLanguageServer; use notify::{watcher, RecursiveMode, Watcher}; use std::{ + collections::HashMap, convert::TryInto, env::current_dir, path::PathBuf, @@ -216,7 +217,9 @@ fn run(options: CandyRunOptions) { vm.run(&mut DbUseProvider { db: &db }, &mut RunForever); } Status::WaitingForOperations => { - todo!("VM can't proceed until some operations complete."); + todo!( + "Running the module results in a deadlock caused by some channel operations." + ); } _ => break, } @@ -265,7 +268,7 @@ fn run(options: CandyRunOptions) { let environment = { let stdout_symbol = heap.create_symbol("Stdout".to_string()); let stdout_port = heap.create_send_port(stdout); - heap.create_struct([(stdout_symbol, stdout_port)].into_iter().collect()) + heap.create_struct(HashMap::from([(stdout_symbol, stdout_port)])) }; vm.set_up_for_running_closure(heap, main, &[environment]); loop { diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index d50f20c4b..179f65f23 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -144,10 +144,14 @@ impl Fiber { pub fn complete_channel_create(&mut self, channel: ChannelId) { assert!(matches!(self.status, Status::CreatingChannel { .. })); + let send_port_symbol = self.heap.create_symbol("SendPort".to_string()); + let receive_port_symbol = self.heap.create_symbol("ReceivePort".to_string()); let send_port = self.heap.create_send_port(channel); let receive_port = self.heap.create_receive_port(channel); - self.data_stack - .push(self.heap.create_list(&[send_port, receive_port])); + self.data_stack.push(self.heap.create_struct(HashMap::from([ + (send_port_symbol, send_port), + (receive_port_symbol, receive_port), + ]))); self.status = Status::Running; } pub fn complete_send(&mut self) { @@ -165,12 +169,15 @@ impl Fiber { self.data_stack.push(address); self.status = Status::Running; } - pub fn complete_parallel_scope(&mut self, result: Result<(), String>) { + pub fn complete_parallel_scope(&mut self, result: Result) { assert!(matches!(self.status, Status::InParallelScope { .. })); match result { - Ok(()) => { - self.data_stack.push(self.heap.create_nothing()); + Ok(packet) => { + let value = packet + .heap + .clone_single_to_other_heap(&mut self.heap, packet.value); + self.data_stack.push(value); self.status = Status::Running; } Err(reason) => self.panic(reason), diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 7b878565f..6aa74e5c2 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -467,10 +467,13 @@ impl Vm { let Parallel { mut paused_fiber, nursery, + return_value, .. } = tree.into_parallel().unwrap(); self.channels.remove(&nursery).unwrap(); - paused_fiber.fiber.complete_parallel_scope(result); + paused_fiber + .fiber + .complete_parallel_scope(result.map(|_| return_value.unwrap())); FiberTree::Single(paused_fiber) }); } diff --git a/packages/Benchmark.candy b/packages/Benchmark.candy index 8642a9297..dfaee9c4c 100644 --- a/packages/Benchmark.candy +++ b/packages/Benchmark.candy @@ -14,14 +14,12 @@ core = use "..Core" # twentyOne := fib 8 channel = core.channel.create 3 -sendPort = core.struct.getUnwrap channel 0 -receivePort = core.struct.getUnwrap channel 1 -core.channel.send sendPort "One" -core.channel.send sendPort "Two" -✨.print (core.channel.receive receivePort) -core.channel.send sendPort "Three" -✨.print (core.channel.receive receivePort) -✨.print (core.channel.receive receivePort) +core.channel.send channel.sendPort "One" +core.channel.send channel.sendPort "Two" +✨.print (core.channel.receive channel.receivePort) +core.channel.send channel.sendPort "Three" +✨.print (core.channel.receive channel.receivePort) +✨.print (core.channel.receive channel.receivePort) core.parallel { nursery -> ✨.print (core.await (core.async nursery { diff --git a/packages/Core/concurrency.candy b/packages/Core/concurrency.candy index 9bf206b91..81c1f99da 100644 --- a/packages/Core/concurrency.candy +++ b/packages/Core/concurrency.candy @@ -10,10 +10,8 @@ async nursery body := needs (channel.isSendPort nursery) needs (function.is0 body) returnChannel = channel.create 1 - returnSendPort = structGetUnwrap returnChannel 0 - returnReceivePort = structGetUnwrap returnChannel 1 - channel.send nursery [Closure: body, ReturnChannel: returnSendPort] - returnReceivePort + channel.send nursery [Closure: body, ReturnChannel: returnChannel.sendPort] + returnChannel.receivePort await fiber := needs (channel.isReceivePort fiber) From 375fd78ddf8385263f733bf78220624e78be134e Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 13 Oct 2022 15:03:35 +0200 Subject: [PATCH 55/59] Restructure how channels to the outside work --- compiler/src/main.rs | 50 ++-- compiler/src/vm/builtin_functions.rs | 3 +- compiler/src/vm/channel.rs | 122 +++++++++- compiler/src/vm/fiber.rs | 8 +- compiler/src/vm/heap/mod.rs | 5 +- compiler/src/vm/heap/object.rs | 17 +- compiler/src/vm/ids.rs | 63 +++++ compiler/src/vm/mod.rs | 343 ++++++++------------------- 8 files changed, 323 insertions(+), 288 deletions(-) create mode 100644 compiler/src/vm/ids.rs diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 13fbe8ce2..532516122 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -48,6 +48,7 @@ use structopt::StructOpt; use tower_lsp::{LspService, Server}; use tracing::{debug, error, info, warn, Level, Metadata}; use tracing_subscriber::{filter, fmt::format::FmtSpan, prelude::*}; +use vm::{ChannelId, CompletedOperation, OperationId}; #[derive(StructOpt, Debug)] #[structopt(name = "candy", about = "The 🍭 Candy CLI.")] @@ -264,10 +265,10 @@ fn run(options: CandyRunOptions) { info!("Running main function."); // TODO: Add environment stuff. let mut vm = Vm::new(); - let stdout = vm.create_channel(); + let mut stdout = StdoutService::new(&mut vm); let environment = { let stdout_symbol = heap.create_symbol("Stdout".to_string()); - let stdout_port = heap.create_send_port(stdout); + let stdout_port = heap.create_send_port(stdout.channel); heap.create_struct(HashMap::from([(stdout_symbol, stdout_port)])) }; vm.set_up_for_running_closure(heap, main, &[environment]); @@ -284,25 +285,7 @@ fn run(options: CandyRunOptions) { } _ => break, } - let stdout_operations = vm - .external_operations - .get_mut(&stdout) - .unwrap() - .drain(..) - .collect_vec(); - for operation in stdout_operations { - match operation { - vm::Operation::Send { - performing_fiber, - packet, - } => { - info!("Sent to stdout: {}", packet.value.format(&packet.heap)); - vm.complete_send(performing_fiber); - } - vm::Operation::Receive { .. } => unreachable!(), - vm::Operation::Drop => vm.free_channel(stdout), - } - } + stdout.run(&mut vm); } info!("Tree: {:#?}", vm); let TearDownResult { @@ -322,6 +305,31 @@ fn run(options: CandyRunOptions) { } } +/// A state machine that corresponds to a loop that always calls `receive` on +/// the stdout channel and then logs that packet. +struct StdoutService { + channel: ChannelId, + current_receive: OperationId, +} +impl StdoutService { + fn new(vm: &mut Vm) -> Self { + let channel = vm.create_channel(1); + let current_receive = vm.receive(channel); + Self { + channel, + current_receive, + } + } + fn run(&mut self, vm: &mut Vm) { + if let Some(CompletedOperation::Received { packet }) = + vm.completed_operations.remove(&self.current_receive) + { + info!("Sent to stdout: {}", packet.value.format(&packet.heap)); + self.current_receive = vm.receive(self.channel); + } + } +} + async fn fuzz(options: CandyFuzzOptions) { let module = Module::from_package_root_and_file( current_dir().unwrap(), diff --git a/compiler/src/vm/builtin_functions.rs b/compiler/src/vm/builtin_functions.rs index 50e263a17..3d4f4ba4a 100644 --- a/compiler/src/vm/builtin_functions.rs +++ b/compiler/src/vm/builtin_functions.rs @@ -2,7 +2,8 @@ use super::{ channel::{Capacity, Packet}, context::PanickingUseProvider, fiber::{Fiber, Status}, - heap::{ChannelId, Closure, Data, Int, Pointer, ReceivePort, SendPort, Struct, Symbol, Text}, + heap::{Closure, Data, Int, Pointer, ReceivePort, SendPort, Struct, Symbol, Text}, + ids::ChannelId, Heap, }; use crate::{builtin_functions::BuiltinFunction, compiler::lir::Instruction}; diff --git a/compiler/src/vm/channel.rs b/compiler/src/vm/channel.rs index ac56dddd1..ba8d2d8f0 100644 --- a/compiler/src/vm/channel.rs +++ b/compiler/src/vm/channel.rs @@ -1,4 +1,5 @@ -use super::{Heap, Pointer}; +use super::{ids::OperationId, FiberId, Heap, Pointer}; +use itertools::Itertools; use std::{collections::VecDeque, fmt}; /// A conveyer belt or pipe that flows between send and receive ports in the @@ -9,7 +10,7 @@ use std::{collections::VecDeque, fmt}; /// simultaneously – you can set it to something large, but having no limit /// enables buggy code that leaks memory. #[derive(Clone)] -pub struct ChannelBuf { +struct ChannelBuf { pub capacity: Capacity, packets: VecDeque, } @@ -24,34 +25,106 @@ pub struct Packet { } impl ChannelBuf { - pub fn new(capacity: Capacity) -> Self { + fn new(capacity: Capacity) -> Self { Self { capacity, packets: VecDeque::with_capacity(capacity), } } - pub fn is_empty(&self) -> bool { + fn is_empty(&self) -> bool { self.packets.is_empty() } - pub fn is_full(&self) -> bool { + fn is_full(&self) -> bool { self.packets.len() == self.capacity } - pub fn send(&mut self, packet: Packet) { + fn send(&mut self, packet: Packet) { if self.is_full() { panic!("Tried to send on a channel that is full."); } self.packets.push_back(packet); } - pub fn receive(&mut self) -> Packet { + fn receive(&mut self) -> Packet { self.packets .pop_front() .expect("Tried to receive from a channel that is empty.") } } +/// A wrapper around `ChannelBuf` that also stores pending operations and +/// completes them lazily. +#[derive(Clone)] +pub struct Channel { + buffer: ChannelBuf, + pending_sends: VecDeque<(Performer, Packet)>, + pending_receives: VecDeque, +} + +#[derive(Clone)] +pub enum Performer { + Fiber(FiberId), + Nursery, + External(OperationId), +} + +pub trait Completer { + fn complete_send(&mut self, performer: Performer); + fn complete_receive(&mut self, performer: Performer, received: Packet); +} + +impl Channel { + pub fn new(capacity: usize) -> Self { + Self { + buffer: ChannelBuf::new(capacity), + pending_sends: Default::default(), + pending_receives: Default::default(), + } + } + + pub fn send(&mut self, completer: &mut dyn Completer, performer: Performer, packet: Packet) { + self.pending_sends.push_back((performer, packet)); + self.work_on_pending_operations(completer); + } + + pub fn receive(&mut self, completer: &mut dyn Completer, performer: Performer) { + self.pending_receives.push_back(performer); + self.work_on_pending_operations(completer); + } + + fn work_on_pending_operations(&mut self, completer: &mut dyn Completer) { + if self.buffer.capacity == 0 { + while !self.pending_sends.is_empty() && !self.pending_receives.is_empty() { + let (sender, packet) = self.pending_sends.pop_front().unwrap(); + let receiver = self.pending_receives.pop_front().unwrap(); + completer.complete_send(sender); + completer.complete_receive(receiver, packet); + } + } else { + loop { + let mut did_perform_operation = false; + + if !self.buffer.is_full() && let Some((performer, packet)) = self.pending_sends.pop_front() { + self.buffer.send(packet); + completer.complete_send(performer); + did_perform_operation = true; + } + + if !self.buffer.is_empty() && let Some(performer) = self.pending_receives.pop_front() { + let packet = self.buffer.receive(); + completer.complete_receive(performer, packet); + did_perform_operation = true; + } + + if !did_perform_operation { + break; + } + } + } + } +} + impl fmt::Debug for ChannelBuf { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.packets.iter()).finish() @@ -62,3 +135,38 @@ impl fmt::Debug for Packet { write!(f, "{}", self.value.format(&self.heap)) } } +impl fmt::Debug for Channel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Channel") + .field("buffer", &self.buffer) + .field( + "operations", + &self + .pending_sends + .iter() + .map(|(performer, packet)| { + format!( + "{}sending {:?}", + if let Performer::Fiber(fiber) = performer { + format!("{:?} ", fiber) + } else { + "".to_string() + }, + packet + ) + }) + .chain(self.pending_receives.iter().map(|performer| { + format!( + "{}receiving", + if let Performer::Fiber(fiber) = performer { + format!("{:?} ", fiber) + } else { + "".to_string() + }, + ) + })) + .collect_vec(), + ) + .finish() + } +} diff --git a/compiler/src/vm/fiber.rs b/compiler/src/vm/fiber.rs index 179f65f23..a8f3e9f8b 100644 --- a/compiler/src/vm/fiber.rs +++ b/compiler/src/vm/fiber.rs @@ -1,7 +1,8 @@ use super::{ channel::{Capacity, Packet}, context::{ExecutionController, UseProvider}, - heap::{Builtin, ChannelId, Closure, Data, Heap, Pointer}, + heap::{Builtin, Closure, Data, Heap, Pointer}, + ids::ChannelId, tracer::{EventData, Tracer}, }; use crate::{ @@ -197,7 +198,10 @@ impl Fiber { self.data_stack[self.data_stack.len() - 1 - offset as usize] } pub fn panic(&mut self, reason: String) { - assert!(matches!(self.status, Status::Running)); + assert!(!matches!( + self.status, + Status::Done | Status::Panicked { .. } + )); self.status = Status::Panicked { reason }; } diff --git a/compiler/src/vm/heap/mod.rs b/compiler/src/vm/heap/mod.rs index c30dd32eb..b428584b7 100644 --- a/compiler/src/vm/heap/mod.rs +++ b/compiler/src/vm/heap/mod.rs @@ -2,11 +2,10 @@ mod object; mod pointer; pub use self::{ - object::{ - Builtin, ChannelId, Closure, Data, Int, Object, ReceivePort, SendPort, Struct, Symbol, Text, - }, + object::{Builtin, Closure, Data, Int, Object, ReceivePort, SendPort, Struct, Symbol, Text}, pointer::Pointer, }; +use super::ids::ChannelId; use crate::builtin_functions::BuiltinFunction; use itertools::Itertools; use num_bigint::BigInt; diff --git a/compiler/src/vm/heap/object.rs b/compiler/src/vm/heap/object.rs index 08fabb48f..cd3021df7 100644 --- a/compiler/src/vm/heap/object.rs +++ b/compiler/src/vm/heap/object.rs @@ -7,13 +7,12 @@ use crate::{ }, database::Database, module::Module, - vm::CountableId, + vm::ids::ChannelId, }; use itertools::Itertools; use num_bigint::BigInt; use std::{ collections::{hash_map::DefaultHasher, HashMap}, - fmt, hash::{Hash, Hasher}, ops::Deref, }; @@ -164,20 +163,6 @@ pub struct ReceivePort { pub channel: ChannelId, } -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -pub struct ChannelId(usize); - -impl CountableId for ChannelId { - fn from_usize(id: usize) -> Self { - Self(id) - } -} -impl fmt::Debug for ChannelId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "channel_{:x}", self.0) - } -} - impl SendPort { pub fn new(channel: ChannelId) -> Self { Self { channel } diff --git a/compiler/src/vm/ids.rs b/compiler/src/vm/ids.rs new file mode 100644 index 000000000..4148211a7 --- /dev/null +++ b/compiler/src/vm/ids.rs @@ -0,0 +1,63 @@ +use std::{fmt, marker::PhantomData}; + +#[derive(Clone)] +pub struct IdGenerator { + next_id: usize, + _data: PhantomData, +} +impl IdGenerator { + pub fn start_at(id: usize) -> Self { + Self { + next_id: id, + _data: Default::default(), + } + } + pub fn generate(&mut self) -> T { + let id = self.next_id; + self.next_id += 1; + T::from_usize(id) + } +} +pub trait CountableId { + fn from_usize(id: usize) -> Self; +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct FiberId(usize); +impl CountableId for FiberId { + fn from_usize(id: usize) -> Self { + Self(id) + } +} +impl fmt::Debug for FiberId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "fiber_{:x}", self.0) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct ChannelId(usize); + +impl CountableId for ChannelId { + fn from_usize(id: usize) -> Self { + Self(id) + } +} +impl fmt::Debug for ChannelId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "channel_{:x}", self.0) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct OperationId(usize); +impl CountableId for OperationId { + fn from_usize(id: usize) -> Self { + Self(id) + } +} +impl fmt::Debug for OperationId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "operation_{:x}", self.0) + } +} diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 6aa74e5c2..08117886b 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -3,26 +3,30 @@ mod channel; pub mod context; mod fiber; mod heap; +mod ids; pub mod tracer; mod use_module; use self::{ - channel::{ChannelBuf, Packet}, + channel::{Channel, Completer, Packet, Performer}, context::{ CombiningExecutionController, ExecutionController, RunLimitedNumberOfInstructions, UseProvider, }, - heap::{ChannelId, SendPort}, + heap::SendPort, + ids::{FiberId, IdGenerator}, +}; +pub use self::{ + fiber::{Fiber, TearDownResult}, + heap::{Closure, Heap, Object, Pointer, Struct}, + ids::{ChannelId, OperationId}, }; -pub use fiber::{Fiber, TearDownResult}; -pub use heap::{Closure, Heap, Object, Pointer, Struct}; use itertools::Itertools; use rand::seq::SliceRandom; use std::{ - collections::{HashMap, HashSet, VecDeque}, + collections::{HashMap, HashSet}, fmt, hash::Hash, - marker::PhantomData, }; use tracing::{info, warn}; @@ -34,21 +38,14 @@ pub struct Vm { fibers: HashMap, root_fiber: Option, // only None when no fiber is created yet - channels: HashMap, - pub external_operations: HashMap>, + channels: HashMap, + pub completed_operations: HashMap, + operation_id_generator: IdGenerator, fiber_id_generator: IdGenerator, channel_id_generator: IdGenerator, } -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -pub struct FiberId(usize); -impl CountableId for FiberId { - fn from_usize(id: usize) -> Self { - Self(id) - } -} - #[derive(Clone)] enum FiberTree { /// This tree is currently focused on running a single fiber. @@ -97,28 +94,15 @@ struct Try { } #[derive(Clone)] -enum Channel { - Internal(InternalChannel), - External, +enum ChannelLike { + Channel(Channel), Nursery(FiberId), } -#[derive(Clone, Debug)] -struct InternalChannel { - buffer: ChannelBuf, - pending_sends: VecDeque<(Option, Packet)>, - pending_receives: VecDeque>, -} #[derive(Clone)] -pub enum Operation { - Send { - performing_fiber: Option, - packet: Packet, - }, - Receive { - performing_fiber: Option, - }, - Drop, +pub enum CompletedOperation { + Sent, + Received { packet: Packet }, } #[derive(Clone, Debug)] @@ -132,10 +116,11 @@ pub enum Status { impl Vm { pub fn new() -> Self { Self { - channels: Default::default(), fibers: HashMap::new(), root_fiber: None, - external_operations: Default::default(), + channels: HashMap::new(), + completed_operations: Default::default(), + operation_id_generator: IdGenerator::start_at(0), channel_id_generator: IdGenerator::start_at(0), fiber_id_generator: IdGenerator::start_at(0), } @@ -211,33 +196,34 @@ impl Vm { /// Can be called at any time from outside the VM to create a channel that /// can be used to communicate with the outside world. - pub fn create_channel(&mut self) -> ChannelId { + pub fn create_channel(&mut self, capacity: usize) -> ChannelId { let id = self.channel_id_generator.generate(); - self.channels.insert(id, Channel::External); - self.external_operations.insert(id, vec![]); + self.channels + .insert(id, ChannelLike::Channel(Channel::new(capacity))); id } - pub fn complete_send(&mut self, performing_fiber: Option) { - if let Some(fiber) = performing_fiber { - let tree = self.fibers.get_mut(&fiber).unwrap(); - tree.as_single_mut().unwrap().fiber.complete_send(); - } + // This will be used as soon as the outside world tries to send something + // into the VM. + #[allow(dead_code)] + pub fn send(&mut self, channel: ChannelId, packet: Packet) -> OperationId { + let operation_id = self.operation_id_generator.generate(); + self.send_to_channel(Performer::External(operation_id), channel, packet); + operation_id } - #[allow(dead_code)] - pub fn complete_receive(&mut self, performing_fiber: Option, packet: Packet) { - if let Some(fiber) = performing_fiber { - let tree = self.fibers.get_mut(&fiber).unwrap(); - tree.as_single_mut().unwrap().fiber.complete_receive(packet); - } + pub fn receive(&mut self, channel: ChannelId) -> OperationId { + let operation_id = self.operation_id_generator.generate(); + self.receive_from_channel(Performer::External(operation_id), channel); + operation_id } /// May only be called if a drop operation was emitted for that channel. - pub fn free_channel(&mut self, channel: ChannelId) { - self.channels.remove(&channel); - self.external_operations.remove(&channel); - } + /// TODO: handle drops + // pub fn free_channel(&mut self, channel: ChannelId) { + // self.channels.remove(&channel); + // self.external_operations.remove(&channel); + // } pub fn run( &mut self, @@ -287,28 +273,23 @@ impl Vm { fiber::Status::Running => false, fiber::Status::CreatingChannel { capacity } => { let channel_id = self.channel_id_generator.generate(); - self.channels.insert( - channel_id, - Channel::Internal(InternalChannel { - buffer: ChannelBuf::new(capacity), - pending_sends: Default::default(), - pending_receives: Default::default(), - }), - ); + self.channels + .insert(channel_id, ChannelLike::Channel(Channel::new(capacity))); fiber.complete_channel_create(channel_id); false } fiber::Status::Sending { channel, packet } => { - self.send_to_channel(Some(fiber_id), channel, packet); + self.send_to_channel(Performer::Fiber(fiber_id), channel, packet); false } fiber::Status::Receiving { channel } => { - self.receive_from_channel(Some(fiber_id), channel); + self.receive_from_channel(Performer::Fiber(fiber_id), channel); false } fiber::Status::InParallelScope { body } => { let nursery_id = self.channel_id_generator.generate(); - self.channels.insert(nursery_id, Channel::Nursery(fiber_id)); + self.channels + .insert(nursery_id, ChannelLike::Nursery(fiber_id)); let first_child_id = { let mut heap = Heap::default(); @@ -399,7 +380,7 @@ impl Vm { match child { ChildKind::InitialChild => parallel.return_value = Some(packet), ChildKind::SpawnedChild(return_channel) => { - self.send_to_channel(None, return_channel, packet) + self.send_to_channel(Performer::Nursery, return_channel, packet) } } @@ -429,13 +410,14 @@ impl Vm { known_channels.extend(single.fiber.heap.known_channels().into_iter()); } } + // TODO: handle dropping channels let forgotten_channels = all_channels.difference(&known_channels); for channel in forgotten_channels { match self.channels.get(channel).unwrap() { // If an internal channel is not referenced by any fiber, no // reference to it can be obtained in the future. Thus, it's // safe to remove such channels. - Channel::Internal(_) => { + ChannelLike::Channel(_) => { self.channels.remove(channel); } // External channels may be re-sent into the VM from the outside @@ -443,11 +425,11 @@ impl Vm { // directly, we communicate to the outside that no fiber // references them anymore. The outside can then call // `free_channel` when it doesn't intend to re-use the channel. - Channel::External => { - self.push_external_operation(*channel, Operation::Drop); - } + // ChannelLike::External => { + // self.complete_external_operation(*channel, Operation::Drop); + // } // Nurseries are automatically removed when they are exited. - Channel::Nursery(_) => {} + ChannelLike::Nursery(_) => {} } } } @@ -496,17 +478,13 @@ impl Vm { } } - fn send_to_channel( - &mut self, - performing_fiber: Option, - channel_id: ChannelId, - packet: Packet, - ) { + fn send_to_channel(&mut self, performer: Performer, channel_id: ChannelId, packet: Packet) { let channel = match self.channels.get_mut(&channel_id) { Some(channel) => channel, None => { // The channel was a nursery that died. - if let Some(fiber) = performing_fiber { + // TODO: also panic a nursery performer + if let Performer::Fiber(fiber) = performer { let tree = self.fibers.get_mut(&fiber).unwrap(); tree.as_single_mut().unwrap().fiber.panic( "the nursery is already dead because the parallel section ended" @@ -517,17 +495,14 @@ impl Vm { } }; match channel { - Channel::Internal(channel) => { - channel.send(&mut self.fibers, performing_fiber, packet); + ChannelLike::Channel(channel) => { + let mut completer = InternalCompleter { + fibers: &mut self.fibers, + completed_operations: &mut self.completed_operations, + }; + channel.send(&mut completer, performer, packet); } - Channel::External => self.push_external_operation( - channel_id, - Operation::Send { - performing_fiber, - packet, - }, - ), - Channel::Nursery(parent_id) => { + ChannelLike::Nursery(parent_id) => { info!("Nursery received packet {:?}", packet); let parent_id = *parent_id; @@ -556,7 +531,11 @@ impl Vm { ), } - InternalChannel::complete_send(&mut self.fibers, performing_fiber); + InternalCompleter { + fibers: &mut self.fibers, + completed_operations: &mut self.completed_operations, + } + .complete_send(performer); } } } @@ -583,99 +562,58 @@ impl Vm { Some((heap, closure_address, return_channel.channel)) } - fn receive_from_channel(&mut self, performing_fiber: Option, channel: ChannelId) { + fn receive_from_channel(&mut self, performer: Performer, channel: ChannelId) { + let mut completer = InternalCompleter { + fibers: &mut self.fibers, + completed_operations: &mut self.completed_operations, + }; match self.channels.get_mut(&channel).unwrap() { - Channel::Internal(channel) => { - channel.receive(&mut self.fibers, performing_fiber); - } - Channel::External => { - self.push_external_operation(channel, Operation::Receive { performing_fiber }); + ChannelLike::Channel(channel) => { + channel.receive(&mut completer, performer); } - Channel::Nursery { .. } => unreachable!("nurseries are only sent stuff"), + ChannelLike::Nursery { .. } => unreachable!("nurseries are only sent stuff"), } } - - fn push_external_operation(&mut self, channel: ChannelId, operation: Operation) { - self.external_operations - .entry(channel) - .or_default() - .push(operation); - } } -impl InternalChannel { - fn send( - &mut self, - fibers: &mut HashMap, - performing_fiber: Option, - packet: Packet, - ) { - self.pending_sends.push_back((performing_fiber, packet)); - self.work_on_pending_operations(fibers); - } - - fn receive( - &mut self, - fibers: &mut HashMap, - performing_fiber: Option, - ) { - self.pending_receives.push_back(performing_fiber); - self.work_on_pending_operations(fibers); - } - - fn work_on_pending_operations(&mut self, fibers: &mut HashMap) { - if self.buffer.capacity == 0 { - while !self.pending_sends.is_empty() && !self.pending_receives.is_empty() { - let (send_id, packet) = self.pending_sends.pop_front().unwrap(); - let receive_id = self.pending_receives.pop_front().unwrap(); - Self::complete_send(fibers, send_id); - Self::complete_receive(fibers, receive_id, packet); +struct InternalCompleter<'a> { + fibers: &'a mut HashMap, + completed_operations: &'a mut HashMap, +} +impl<'a> Completer for InternalCompleter<'a> { + fn complete_send(&mut self, performer: Performer) { + match performer { + Performer::Fiber(fiber) => { + let tree = self.fibers.get_mut(&fiber).unwrap(); + tree.as_single_mut().unwrap().fiber.complete_send(); } - } else { - loop { - let mut did_perform_operation = false; - - if !self.buffer.is_full() && let Some((fiber, packet)) = self.pending_sends.pop_front() { - self.buffer.send(packet); - Self::complete_send(fibers, fiber); - did_perform_operation = true; - } - - if !self.buffer.is_empty() && let Some(fiber) = self.pending_receives.pop_front() { - let packet = self.buffer.receive(); - Self::complete_receive(fibers, fiber, packet); - did_perform_operation = true; - } - - if !did_perform_operation { - break; - } + Performer::Nursery => {} + Performer::External(id) => { + self.completed_operations + .insert(id, CompletedOperation::Sent); } } } - fn complete_send(fibers: &mut HashMap, fiber: Option) { - if let Some(fiber) = fiber { - let fiber = fibers.get_mut(&fiber).unwrap().as_single_mut().unwrap(); - fiber.fiber.complete_send(); - } - } - fn complete_receive( - fibers: &mut HashMap, - fiber: Option, - packet: Packet, - ) { - if let Some(fiber) = fiber { - let fiber = fibers.get_mut(&fiber).unwrap().as_single_mut().unwrap(); - fiber.fiber.complete_receive(packet); + fn complete_receive(&mut self, performer: Performer, packet: Packet) { + match performer { + Performer::Fiber(fiber) => { + let tree = self.fibers.get_mut(&fiber).unwrap(); + tree.as_single_mut().unwrap().fiber.complete_receive(packet); + } + Performer::Nursery => {} + Performer::External(id) => { + self.completed_operations + .insert(id, CompletedOperation::Received { packet }); + } } } } -impl Channel { +impl ChannelLike { fn to_nursery(&self) -> Option { match self { - Channel::Nursery(fiber) => Some(*fiber), + ChannelLike::Nursery(fiber) => Some(*fiber), _ => None, } } @@ -726,15 +664,9 @@ impl fmt::Debug for Vm { f.debug_struct("Vm") .field("fibers", &self.fibers) .field("channels", &self.channels) - .field("external_operations", &self.external_operations) .finish() } } -impl fmt::Debug for FiberId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "fiber_{:x}", self.0) - } -} impl fmt::Debug for FiberTree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -762,79 +694,14 @@ impl fmt::Debug for ChildKind { } } } -impl fmt::Debug for Channel { +impl fmt::Debug for ChannelLike { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Internal(InternalChannel { - buffer, - pending_sends, - pending_receives, - }) => f - .debug_struct("InternalChannel") - .field("buffer", buffer) - .field( - "operations", - &pending_sends - .iter() - .map(|(fiber, packet)| Operation::Send { - performing_fiber: *fiber, - packet: packet.clone(), - }) - .chain(pending_receives.iter().map(|fiber| Operation::Receive { - performing_fiber: *fiber, - })) - .collect_vec(), - ) - .finish(), - Self::External => f.debug_tuple("External").finish(), + Self::Channel(channel) => channel.fmt(f), Self::Nursery(fiber) => f.debug_tuple("Nursery").field(fiber).finish(), } } } -impl fmt::Debug for Operation { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self { - Operation::Send { - performing_fiber, - packet, - } => { - if let Some(fiber) = performing_fiber { - write!(f, "{:?} ", fiber)?; - } - write!(f, "sending {:?}", packet) - } - Operation::Receive { performing_fiber } => { - if let Some(fiber) = performing_fiber { - write!(f, "{:?} ", fiber)?; - } - write!(f, "receiving") - } - Operation::Drop => write!(f, "dropping"), - } - } -} - -#[derive(Clone)] -struct IdGenerator { - next_id: usize, - _data: PhantomData, -} -impl IdGenerator { - fn start_at(id: usize) -> Self { - Self { - next_id: id, - _data: Default::default(), - } - } - fn generate(&mut self) -> T { - let id = self.next_id; - self.next_id += 1; - T::from_usize(id) - } -} -pub trait CountableId { - fn from_usize(id: usize) -> Self; -} trait ReplaceHashMapValue { fn replace V>(&mut self, key: K, replacer: F); From 275f00f718c24d28fbc7492e4f76909697e403c5 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Thu, 13 Oct 2022 18:30:41 +0200 Subject: [PATCH 56/59] Free channels --- compiler/src/main.rs | 8 +++++-- compiler/src/vm/mod.rs | 52 +++++++++++++++++++----------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 532516122..cf3b526b5 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -263,7 +263,7 @@ fn run(options: CandyRunOptions) { }; info!("Running main function."); - // TODO: Add environment stuff. + // TODO: Add more environment stuff. let mut vm = Vm::new(); let mut stdout = StdoutService::new(&mut vm); let environment = { @@ -278,7 +278,6 @@ fn run(options: CandyRunOptions) { Status::CanRun => { debug!("VM is still running."); vm.run(&mut DbUseProvider { db: &db }, &mut RunForever); - // TODO: handle operations } Status::WaitingForOperations => { todo!("VM can't proceed until some operations complete."); @@ -286,6 +285,11 @@ fn run(options: CandyRunOptions) { _ => break, } stdout.run(&mut vm); + for channel in vm.unreferenced_channels.iter().copied().collect_vec() { + if channel != stdout.channel { + vm.free_channel(channel); + } + } } info!("Tree: {:#?}", vm); let TearDownResult { diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 08117886b..4857eed31 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -40,6 +40,7 @@ pub struct Vm { channels: HashMap, pub completed_operations: HashMap, + pub unreferenced_channels: HashSet, operation_id_generator: IdGenerator, fiber_id_generator: IdGenerator, @@ -120,6 +121,7 @@ impl Vm { root_fiber: None, channels: HashMap::new(), completed_operations: Default::default(), + unreferenced_channels: Default::default(), operation_id_generator: IdGenerator::start_at(0), channel_id_generator: IdGenerator::start_at(0), fiber_id_generator: IdGenerator::start_at(0), @@ -218,12 +220,12 @@ impl Vm { operation_id } - /// May only be called if a drop operation was emitted for that channel. - /// TODO: handle drops - // pub fn free_channel(&mut self, channel: ChannelId) { - // self.channels.remove(&channel); - // self.external_operations.remove(&channel); - // } + /// May only be called if the channel is in the `unreferenced_channels`. + pub fn free_channel(&mut self, channel: ChannelId) { + assert!(self.unreferenced_channels.contains(&channel)); + self.channels.remove(&channel); + self.unreferenced_channels.remove(&channel); + } pub fn run( &mut self, @@ -410,28 +412,22 @@ impl Vm { known_channels.extend(single.fiber.heap.known_channels().into_iter()); } } - // TODO: handle dropping channels - let forgotten_channels = all_channels.difference(&known_channels); - for channel in forgotten_channels { - match self.channels.get(channel).unwrap() { - // If an internal channel is not referenced by any fiber, no - // reference to it can be obtained in the future. Thus, it's - // safe to remove such channels. - ChannelLike::Channel(_) => { - self.channels.remove(channel); - } - // External channels may be re-sent into the VM from the outside - // even after no fibers remember them. Rather than removing them - // directly, we communicate to the outside that no fiber - // references them anymore. The outside can then call - // `free_channel` when it doesn't intend to re-use the channel. - // ChannelLike::External => { - // self.complete_external_operation(*channel, Operation::Drop); - // } - // Nurseries are automatically removed when they are exited. - ChannelLike::Nursery(_) => {} - } - } + // Because we don't track yet which channels have leaked to the outside + // world, any channel may be re-sent into the VM from the outside even + // after no fibers remember it. Rather than removing it directly, we + // communicate to the outside that no fiber references it anymore. If + // the outside doesn't intend to re-use the channel, it should call + // `free_channel`. + let unreferenced_channels = all_channels + .difference(&known_channels) + .filter(|channel| { + // Note that nurseries are automatically removed when their + // parallel scope is exited. + matches!(self.channels.get(channel).unwrap(), ChannelLike::Channel(_)) + }) + .copied() + .collect(); + self.unreferenced_channels = unreferenced_channels; } fn finish_parallel(&mut self, parallel_id: FiberId, result: Result<(), String>) { let parallel = self From f9d8f21404389d487e95d167bf2423a200eb1277 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 17 Oct 2022 22:03:01 +0200 Subject: [PATCH 57/59] Log less things during fuzzing --- compiler/src/fuzzer/mod.rs | 1 + compiler/src/vm/mod.rs | 11 ++--------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/compiler/src/fuzzer/mod.rs b/compiler/src/fuzzer/mod.rs index 03f902f69..221fc63a5 100644 --- a/compiler/src/fuzzer/mod.rs +++ b/compiler/src/fuzzer/mod.rs @@ -29,6 +29,7 @@ pub async fn fuzz(db: &Database, module: Module) { ); for (id, closure) in fuzzables { + info!("Fuzzing {id}."); let mut fuzzer = Fuzzer::new(&fuzzables_heap, closure, id.clone()); fuzzer.run( db, diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 4857eed31..727199bda 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -28,7 +28,7 @@ use std::{ fmt, hash::Hash, }; -use tracing::{info, warn}; +use tracing::info; /// A VM represents a Candy program that thinks it's currently running. Because /// VMs are first-class Rust structs, they enable other code to store "freezed" @@ -345,14 +345,7 @@ impl Vm { false } - fiber::Status::Done => { - info!("A fiber is done."); - true - } - fiber::Status::Panicked { reason } => { - warn!("A fiber panicked because {reason}."); - true - } + fiber::Status::Done | fiber::Status::Panicked { .. } => true, }; if is_finished && fiber_id != self.root_fiber.unwrap() { From d57c0616460a024873805350e672a31273aaa21a Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Mon, 17 Oct 2022 22:07:34 +0200 Subject: [PATCH 58/59] Remove let_else feature flag --- compiler/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/compiler/src/main.rs b/compiler/src/main.rs index cf3b526b5..e91f47f8a 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -1,7 +1,6 @@ #![feature(async_closure)] #![feature(box_patterns)] #![feature(let_chains)] -#![feature(let_else)] #![feature(never_type)] #![feature(try_trait_v2)] #![allow(clippy::module_inception)] From f95e722b15199928269b2537f3184af453feb203 Mon Sep 17 00:00:00 2001 From: Marcel Garus Date: Sun, 23 Oct 2022 12:16:43 +0200 Subject: [PATCH 59/59] Apply suggestions Co-Authored-By: Jonas Wanke --- compiler/src/main.rs | 4 +--- compiler/src/vm/mod.rs | 1 - packages/Core/concurrency.candy | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/compiler/src/main.rs b/compiler/src/main.rs index e91f47f8a..3e9d2df8f 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -285,9 +285,7 @@ fn run(options: CandyRunOptions) { } stdout.run(&mut vm); for channel in vm.unreferenced_channels.iter().copied().collect_vec() { - if channel != stdout.channel { - vm.free_channel(channel); - } + vm.free_channel(channel); } } info!("Tree: {:#?}", vm); diff --git a/compiler/src/vm/mod.rs b/compiler/src/vm/mod.rs index 727199bda..25e161195 100644 --- a/compiler/src/vm/mod.rs +++ b/compiler/src/vm/mod.rs @@ -472,7 +472,6 @@ impl Vm { Some(channel) => channel, None => { // The channel was a nursery that died. - // TODO: also panic a nursery performer if let Performer::Fiber(fiber) = performer { let tree = self.fibers.get_mut(&fiber).unwrap(); tree.as_single_mut().unwrap().fiber.panic( diff --git a/packages/Core/concurrency.candy b/packages/Core/concurrency.candy index 81c1f99da..7d2f28998 100644 --- a/packages/Core/concurrency.candy +++ b/packages/Core/concurrency.candy @@ -1,6 +1,5 @@ channel = use "..channel" function = use "..function" -structGetUnwrap = (use "..struct").getUnwrap parallel body := needs (function.is1 body)