From 657dbcb78c5a9b55e832b12fe3600be7e09783a4 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 10 Sep 2024 18:12:05 +0200 Subject: [PATCH 01/56] Add :evloop to Crystal::Tracing --- src/crystal/tracing.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/crystal/tracing.cr b/src/crystal/tracing.cr index a680bfea717f..684680b10b28 100644 --- a/src/crystal/tracing.cr +++ b/src/crystal/tracing.cr @@ -7,6 +7,7 @@ module Crystal enum Section GC Sched + Evloop def self.from_id(slice) : self {% begin %} From abd0ce4f11f99e39a4d21ce10aa9f145367df4e7 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 19:40:06 +0200 Subject: [PATCH 02/56] Add C bindings for epoll --- src/crystal/system/unix/epoll.cr | 57 ++++++++++++++++++++ src/lib_c/aarch64-android/c/sys/epoll.cr | 32 +++++++++++ src/lib_c/aarch64-linux-gnu/c/sys/epoll.cr | 32 +++++++++++ src/lib_c/aarch64-linux-musl/c/sys/epoll.cr | 32 +++++++++++ src/lib_c/arm-linux-gnueabihf/c/sys/epoll.cr | 32 +++++++++++ src/lib_c/i386-linux-gnu/c/sys/epoll.cr | 32 +++++++++++ src/lib_c/i386-linux-musl/c/sys/epoll.cr | 32 +++++++++++ src/lib_c/x86_64-linux-gnu/c/sys/epoll.cr | 33 ++++++++++++ src/lib_c/x86_64-linux-musl/c/sys/epoll.cr | 33 ++++++++++++ src/lib_c/x86_64-solaris/c/sys/epoll.cr | 33 ++++++++++++ 10 files changed, 348 insertions(+) create mode 100644 src/crystal/system/unix/epoll.cr create mode 100644 src/lib_c/aarch64-android/c/sys/epoll.cr create mode 100644 src/lib_c/aarch64-linux-gnu/c/sys/epoll.cr create mode 100644 src/lib_c/aarch64-linux-musl/c/sys/epoll.cr create mode 100644 src/lib_c/arm-linux-gnueabihf/c/sys/epoll.cr create mode 100644 src/lib_c/i386-linux-gnu/c/sys/epoll.cr create mode 100644 src/lib_c/i386-linux-musl/c/sys/epoll.cr create mode 100644 src/lib_c/x86_64-linux-gnu/c/sys/epoll.cr create mode 100644 src/lib_c/x86_64-linux-musl/c/sys/epoll.cr create mode 100644 src/lib_c/x86_64-solaris/c/sys/epoll.cr diff --git a/src/crystal/system/unix/epoll.cr b/src/crystal/system/unix/epoll.cr new file mode 100644 index 000000000000..1a6f4ac57b87 --- /dev/null +++ b/src/crystal/system/unix/epoll.cr @@ -0,0 +1,57 @@ +require "c/sys/epoll" + +struct Crystal::System::Epoll + def initialize + @epfd = LibC.epoll_create1(LibC::EPOLL_CLOEXEC) + raise RuntimeError.from_errno("epoll_create1") if @epfd == -1 + end + + def fd : Int32 + @epfd + end + + def add(fd : Int32, epoll_event : LibC::EpollEvent*) : Nil + if LibC.epoll_ctl(@epfd, LibC::EPOLL_CTL_ADD, fd, epoll_event) == -1 + raise RuntimeError.from_errno("epoll_ctl(EPOLL_CTL_ADD)") unless Errno.value == Errno::EPERM + end + end + + def add(fd : Int32, events : UInt32, u64 : UInt64) : Nil + epoll_event = uninitialized LibC::EpollEvent + epoll_event.events = events + epoll_event.data.u64 = u64 + add(fd, pointerof(epoll_event)) + end + + def modify(fd : Int32, epoll_event : LibC::EpollEvent*) : Nil + if LibC.epoll_ctl(@epfd, LibC::EPOLL_CTL_MOD, fd, epoll_event) == -1 + raise RuntimeError.from_errno("epoll_ctl(EPOLL_CTL_MOD)") + end + end + + def delete(fd : Int32) : Nil + delete(fd) do + # OPTIMIZE: we might be able to spare the errno checks for EPERM and ENOENT + unless Errno.value.in?(Errno::EPERM, Errno::ENOENT) + raise RuntimeError.from_errno("epoll_ctl(EPOLL_CTL_DEL)") + end + end + end + + def delete(fd : Int32, &) : Nil + if LibC.epoll_ctl(@epfd, LibC::EPOLL_CTL_DEL, fd, nil) == -1 + yield + end + end + + # `timeout` is in milliseconds; -1 will wait indefinitely; 0 will never wait. + def wait(events : Slice(LibC::EpollEvent), timeout : Int32) : Slice(LibC::EpollEvent) + count = LibC.epoll_wait(@epfd, events.to_unsafe, events.size, timeout) + raise RuntimeError.from_errno("epoll_wait") if count == -1 && Errno.value != Errno::EINTR + events[0, count.clamp(0..)] + end + + def close : Nil + LibC.close(@epfd) + end +end diff --git a/src/lib_c/aarch64-android/c/sys/epoll.cr b/src/lib_c/aarch64-android/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/aarch64-android/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/epoll.cr b/src/lib_c/aarch64-linux-gnu/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/aarch64-linux-gnu/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/aarch64-linux-musl/c/sys/epoll.cr b/src/lib_c/aarch64-linux-musl/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/aarch64-linux-musl/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/epoll.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/i386-linux-gnu/c/sys/epoll.cr b/src/lib_c/i386-linux-gnu/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/i386-linux-gnu/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/i386-linux-musl/c/sys/epoll.cr b/src/lib_c/i386-linux-musl/c/sys/epoll.cr new file mode 100644 index 000000000000..ba34b8414732 --- /dev/null +++ b/src/lib_c/i386-linux-musl/c/sys/epoll.cr @@ -0,0 +1,32 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/epoll.cr b/src/lib_c/x86_64-linux-gnu/c/sys/epoll.cr new file mode 100644 index 000000000000..4dc752f64652 --- /dev/null +++ b/src/lib_c/x86_64-linux-gnu/c/sys/epoll.cr @@ -0,0 +1,33 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + @[Packed] + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/x86_64-linux-musl/c/sys/epoll.cr b/src/lib_c/x86_64-linux-musl/c/sys/epoll.cr new file mode 100644 index 000000000000..4dc752f64652 --- /dev/null +++ b/src/lib_c/x86_64-linux-musl/c/sys/epoll.cr @@ -0,0 +1,33 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + @[Packed] + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end diff --git a/src/lib_c/x86_64-solaris/c/sys/epoll.cr b/src/lib_c/x86_64-solaris/c/sys/epoll.cr new file mode 100644 index 000000000000..4dc752f64652 --- /dev/null +++ b/src/lib_c/x86_64-solaris/c/sys/epoll.cr @@ -0,0 +1,33 @@ +lib LibC + EPOLLIN = 0x001_u32 + EPOLLOUT = 0x004_u32 + EPOLLERR = 0x008_u32 + EPOLLHUP = 0x010_u32 + EPOLLRDHUP = 0x2000_u32 + + EPOLLEXCLUSIVE = 1_u32 << 28 + EPOLLET = 1_u32 << 31 + + EPOLL_CTL_ADD = 1 + EPOLL_CTL_DEL = 2 + EPOLL_CTL_MOD = 3 + + EPOLL_CLOEXEC = 0o2000000 + + union EpollDataT + ptr : Void* + fd : Int + u32 : UInt32 + u64 : UInt64 + end + + @[Packed] + struct EpollEvent + events : UInt32 + data : EpollDataT + end + + fun epoll_create1(Int) : Int + fun epoll_ctl(Int, Int, Int, EpollEvent*) : Int + fun epoll_wait(Int, EpollEvent*, Int, Int) : Int +end From e6c722ddf182a1ca3b7f0a03e51f0390039d3f2d Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 19:41:08 +0200 Subject: [PATCH 03/56] Add C bindings for eventfd --- src/crystal/system/unix/eventfd.cr | 31 +++++++++++++++++++ src/lib_c/aarch64-android/c/sys/eventfd.cr | 5 +++ src/lib_c/aarch64-linux-gnu/c/sys/eventfd.cr | 5 +++ src/lib_c/aarch64-linux-musl/c/sys/eventfd.cr | 5 +++ .../arm-linux-gnueabihf/c/sys/eventfd.cr | 5 +++ src/lib_c/i386-linux-gnu/c/sys/eventfd.cr | 5 +++ src/lib_c/i386-linux-musl/c/sys/eventfd.cr | 5 +++ src/lib_c/x86_64-linux-gnu/c/sys/eventfd.cr | 5 +++ src/lib_c/x86_64-linux-musl/c/sys/eventfd.cr | 5 +++ src/lib_c/x86_64-solaris/c/sys/eventfd.cr | 5 +++ 10 files changed, 76 insertions(+) create mode 100644 src/crystal/system/unix/eventfd.cr create mode 100644 src/lib_c/aarch64-android/c/sys/eventfd.cr create mode 100644 src/lib_c/aarch64-linux-gnu/c/sys/eventfd.cr create mode 100644 src/lib_c/aarch64-linux-musl/c/sys/eventfd.cr create mode 100644 src/lib_c/arm-linux-gnueabihf/c/sys/eventfd.cr create mode 100644 src/lib_c/i386-linux-gnu/c/sys/eventfd.cr create mode 100644 src/lib_c/i386-linux-musl/c/sys/eventfd.cr create mode 100644 src/lib_c/x86_64-linux-gnu/c/sys/eventfd.cr create mode 100644 src/lib_c/x86_64-linux-musl/c/sys/eventfd.cr create mode 100644 src/lib_c/x86_64-solaris/c/sys/eventfd.cr diff --git a/src/crystal/system/unix/eventfd.cr b/src/crystal/system/unix/eventfd.cr new file mode 100644 index 000000000000..6180bf90bf23 --- /dev/null +++ b/src/crystal/system/unix/eventfd.cr @@ -0,0 +1,31 @@ +require "c/sys/eventfd" + +struct Crystal::System::EventFD + # NOTE: no need to concern ourselves with endianness: we interpret the bytes + # in the system order and eventfd can only be used locally (no cross system + # issues). + + getter fd : Int32 + + def initialize(value = 0) + @fd = LibC.eventfd(value, LibC::EFD_CLOEXEC) + raise RuntimeError.from_errno("eventfd") if @fd == -1 + end + + def read : UInt64 + buf = uninitialized UInt8[8] + bytes_read = LibC.read(@fd, buf.to_unsafe, buf.size) + raise RuntimeError.from_errno("eventfd_read") unless bytes_read == 8 + buf.unsafe_as(UInt64) + end + + def write(value : UInt64) : Nil + buf = value.unsafe_as(StaticArray(UInt8, 8)) + bytes_written = LibC.write(@fd, buf.to_unsafe, buf.size) + raise RuntimeError.from_errno("eventfd_write") unless bytes_written == 8 + end + + def close : Nil + LibC.close(@fd) + end +end diff --git a/src/lib_c/aarch64-android/c/sys/eventfd.cr b/src/lib_c/aarch64-android/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/aarch64-android/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/eventfd.cr b/src/lib_c/aarch64-linux-gnu/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/aarch64-linux-gnu/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/aarch64-linux-musl/c/sys/eventfd.cr b/src/lib_c/aarch64-linux-musl/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/aarch64-linux-musl/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/eventfd.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/i386-linux-gnu/c/sys/eventfd.cr b/src/lib_c/i386-linux-gnu/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/i386-linux-gnu/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/i386-linux-musl/c/sys/eventfd.cr b/src/lib_c/i386-linux-musl/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/i386-linux-musl/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/eventfd.cr b/src/lib_c/x86_64-linux-gnu/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/x86_64-linux-gnu/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/x86_64-linux-musl/c/sys/eventfd.cr b/src/lib_c/x86_64-linux-musl/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/x86_64-linux-musl/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end diff --git a/src/lib_c/x86_64-solaris/c/sys/eventfd.cr b/src/lib_c/x86_64-solaris/c/sys/eventfd.cr new file mode 100644 index 000000000000..12f24428a8f4 --- /dev/null +++ b/src/lib_c/x86_64-solaris/c/sys/eventfd.cr @@ -0,0 +1,5 @@ +lib LibC + EFD_CLOEXEC = 0o2000000 + + fun eventfd(count : UInt, flags : Int) : Int +end From ceb7a6f6e8a35d43ce85b69f4a7af3deb0e5d5ef Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 19:42:49 +0200 Subject: [PATCH 04/56] Add C bindings for timerfd + itimerspec --- src/crystal/system/unix/timerfd.cr | 33 +++++++++++++++++++ src/lib_c/aarch64-android/c/sys/timerfd.cr | 10 ++++++ src/lib_c/aarch64-android/c/time.cr | 5 +++ src/lib_c/aarch64-linux-gnu/c/sys/timerfd.cr | 10 ++++++ src/lib_c/aarch64-linux-gnu/c/time.cr | 5 +++ src/lib_c/aarch64-linux-musl/c/sys/timerfd.cr | 10 ++++++ src/lib_c/aarch64-linux-musl/c/time.cr | 5 +++ .../arm-linux-gnueabihf/c/sys/timerfd.cr | 10 ++++++ src/lib_c/arm-linux-gnueabihf/c/time.cr | 5 +++ src/lib_c/i386-linux-gnu/c/sys/timerfd.cr | 10 ++++++ src/lib_c/i386-linux-gnu/c/time.cr | 5 +++ src/lib_c/i386-linux-musl/c/sys/timerfd.cr | 10 ++++++ src/lib_c/i386-linux-musl/c/time.cr | 5 +++ src/lib_c/x86_64-linux-gnu/c/sys/timerfd.cr | 10 ++++++ src/lib_c/x86_64-linux-gnu/c/time.cr | 5 +++ src/lib_c/x86_64-linux-musl/c/sys/timerfd.cr | 10 ++++++ src/lib_c/x86_64-linux-musl/c/time.cr | 5 +++ src/lib_c/x86_64-solaris/c/sys/timerfd.cr | 10 ++++++ src/lib_c/x86_64-solaris/c/time.cr | 5 +++ 19 files changed, 168 insertions(+) create mode 100644 src/crystal/system/unix/timerfd.cr create mode 100644 src/lib_c/aarch64-android/c/sys/timerfd.cr create mode 100644 src/lib_c/aarch64-linux-gnu/c/sys/timerfd.cr create mode 100644 src/lib_c/aarch64-linux-musl/c/sys/timerfd.cr create mode 100644 src/lib_c/arm-linux-gnueabihf/c/sys/timerfd.cr create mode 100644 src/lib_c/i386-linux-gnu/c/sys/timerfd.cr create mode 100644 src/lib_c/i386-linux-musl/c/sys/timerfd.cr create mode 100644 src/lib_c/x86_64-linux-gnu/c/sys/timerfd.cr create mode 100644 src/lib_c/x86_64-linux-musl/c/sys/timerfd.cr create mode 100644 src/lib_c/x86_64-solaris/c/sys/timerfd.cr diff --git a/src/crystal/system/unix/timerfd.cr b/src/crystal/system/unix/timerfd.cr new file mode 100644 index 000000000000..34edbbec7482 --- /dev/null +++ b/src/crystal/system/unix/timerfd.cr @@ -0,0 +1,33 @@ +require "c/sys/timerfd" + +struct Crystal::System::TimerFD + getter fd : Int32 + + # Create a `timerfd` instance set to the monotonic clock. + def initialize + @fd = LibC.timerfd_create(LibC::CLOCK_MONOTONIC, LibC::TFD_CLOEXEC) + raise RuntimeError.from_errno("timerfd_settime") if @fd == -1 + end + + # Arm (start) the timer to run at *time* (absolute time). + def set(time : ::Time::Span) : Nil + itimerspec = uninitialized LibC::Itimerspec + itimerspec.it_interval.tv_sec = 0 + itimerspec.it_interval.tv_nsec = 0 + itimerspec.it_value.tv_sec = typeof(itimerspec.it_value.tv_sec).new!(time.@seconds) + itimerspec.it_value.tv_nsec = typeof(itimerspec.it_value.tv_nsec).new!(time.@nanoseconds) + ret = LibC.timerfd_settime(@fd, LibC::TFD_TIMER_ABSTIME, pointerof(itimerspec), nil) + raise RuntimeError.from_errno("timerfd_settime") if ret == -1 + end + + # Disarm (stop) the timer. + def cancel : Nil + itimerspec = LibC::Itimerspec.new + ret = LibC.timerfd_settime(@fd, LibC::TFD_TIMER_ABSTIME, pointerof(itimerspec), nil) + raise RuntimeError.from_errno("timerfd_settime") if ret == -1 + end + + def close + LibC.close(@fd) + end +end diff --git a/src/lib_c/aarch64-android/c/sys/timerfd.cr b/src/lib_c/aarch64-android/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/aarch64-android/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/aarch64-android/c/time.cr b/src/lib_c/aarch64-android/c/time.cr index 8f8b81291f0d..5007584d3069 100644 --- a/src/lib_c/aarch64-android/c/time.cr +++ b/src/lib_c/aarch64-android/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(__clock : ClockidT, __ts : Timespec*) : Int fun clock_settime(__clock : ClockidT, __ts : Timespec*) : Int fun gmtime_r(__t : TimeT*, __tm : Tm*) : Tm* diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/timerfd.cr b/src/lib_c/aarch64-linux-gnu/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/aarch64-linux-gnu/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/aarch64-linux-gnu/c/time.cr b/src/lib_c/aarch64-linux-gnu/c/time.cr index 710d477e269b..d00579281b41 100644 --- a/src/lib_c/aarch64-linux-gnu/c/time.cr +++ b/src/lib_c/aarch64-linux-gnu/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(clock_id : ClockidT, tp : Timespec*) : Int fun clock_settime(clock_id : ClockidT, tp : Timespec*) : Int fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* diff --git a/src/lib_c/aarch64-linux-musl/c/sys/timerfd.cr b/src/lib_c/aarch64-linux-musl/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/aarch64-linux-musl/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/aarch64-linux-musl/c/time.cr b/src/lib_c/aarch64-linux-musl/c/time.cr index f687c8b35db4..4bf25a7f9efc 100644 --- a/src/lib_c/aarch64-linux-musl/c/time.cr +++ b/src/lib_c/aarch64-linux-musl/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(x0 : ClockidT, x1 : Timespec*) : Int fun clock_settime(x0 : ClockidT, x1 : Timespec*) : Int fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/timerfd.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/arm-linux-gnueabihf/c/time.cr b/src/lib_c/arm-linux-gnueabihf/c/time.cr index 710d477e269b..d00579281b41 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/time.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(clock_id : ClockidT, tp : Timespec*) : Int fun clock_settime(clock_id : ClockidT, tp : Timespec*) : Int fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* diff --git a/src/lib_c/i386-linux-gnu/c/sys/timerfd.cr b/src/lib_c/i386-linux-gnu/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/i386-linux-gnu/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/i386-linux-gnu/c/time.cr b/src/lib_c/i386-linux-gnu/c/time.cr index 710d477e269b..d00579281b41 100644 --- a/src/lib_c/i386-linux-gnu/c/time.cr +++ b/src/lib_c/i386-linux-gnu/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(clock_id : ClockidT, tp : Timespec*) : Int fun clock_settime(clock_id : ClockidT, tp : Timespec*) : Int fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* diff --git a/src/lib_c/i386-linux-musl/c/sys/timerfd.cr b/src/lib_c/i386-linux-musl/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/i386-linux-musl/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/i386-linux-musl/c/time.cr b/src/lib_c/i386-linux-musl/c/time.cr index f687c8b35db4..4bf25a7f9efc 100644 --- a/src/lib_c/i386-linux-musl/c/time.cr +++ b/src/lib_c/i386-linux-musl/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(x0 : ClockidT, x1 : Timespec*) : Int fun clock_settime(x0 : ClockidT, x1 : Timespec*) : Int fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/timerfd.cr b/src/lib_c/x86_64-linux-gnu/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/x86_64-linux-gnu/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/x86_64-linux-gnu/c/time.cr b/src/lib_c/x86_64-linux-gnu/c/time.cr index 710d477e269b..d00579281b41 100644 --- a/src/lib_c/x86_64-linux-gnu/c/time.cr +++ b/src/lib_c/x86_64-linux-gnu/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(clock_id : ClockidT, tp : Timespec*) : Int fun clock_settime(clock_id : ClockidT, tp : Timespec*) : Int fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* diff --git a/src/lib_c/x86_64-linux-musl/c/sys/timerfd.cr b/src/lib_c/x86_64-linux-musl/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/x86_64-linux-musl/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/x86_64-linux-musl/c/time.cr b/src/lib_c/x86_64-linux-musl/c/time.cr index f687c8b35db4..4bf25a7f9efc 100644 --- a/src/lib_c/x86_64-linux-musl/c/time.cr +++ b/src/lib_c/x86_64-linux-musl/c/time.cr @@ -23,6 +23,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(x0 : ClockidT, x1 : Timespec*) : Int fun clock_settime(x0 : ClockidT, x1 : Timespec*) : Int fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* diff --git a/src/lib_c/x86_64-solaris/c/sys/timerfd.cr b/src/lib_c/x86_64-solaris/c/sys/timerfd.cr new file mode 100644 index 000000000000..0632646b0e14 --- /dev/null +++ b/src/lib_c/x86_64-solaris/c/sys/timerfd.cr @@ -0,0 +1,10 @@ +require "../time" + +lib LibC + TFD_NONBLOCK = 0o0004000 + TFD_CLOEXEC = 0o2000000 + TFD_TIMER_ABSTIME = 1 << 0 + + fun timerfd_create(ClockidT, Int) : Int + fun timerfd_settime(Int, Int, Itimerspec*, Itimerspec*) : Int +end diff --git a/src/lib_c/x86_64-solaris/c/time.cr b/src/lib_c/x86_64-solaris/c/time.cr index 531f8e373f4b..0aa8f3fce053 100644 --- a/src/lib_c/x86_64-solaris/c/time.cr +++ b/src/lib_c/x86_64-solaris/c/time.cr @@ -21,6 +21,11 @@ lib LibC tv_nsec : Long end + struct Itimerspec + it_interval : Timespec + it_value : Timespec + end + fun clock_gettime(x0 : ClockidT, x1 : Timespec*) : Int fun clock_settime(x0 : ClockidT, x1 : Timespec*) : Int fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* From 74f58d8f58ee7907dea55850d8d3db68a74b80b5 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 6 Sep 2024 19:44:38 +0200 Subject: [PATCH 05/56] Add C bindings for kqueue --- src/crystal/system/unix/kqueue.cr | 74 +++++++++++++++++++++++ src/lib_c/aarch64-darwin/c/sys/event.cr | 31 ++++++++++ src/lib_c/x86_64-darwin/c/sys/event.cr | 31 ++++++++++ src/lib_c/x86_64-dragonfly/c/sys/event.cr | 30 +++++++++ src/lib_c/x86_64-freebsd/c/sys/event.cr | 32 ++++++++++ src/lib_c/x86_64-netbsd/c/sys/event.cr | 32 ++++++++++ src/lib_c/x86_64-openbsd/c/sys/event.cr | 28 +++++++++ 7 files changed, 258 insertions(+) create mode 100644 src/crystal/system/unix/kqueue.cr create mode 100644 src/lib_c/aarch64-darwin/c/sys/event.cr create mode 100644 src/lib_c/x86_64-darwin/c/sys/event.cr create mode 100644 src/lib_c/x86_64-dragonfly/c/sys/event.cr create mode 100644 src/lib_c/x86_64-freebsd/c/sys/event.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/event.cr create mode 100644 src/lib_c/x86_64-openbsd/c/sys/event.cr diff --git a/src/crystal/system/unix/kqueue.cr b/src/crystal/system/unix/kqueue.cr new file mode 100644 index 000000000000..68f95ebba36f --- /dev/null +++ b/src/crystal/system/unix/kqueue.cr @@ -0,0 +1,74 @@ +require "c/sys/event" + +struct Crystal::System::Kqueue + @kq : LibC::Int + + def initialize + @kq = + {% if LibC.has_method?(:kqueue1) %} + LibC.kqueue1(LibC::O_CLOEXEC) + {% else %} + LibC.kqueue + {% end %} + raise RuntimeError.from_errno("kqueue1") if @kq == -1 + end + + # Registers *changes* and returns a subslice to *events*. + # + # Timeout is relative to now; blocks indefinitely if `nil`; returns + # immediately if zero. + def kevent(changes : Slice(LibC::Kevent), events : Slice(LibC::Kevent), timeout : ::Time::Span? = nil) : Slice(LibC::Kevent) + if timeout + ts = uninitialized LibC::Timespec + ts.tv_sec = typeof(ts.tv_sec).new!(timeout.@seconds) + ts.tv_nsec = typeof(ts.tv_nsec).new!(timeout.@nanoseconds) + tsp = pointerof(ts) + else + tsp = Pointer(LibC::Timespec).null + end + count = LibC.kevent(@kq, changes.to_unsafe, changes.size, events.to_unsafe, events.size, tsp) + raise RuntimeError.from_errno("kevent") if count == -1 && !Errno.value.in?(Errno::EINTR, Errno::ENOENT) + events[0, count.clamp(0..)] + end + + # Helper to register a single event. Returns immediately. + def kevent(ident, filter, flags, fflags = 0, data = 0, udata = nil, &) : Nil + kevent = uninitialized LibC::Kevent + Kqueue.set pointerof(kevent), ident, filter, flags, fflags, data, udata + ret = LibC.kevent(@kq, pointerof(kevent), 1, nil, 0, nil) + yield if ret == -1 + end + + # Helper to register multiple *changes*. Returns immediately. + def kevent(changes : Slice(LibC::Kevent)) : Nil + ret = LibC.kevent(@kq, changes.to_unsafe, changes.size, nil, 0, nil) + yield if ret == -1 # && !Errno.value.in?(Errno::ENOENT, Errno::EPIPE, Errno::ENODEV) + end + + # Helper to wait for registered events to become active. Returns a subslice to + # *events*. + # + # Timeout is relative to now; blocks indefinitely if `nil`; returns + # immediately if zero. + def wait(events : Slice(LibC::Kevent), timeout : ::Time::Span? = nil) : Slice(LibC::Kevent) + changes = uninitialized LibC::Kevent[0] + kevent(changes.to_slice, events, timeout) + end + + def close : Nil + LibC.close(@kq) + end + + @[AlwaysInline] + def self.set(kevent : LibC::Kevent*, ident, filter, flags, fflags = 0, data = 0, udata = nil) : Nil + kevent.value.ident = ident + kevent.value.filter = filter + kevent.value.flags = flags + kevent.value.fflags = fflags + kevent.value.data = data + kevent.value.udata = udata ? udata.as(Void*) : Pointer(Void).null + {% if LibC::Kevent.has_method?(:ext) %} + kevent.value.ext.fill(0) + {% end %} + end +end diff --git a/src/lib_c/aarch64-darwin/c/sys/event.cr b/src/lib_c/aarch64-darwin/c/sys/event.cr new file mode 100644 index 000000000000..6b80c8d2ccb9 --- /dev/null +++ b/src/lib_c/aarch64-darwin/c/sys/event.cr @@ -0,0 +1,31 @@ +require "../time" + +lib LibC + EVFILT_READ = -1_i16 + EVFILT_WRITE = -2_i16 + EVFILT_TIMER = -7_i16 + EVFILT_USER = -10_i16 + + EV_ADD = 0x0001_u16 + EV_DELETE = 0x0002_u16 + EV_ONESHOT = 0x0010_u16 + EV_CLEAR = 0x0020_u16 + EV_EOF = 0x8000_u16 + EV_ERROR = 0x4000_u16 + + NOTE_NSECONDS = 0x00000004_u32 + NOTE_FFCOPY = 0xc0000000_u32 + NOTE_TRIGGER = 0x01000000_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : Int16 + flags : UInt16 + fflags : UInt32 + data : SSizeT # IntptrT + udata : Void* + end + + fun kqueue : Int + fun kevent(kq : Int, changelist : Kevent*, nchanges : Int, eventlist : Kevent*, nevents : Int, timeout : Timespec*) : Int +end diff --git a/src/lib_c/x86_64-darwin/c/sys/event.cr b/src/lib_c/x86_64-darwin/c/sys/event.cr new file mode 100644 index 000000000000..6b80c8d2ccb9 --- /dev/null +++ b/src/lib_c/x86_64-darwin/c/sys/event.cr @@ -0,0 +1,31 @@ +require "../time" + +lib LibC + EVFILT_READ = -1_i16 + EVFILT_WRITE = -2_i16 + EVFILT_TIMER = -7_i16 + EVFILT_USER = -10_i16 + + EV_ADD = 0x0001_u16 + EV_DELETE = 0x0002_u16 + EV_ONESHOT = 0x0010_u16 + EV_CLEAR = 0x0020_u16 + EV_EOF = 0x8000_u16 + EV_ERROR = 0x4000_u16 + + NOTE_NSECONDS = 0x00000004_u32 + NOTE_FFCOPY = 0xc0000000_u32 + NOTE_TRIGGER = 0x01000000_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : Int16 + flags : UInt16 + fflags : UInt32 + data : SSizeT # IntptrT + udata : Void* + end + + fun kqueue : Int + fun kevent(kq : Int, changelist : Kevent*, nchanges : Int, eventlist : Kevent*, nevents : Int, timeout : Timespec*) : Int +end diff --git a/src/lib_c/x86_64-dragonfly/c/sys/event.cr b/src/lib_c/x86_64-dragonfly/c/sys/event.cr new file mode 100644 index 000000000000..274a0da7791e --- /dev/null +++ b/src/lib_c/x86_64-dragonfly/c/sys/event.cr @@ -0,0 +1,30 @@ +require "../time" + +lib LibC + EVFILT_READ = -1_i16 + EVFILT_WRITE = -2_i16 + EVFILT_TIMER = -7_i16 + EVFILT_USER = -9_i16 + + EV_ADD = 0x0001_u16 + EV_DELETE = 0x0002_u16 + EV_ONESHOT = 0x0010_u16 + EV_CLEAR = 0x0020_u16 + EV_EOF = 0x8000_u16 + EV_ERROR = 0x4000_u16 + + NOTE_FFCOPY = 0xc0000000_u32 + NOTE_TRIGGER = 0x01000000_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : Short + flags : UShort + fflags : UInt + data : SSizeT # IntptrT + udata : Void* + end + + fun kqueue : Int + fun kevent(kq : Int, changelist : Kevent*, nchanges : Int, eventlist : Kevent*, nevents : Int, timeout : Timespec*) : Int +end diff --git a/src/lib_c/x86_64-freebsd/c/sys/event.cr b/src/lib_c/x86_64-freebsd/c/sys/event.cr new file mode 100644 index 000000000000..ac1a1580a704 --- /dev/null +++ b/src/lib_c/x86_64-freebsd/c/sys/event.cr @@ -0,0 +1,32 @@ +require "../time" + +lib LibC + EVFILT_READ = -1_i16 + EVFILT_WRITE = -2_i16 + EVFILT_TIMER = -7_i16 + EVFILT_USER = -11_i16 + + EV_ADD = 0x0001_u16 + EV_DELETE = 0x0002_u16 + EV_ONESHOT = 0x0010_u16 + EV_CLEAR = 0x0020_u16 + EV_EOF = 0x8000_u16 + EV_ERROR = 0x4000_u16 + + NOTE_NSECONDS = 0x00000008_u32 + NOTE_FFCOPY = 0xc0000000_u32 + NOTE_TRIGGER = 0x01000000_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : Short + flags : UShort + fflags : UInt + data : Int64 + udata : Void* + ext : UInt64[4] + end + + fun kqueue1(flags : Int) : Int + fun kevent(kq : Int, changelist : Kevent*, nchanges : Int, eventlist : Kevent*, nevents : Int, timeout : Timespec*) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/event.cr b/src/lib_c/x86_64-netbsd/c/sys/event.cr new file mode 100644 index 000000000000..ce84b00267af --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/event.cr @@ -0,0 +1,32 @@ +require "../time" + +lib LibC + EVFILT_READ = 0_u32 + EVFILT_WRITE = 1_u32 + EVFILT_TIMER = 6_u32 + EVFILT_USER = 8_u32 + + EV_ADD = 0x0001_u32 + EV_DELETE = 0x0002_u32 + EV_ONESHOT = 0x0010_u32 + EV_CLEAR = 0x0020_u32 + EV_EOF = 0x8000_u32 + EV_ERROR = 0x4000_u32 + + NOTE_NSECONDS = 0x00000003_u32 + NOTE_FFCOPY = 0xc0000000_u32 + NOTE_TRIGGER = 0x01000000_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : UInt32 + flags : UInt32 + fflags : UInt32 + data : Int64 + udata : Void* + ext : UInt64[4] + end + + fun kqueue1(flags : Int) : Int + fun kevent = __kevent100(kq : Int, changelist : Kevent*, nchanges : SizeT, eventlist : Kevent*, nevents : SizeT, timeout : Timespec*) : Int +end diff --git a/src/lib_c/x86_64-openbsd/c/sys/event.cr b/src/lib_c/x86_64-openbsd/c/sys/event.cr new file mode 100644 index 000000000000..b95764cb7f54 --- /dev/null +++ b/src/lib_c/x86_64-openbsd/c/sys/event.cr @@ -0,0 +1,28 @@ +require "../time" + +lib LibC + EVFILT_READ = -1_i16 + EVFILT_WRITE = -2_i16 + EVFILT_TIMER = -7_i16 + + EV_ADD = 0x0001_u16 + EV_DELETE = 0x0002_u16 + EV_ONESHOT = 0x0010_u16 + EV_CLEAR = 0x0020_u16 + EV_EOF = 0x8000_u16 + EV_ERROR = 0x4000_u16 + + NOTE_NSECONDS = 0x00000003_u32 + + struct Kevent + ident : SizeT # UintptrT + filter : Short + flags : UShort + fflags : UInt + data : Int64 + udata : Void* + end + + fun kqueue1(flags : Int) : Int + fun kevent(kq : Int, changelist : Kevent*, nchanges : Int, eventlist : Kevent*, nevents : Int, timeout : Timespec*) : Int +end From 24210084db25e29b088607cf1c1d6a3f68a85601 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 9 Sep 2024 18:02:41 +0200 Subject: [PATCH 06/56] Add C bindings for getrlimit(RLIMIT_NOFILE) --- src/lib_c/aarch64-android/c/sys/resource.cr | 11 +++++++++++ src/lib_c/aarch64-darwin/c/sys/resource.cr | 2 ++ src/lib_c/aarch64-linux-gnu/c/sys/resource.cr | 11 +++++++++++ src/lib_c/aarch64-linux-musl/c/sys/resource.cr | 11 +++++++++++ src/lib_c/arm-linux-gnueabihf/c/sys/resource.cr | 11 +++++++++++ src/lib_c/i386-linux-gnu/c/sys/resource.cr | 11 +++++++++++ src/lib_c/i386-linux-musl/c/sys/resource.cr | 11 +++++++++++ src/lib_c/x86_64-darwin/c/sys/resource.cr | 2 ++ src/lib_c/x86_64-dragonfly/c/sys/resource.cr | 11 +++++++++++ src/lib_c/x86_64-freebsd/c/sys/resource.cr | 11 +++++++++++ src/lib_c/x86_64-linux-gnu/c/sys/resource.cr | 11 +++++++++++ src/lib_c/x86_64-linux-musl/c/sys/resource.cr | 11 +++++++++++ src/lib_c/x86_64-netbsd/c/sys/resource.cr | 11 +++++++++++ src/lib_c/x86_64-openbsd/c/sys/resource.cr | 11 +++++++++++ src/lib_c/x86_64-solaris/c/sys/resource.cr | 11 +++++++++++ 15 files changed, 147 insertions(+) diff --git a/src/lib_c/aarch64-android/c/sys/resource.cr b/src/lib_c/aarch64-android/c/sys/resource.cr index c6bfe1cf2e7b..52fe82cd446a 100644 --- a/src/lib_c/aarch64-android/c/sys/resource.cr +++ b/src/lib_c/aarch64-android/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(__who : Int, __usage : RUsage*) : Int + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/aarch64-darwin/c/sys/resource.cr b/src/lib_c/aarch64-darwin/c/sys/resource.cr index daa583ac5895..4759e8c9b3e3 100644 --- a/src/lib_c/aarch64-darwin/c/sys/resource.cr +++ b/src/lib_c/aarch64-darwin/c/sys/resource.cr @@ -6,6 +6,8 @@ lib LibC rlim_max : RlimT end + RLIMIT_NOFILE = 8 + fun getrlimit(Int, Rlimit*) : Int RLIMIT_STACK = 3 diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/resource.cr b/src/lib_c/aarch64-linux-gnu/c/sys/resource.cr index a0900a4730c4..444c4ba692c8 100644 --- a/src/lib_c/aarch64-linux-gnu/c/sys/resource.cr +++ b/src/lib_c/aarch64-linux-gnu/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/aarch64-linux-musl/c/sys/resource.cr b/src/lib_c/aarch64-linux-musl/c/sys/resource.cr index daa583ac5895..b0b1dc6ec2b2 100644 --- a/src/lib_c/aarch64-linux-musl/c/sys/resource.cr +++ b/src/lib_c/aarch64-linux-musl/c/sys/resource.cr @@ -33,4 +33,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/resource.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/resource.cr index 7f550c37a622..1c2c2fb678f5 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/sys/resource.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/i386-linux-gnu/c/sys/resource.cr b/src/lib_c/i386-linux-gnu/c/sys/resource.cr index a0900a4730c4..444c4ba692c8 100644 --- a/src/lib_c/i386-linux-gnu/c/sys/resource.cr +++ b/src/lib_c/i386-linux-gnu/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/i386-linux-musl/c/sys/resource.cr b/src/lib_c/i386-linux-musl/c/sys/resource.cr index daa583ac5895..b0b1dc6ec2b2 100644 --- a/src/lib_c/i386-linux-musl/c/sys/resource.cr +++ b/src/lib_c/i386-linux-musl/c/sys/resource.cr @@ -33,4 +33,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-darwin/c/sys/resource.cr b/src/lib_c/x86_64-darwin/c/sys/resource.cr index daa583ac5895..4759e8c9b3e3 100644 --- a/src/lib_c/x86_64-darwin/c/sys/resource.cr +++ b/src/lib_c/x86_64-darwin/c/sys/resource.cr @@ -6,6 +6,8 @@ lib LibC rlim_max : RlimT end + RLIMIT_NOFILE = 8 + fun getrlimit(Int, Rlimit*) : Int RLIMIT_STACK = 3 diff --git a/src/lib_c/x86_64-dragonfly/c/sys/resource.cr b/src/lib_c/x86_64-dragonfly/c/sys/resource.cr index d52182f69bce..388b52651f21 100644 --- a/src/lib_c/x86_64-dragonfly/c/sys/resource.cr +++ b/src/lib_c/x86_64-dragonfly/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = UInt64 + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 8 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-freebsd/c/sys/resource.cr b/src/lib_c/x86_64-freebsd/c/sys/resource.cr index 7f550c37a622..6f078dda986d 100644 --- a/src/lib_c/x86_64-freebsd/c/sys/resource.cr +++ b/src/lib_c/x86_64-freebsd/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 + + alias RlimT = UInt64 + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 8 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/resource.cr b/src/lib_c/x86_64-linux-gnu/c/sys/resource.cr index a0900a4730c4..444c4ba692c8 100644 --- a/src/lib_c/x86_64-linux-gnu/c/sys/resource.cr +++ b/src/lib_c/x86_64-linux-gnu/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-linux-musl/c/sys/resource.cr b/src/lib_c/x86_64-linux-musl/c/sys/resource.cr index daa583ac5895..b0b1dc6ec2b2 100644 --- a/src/lib_c/x86_64-linux-musl/c/sys/resource.cr +++ b/src/lib_c/x86_64-linux-musl/c/sys/resource.cr @@ -33,4 +33,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 7 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-netbsd/c/sys/resource.cr b/src/lib_c/x86_64-netbsd/c/sys/resource.cr index d52182f69bce..388b52651f21 100644 --- a/src/lib_c/x86_64-netbsd/c/sys/resource.cr +++ b/src/lib_c/x86_64-netbsd/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = UInt64 + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 8 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-openbsd/c/sys/resource.cr b/src/lib_c/x86_64-openbsd/c/sys/resource.cr index 7f550c37a622..6f078dda986d 100644 --- a/src/lib_c/x86_64-openbsd/c/sys/resource.cr +++ b/src/lib_c/x86_64-openbsd/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 + + alias RlimT = UInt64 + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 8 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-solaris/c/sys/resource.cr b/src/lib_c/x86_64-solaris/c/sys/resource.cr index d52182f69bce..74f9b56f9971 100644 --- a/src/lib_c/x86_64-solaris/c/sys/resource.cr +++ b/src/lib_c/x86_64-solaris/c/sys/resource.cr @@ -22,4 +22,15 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int + + alias RlimT = ULongLong + + struct Rlimit + rlim_cur : RlimT + rlim_max : RlimT + end + + RLIMIT_NOFILE = 5 + + fun getrlimit(resource : Int, rlim : Rlimit*) : Int end From 3f34458ad350cbf6f3b64e048357141d75ff5753 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 10 Sep 2024 17:27:11 +0200 Subject: [PATCH 07/56] Add Event and related FiberChannel objects Keeps information about the event that a fiber is waiting on, can be a time event and/or an IO event. --- src/crystal/system/unix/evented/event.cr | 52 +++++++++++++++++++ .../system/unix/evented/fiber_event.cr | 31 +++++++++++ 2 files changed, 83 insertions(+) create mode 100644 src/crystal/system/unix/evented/event.cr create mode 100644 src/crystal/system/unix/evented/fiber_event.cr diff --git a/src/crystal/system/unix/evented/event.cr b/src/crystal/system/unix/evented/event.cr new file mode 100644 index 000000000000..314cf0cbd151 --- /dev/null +++ b/src/crystal/system/unix/evented/event.cr @@ -0,0 +1,52 @@ +# Information about the event that a `Fiber` is waiting on. +# +# The event can be waiting for `IO` with or without a timeout, or be a timed +# event such as sleep or a select timeout. +# +# The events can be found in different queues, for example `Timers` and/or +# `Waiters` depending on their type. +struct Crystal::Evented::Event + enum Type + IoRead + IoWrite + Sleep + SelectTimeout + end + + getter type : Type + + # The `Fiber` that is waiting on the event and that the `EventLoop` shall + # resume. + getter fiber : Fiber + + # Arena index to access the associated `PollDescriptor` when processing the + # event. + getter! gen_index : Int64? + + # The absolute time (against the monotonic clock) at which a timed event shall + # trigger. Can be nil for IO events without a timeout. + getter! wake_at : Time::Span + + # True if an IO event has timed out (we're past `#wake_at`). + getter? timed_out : Bool = false + + # The event can be added into + include PointerLinkedList::Node + + def initialize(@type : Type, @fiber, @gen_index = nil, timeout : Time::Span? = nil) + @wake_at = Time.monotonic + timeout if timeout + end + + # Mark the IO event as timed out. + def timed_out! : Bool + @timed_out = true + end + + # Manually set the absolute time (against the monotonic clock). This is meant + # for `FiberEvent` to set and cancel its inner sleep or select timeout + # (allocated once per `Fiber`). + # + # NOTE: musn't be changed after registering the event into timers! + def wake_at=(@wake_at) + end +end diff --git a/src/crystal/system/unix/evented/fiber_event.cr b/src/crystal/system/unix/evented/fiber_event.cr new file mode 100644 index 000000000000..640606a04e93 --- /dev/null +++ b/src/crystal/system/unix/evented/fiber_event.cr @@ -0,0 +1,31 @@ +class Crystal::Evented::FiberEvent + include Crystal::EventLoop::Event + + def initialize(@event_loop : EventLoop, fiber : Fiber, type : Evented::Event::Type) + @event = Evented::Event.new(type, fiber) + end + + # sleep or select timeout + def add(timeout : Time::Span) : Nil + @event.wake_at = Time.monotonic + timeout + @event_loop.add_timer(pointerof(@event)) + end + + # select timeout has been cancelled + def delete : Nil + return unless @event.wake_at? + + @event.wake_at = nil + @event_loop.delete_timer(pointerof(@event)) + end + + # fiber died + def free : Nil + delete + end + + # the timer triggered (already dequeued from eventloop) + def clear : Nil + @event.wake_at = nil + end +end From 95b64d959abc81cb9758e74c3ad9318879d327e7 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 10 Sep 2024 17:28:00 +0200 Subject: [PATCH 08/56] Add PollDescriptor and Waiters objects Keeps waiting reader and writer events for an IO. The event themselves keep the information about the event and the associated fiber. --- .../system/unix/evented/poll_descriptor.cr | 51 +++++++++++++++++ src/crystal/system/unix/evented/waiters.cr | 57 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 src/crystal/system/unix/evented/poll_descriptor.cr create mode 100644 src/crystal/system/unix/evented/waiters.cr diff --git a/src/crystal/system/unix/evented/poll_descriptor.cr b/src/crystal/system/unix/evented/poll_descriptor.cr new file mode 100644 index 000000000000..cc62a5c146ad --- /dev/null +++ b/src/crystal/system/unix/evented/poll_descriptor.cr @@ -0,0 +1,51 @@ +require "./waiters" + +# Information related to the evloop for a fd, such as the read and write queues +# (waiting `Event`), as well as which evloop instance currently owns the fd. +# +# Thread-safe: mutations are protected with a lock. +struct Crystal::Evented::PollDescriptor + @event_loop : Evented::EventLoop? + @lock = SpinLock.new + @readers = Waiters.new + @writers = Waiters.new + + # Makes *event_loop* the new owner of *fd*. + # Removes *fd* from the current event loop (if any). + def take_ownership(event_loop : EventLoop, fd : Int32, gen_index : Int64) : Nil + @lock.sync do + current = @event_loop + + if event_loop == current + raise "BUG: evloop already owns the poll-descriptor for fd=#{fd}" + end + + # ensure we can't have cross enqueues after we transfer the fd, so we + # can optimize (all enqueues are local) and we don't end up with a timer + # from evloop A to cancel an event from evloop B (currently unsafe) + if current && @readers.@list.empty? && @writers.@list.empty? + raise RuntimeError.new("BUG: transfering fd=#{fd} to another evloop with pending reader/writer fibers") + end + + @event_loop = event_loop + event_loop.system_add(fd, gen_index) + current.try(&.system_del(fd)) + end + end + + # Removes *fd* from the current event loop. Raises on errors. + def release(fd : Int32) : Nil + @lock.sync do + current, @event_loop = @event_loop, nil + current.try(&.system_del(fd)) + end + end + + # Same as `#release` but yields on errors. + def release(fd : Int32, &) : Nil + @lock.sync do + current, @event_loop = @event_loop, nil + current.try(&.system_del(fd) { yield }) + end + end +end diff --git a/src/crystal/system/unix/evented/waiters.cr b/src/crystal/system/unix/evented/waiters.cr new file mode 100644 index 000000000000..0ba2e9e15aab --- /dev/null +++ b/src/crystal/system/unix/evented/waiters.cr @@ -0,0 +1,57 @@ +# A FIFO queue of `Event` waiting on the same operation (either read or write) +# for a fd. See `PollDescriptor`. +# +# Thread safe: mutations are protected with a lock, and race conditions are +# handled through the ready atomic. +struct Crystal::Evented::Waiters + @ready = Atomic(Bool).new(false) + @lock = SpinLock.new + @list = PointerLinkedList(Event).new + + def add(event : Pointer(Event)) : Bool + {% if flag?(:preview_mt) %} + # we check for readyness to avoid a race condition with another thread + # running the evloop and trying to wakeup a waiting fiber while we try to + # add a waiting fiber + return false if ready? + + @lock.sync do + return false if ready? + @list.push(event) + end + {% else %} + @list.push(event) + {% end %} + + true + end + + def delete(event) : Nil + @lock.sync { @list.delete(event) } + end + + def consume_each(&) : Nil + @lock.sync do + @list.consume_each { |event| yield event } + end + end + + def ready? : Bool + @ready.swap(false, :relaxed) + end + + def ready! : Pointer(Event)? + @lock.sync do + {% if flag?(:preview_mt) %} + if event = @list.shift? + event + else + @ready.set(true, :relaxed) + nil + end + {% else %} + @list.shift? + {% end %} + end + end +end From dfa56a599291f54e714660661ef0aec340edc07d Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 10 Sep 2024 17:29:47 +0200 Subject: [PATCH 09/56] Add Timers object A simple, unoptimized, data structure to keep a list of timed events (IO timeout, sleep or select timeout). --- src/crystal/system/unix/evented/timers.cr | 72 +++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/crystal/system/unix/evented/timers.cr diff --git a/src/crystal/system/unix/evented/timers.cr b/src/crystal/system/unix/evented/timers.cr new file mode 100644 index 000000000000..0b70e7958325 --- /dev/null +++ b/src/crystal/system/unix/evented/timers.cr @@ -0,0 +1,72 @@ +# List of `Event` ordered by `Event#wake_at` ascending. Optimized for fast +# dequeue and determining when is the next timer event. +# +# Thread unsafe: parallel accesses much be protected. +# +# NOTE: this is a struct because it only wraps a const pointer to a deque +# allocated in the heap +# +# OPTIMIZE: consider a skiplist for quicker lookups + avoid memmove on `#add` +# and `#delete`. +struct Crystal::Evented::Timers + def initialize + @list = Deque(Evented::Event*).new + end + + def empty? : Bool + @list.empty? + end + + # Returns the time at which the next timer is supposed to run. + def next_ready? : Time::Span? + @list.first?.try(&.value.wake_at) + end + + # Dequeues and yields each ready timer (their `#wake_at` is lower than + # `Time.monotonic`) from the oldest to the most recent (i.e. time ascending). + def dequeue_ready(&) : Nil + return if @list.empty? + + now = Time.monotonic + n = 0 + + @list.each do |event| + break if event.value.wake_at > now + yield event + n += 1 + end + + # OPTIMIZE: consume the n entries at once + n.times { @list.shift } + end + + # Add a new timer into the list. Returns true if it is the next ready timer. + def add(event : Evented::Event*) : Bool + if @list.empty? + @list << event + true + elsif index = lookup(event.value.wake_at) + @list.insert(index, event) + index == 0 + else + @list.push(event) + false + end + end + + private def lookup(wake_at) + @list.each_with_index do |event, index| + return index if event.value.wake_at >= wake_at + end + end + + # Removes a timer from the list. Returns true if it was the next ready timer. + def delete(event : Evented::Event*) : Bool + if index = @list.index(event) + @list.delete_at(index) + index == 0 + else + false + end + end +end From 7a12a86bcac1bc781ec1830eb805a8f8970c5559 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 10 Sep 2024 17:26:13 +0200 Subject: [PATCH 10/56] Add generational arena --- src/crystal/system/unix/evented/arena.cr | 177 +++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 src/crystal/system/unix/evented/arena.cr diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/system/unix/evented/arena.cr new file mode 100644 index 000000000000..54ac180fc3e1 --- /dev/null +++ b/src/crystal/system/unix/evented/arena.cr @@ -0,0 +1,177 @@ +# Generational Arena. +# +# Allocates a `Slice` of `T` through `mmap`. `T` is supposed to be a struct, so +# it can be embedded right into the memory. +# +# The arena allocates objects `T` at a predefined index. The object iself is +# uninitialized (outside of having its memory initialized to zero). The object +# can be allocated and later retrieved using the generation index (Int64) that +# contains both the actual index (Int32) and the generation number (UInt32). +# Deallocating the object increases the generation number, which allows the +# object to be reallocated later on. Trying to retrieve the allocation using the +# generation index will fail if the generation number changed (it's a new +# allocation). +# +# This arena isn't generic as it won't keep a list of free indexes. It assumes +# that something else will maintain the uniqueness of indexes and reuse indexes +# as much as possible instead of growing. +# +# For example this arena is used to hold `Crystal::Evented::PollDescriptor` +# allocations for all the fd in a program, where the fd is used as the index. +# They're unique to the process and the OS always reuses the lowest fd numbers +# before growing. +# +# Thread safety: the memory region is pre-allocated (up to capacity) using mmap +# (virtual allocation) and pointers are never invalidated. Individual +# (de)allocations of objects are protected with a fine grained lock. +# +# Guarantees: `mmap` initializes the memory to zero, which means `T` objects are +# initialized to zero by default, then `#free` will also clear the memory, so +# the next allocation shall be initialized to zero, too. +# +# TODO: we could use a growing/shrinking buffer (realloc) though it would +# require a rwlock to borrow accesses during which we can mutate the pointed +# memory, but growing/shrinking would need exclusive write access (it +# reallocates, hence invalidate all pointers); resizing could be delayed and +# thus shouldn't happen often + borrowing accesses should be as quick/small as +# possible. +class Crystal::Evented::Arena(T) + struct Entry(T) + @lock = SpinLock.new # protects parallel allocate/free calls + property? allocated = false + property generation = 0_u32 + @object = uninitialized T + + def pointer : Pointer(T) + pointerof(@object) + end + + def free : Nil + @generation &+= 1_u32 + @allocated = false + pointer.clear(1) + end + end + + @buffer : Slice(Entry(T)) + + def initialize(capacity : Int32) + pointer = self.class.mmap(LibC::SizeT.new(sizeof(Entry(T))) * capacity) + @buffer = Slice.new(pointer.as(Pointer(Entry(T))), capacity) + end + + protected def self.mmap(bytesize) + flags = LibC::MAP_PRIVATE | LibC::MAP_ANON + prot = LibC::PROT_READ | LibC::PROT_WRITE + + pointer = LibC.mmap(nil, bytesize, prot, flags, -1, 0) + System.panic("mmap", Errno.value) if pointer == LibC::MAP_FAILED + + {% if flag?(:linux) %} + LibC.madvise(pointer, bytesize, LibC::MADV_NOHUGEPAGE) + {% end %} + + pointer + end + + def finalize + LibC.munmap(@buffer.to_unsafe, @buffer.bytesize) + end + + # Returns a pointer to the object allocated at *gen_idx* (generation index). + # + # Raises if the object isn't allocated. + # Raises if the generation has changed (i.e. the object has been freed then reallocated) + # Raises if *index* is negative. + def get(gen_idx : Int64) : Pointer(T) + index, generation = from_gen_index(gen_idx) + entry = at(index) + + unless entry.value.allocated? + raise RuntimeError.new("#{self.class.name}: object not allocated at index #{index}") + end + + unless (actual = entry.value.generation) == generation + raise RuntimeError.new("#{self.class.name}: object generation changed at index #{index} (#{generation} => #{actual})") + end + + entry.value.pointer + end + + # Yields and allocates the object at *index* unless already allocated, then + # returns a pointer to the object at *index* and the generation index. + # + # There are no generational checks. + # Raises if *index* is negative. + def allocate(index : Int32, &) : {Pointer(T), Int64} + entry = at(index) + + entry.value.@lock.sync do + pointer = entry.value.pointer + gen_index = to_gen_index(index, entry) + + unless entry.value.allocated? + entry.value.allocated = true + yield pointer, gen_index + end + + {pointer, gen_index} + end + end + + # Yields the object allocated at *index* then releases it. + # Does nothing if the object wasn't allocated. + # + # Raises if *index* is negative. + def free(index : Int32, &) : Nil + return unless entry = at?(index) + + entry.value.@lock.sync do + return unless entry.value.allocated? + + yield entry.value.pointer + entry.value.free + end + end + + private def at(index : Int32) : Pointer(Entry(T)) + if index.negative? + raise ArgumentError.new("#{self.class.name}: negative index #{index}") + end + if index >= @buffer.size + raise ArgumentError.new("#{self.class.name}: out of bounds index #{index} >= #{@buffer.size}") + end + @buffer.to_unsafe + index + end + + private def at?(index : Int32) : Pointer(Entry(T))? + if index.negative? + raise ArgumentError.new("#{self.class.name}: negative index #{index}") + end + if index < @buffer.size + @buffer.to_unsafe + index + end + end + + # Iterates all allocated objects, yields the actual index as well as the + # generation index. + def each(&) : Nil + pointer = @buffer.to_unsafe + + @buffer.size.times do |index| + entry = pointer + index + + if entry.value.allocated? + yield index, to_gen_index(index, entry) + end + end + end + + private def to_gen_index(index : Int32, entry : Pointer(Entry(T))) : Int64 + (index.to_i64! << 32) | entry.value.generation.to_u64! + end + + private def from_gen_index(gen_index : Int64) : {Int32, UInt32} + {(gen_index >> 32).to_i32!, gen_index.to_u32!} + end +end From 2363de1003052bbd8a66da432390d5b10553f4a8 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 10 Sep 2024 17:30:52 +0200 Subject: [PATCH 11/56] Add polling EventLoop (abstract base) The foundation for the system specific epoll and kqueue event loops. --- src/crystal/system/unix/evented/event_loop.cr | 506 ++++++++++++++++++ 1 file changed, 506 insertions(+) create mode 100644 src/crystal/system/unix/evented/event_loop.cr diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr new file mode 100644 index 000000000000..d5564a336692 --- /dev/null +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -0,0 +1,506 @@ +require "./*" +require "./arena" + +module Crystal::System::FileDescriptor + # user data (generation index for the arena) + property __evloop_data : Int64 = -1_i64 +end + +module Crystal::System::Socket + # user data (generation index for the arena) + property __evloop_data : Int64 = -1_i64 +end + +module Crystal::Evented + # The choice of a generational arena permits to avoid pushing raw pointers into + # IO objects into kernel data structures that are unknown to the GC, and to + # safely check whether the allocation is still valid before trying to + # dereference the pointer. Since `PollDescriptor` also doesn't have pointers to + # the actual IO object, it won't prevent the GC from collecting lost IO objects + # (and spares us from using + # + # To a lesser extent, it also allows to keep the `PollDescriptor` allocated + # together in the same region, and polluting the IO object itself with specific + # evloop data (except for the generation index). + # + # We assume the fd is unique (OS guarantee) and that the OS will always reuse + # the lowest fds before growing, so the memory region should never grow too + # big without a good reason (i.e. we need that many fds at that time). This + # assumption allows the arena to not have to keep a list of free indexes. + protected class_getter arena = Arena(PollDescriptor).new(max_fds) + + private def self.max_fds : Int32 + if LibC.getrlimit(LibC::RLIMIT_NOFILE, out rlimit) == -1 + raise RuntimeError.from_errno("getrlimit(RLIMIT_NOFILE)") + end + rlimit.rlim_max.clamp(..Int32::MAX).to_i32! + end +end + +# Polling EventLoop. +# +# This is the abstract interface that implements `Crystal::EventLoop` for +# polling based UNIX targets, such as epoll (linux), kqueue (bsd), or poll +# (posix) syscalls. This class only implements the generic parts for the +# external world to interact with the loop. A specific implementation is +# required to handle the actual syscalls. +# +# The event loop registers the fd into the kernel data structures when an IO +# operation would block, then keeps it there until the fd is closed. +# +# NOTE: the fds must have `O_NONBLOCK` set. +# +# It is possible to have multiple event loop instances, but an fd can only be in +# one instance at a time. When trying to block from another loop, the fd will be +# removed from its associated loop and added to the current one (this is +# automatic). Trying to move a fd to another loop with pending waiters is +# unsupported and will raise an exception. See `PollDescriptor#release`. +# +# A timed event such as sleep or select timeout follows the following logic: +# +# 1. create an `Event` (actually reuses it, see `FiberChannel`); +# 2. register the event in `@timers`; +# 3. supend the current fiber. +# +# The timer will eventually trigger and resume the fiber. +# When an IO operation on fd would block, the loop follows the following logic: +# +# 1. register the fd (once); +# 2. create an `Event`; +# 3. suspend the current fiber; +# +# When the IO operation is ready, the fiber will eventually be resumed (one +# fiber at a time). If it's an IO operation, the operation is tried again which +# may block again, until the operation succeeds or an error occured (e.g. +# closed, broken pipe). +# +# If the IO operation has a timeout, the event is also registered into `@timers` +# before suspending the fiber, then after resume it will raise +# `IO::TimeoutError` if the event timed out, and continue otherwise. +# +# OPTIMIZE: collect fibers & canceled timers, delete canceled timers when +# processing timers, and eventually enqueue all fibers; it would avoid repeated +# lock/unlock timers on each #resume_io and allow to replace individual fiber +# enqueues with a single batch enqueue (simpler). +abstract class Crystal::Evented::EventLoop < Crystal::EventLoop + {% if flag?(:preview_mt) %} + @run_lock = Atomic::Flag.new # protects parallel runs + {% end %} + + def initialize + @lock = SpinLock.new # protects parallel accesses to @timers + @timers = Timers.new + end + + # reset the mutexes since another thread may have acquired the lock of one + # event loop, which would prevent closing file descriptors for example. + def after_fork_before_exec : Nil + {% if flag?(:preview_mt) %} @run_lock.clear {% end %} + @lock = SpinLock.new + end + + {% unless flag?(:preview_mt) %} + # no parallelism issues, but let's clean-up anyway + def after_fork : Nil + {% if flag?(:preview_mt) %} @run_lock.clear {% end %} + @lock = SpinLock.new + end + {% end %} + + # thread unsafe: must hold `@run_mutex` before calling! + def run(blocking : Bool) : Bool + system_run(blocking) + true + end + + def try_lock?(&) : Bool + {% if flag?(:preview_mt) %} + if @run_lock.test_and_set + begin + yield + true + ensure + @run_lock.clear + end + else + false + end + {% else %} + yield + true + {% end %} + end + + def try_run?(blocking : Bool) : Bool + try_lock? { run(blocking) } + end + + # fiber interface, see Crystal::EventLoop + + def create_resume_event(fiber : Fiber) : FiberEvent + FiberEvent.new(self, fiber, :sleep) + end + + def create_timeout_event(fiber : Fiber) : FiberEvent + FiberEvent.new(self, fiber, :select_timeout) + end + + # file descriptor interface, see Crystal::EventLoop::FileDescriptor + + def read(file_descriptor : System::FileDescriptor, slice : Bytes) : Int32 + size = evented_read(file_descriptor, slice, file_descriptor.@read_timeout) + + if size == -1 + if Errno.value == Errno::EBADF + raise IO::Error.new("File not open for reading", target: file_descriptor) + else + raise IO::Error.from_errno("read", target: file_descriptor) + end + else + size.to_i32 + end + end + + def write(file_descriptor : System::FileDescriptor, slice : Bytes) : Int32 + size = evented_write(file_descriptor, slice, file_descriptor.@write_timeout) + + if size == -1 + if Errno.value == Errno::EBADF + raise IO::Error.new("File not open for writing", target: file_descriptor) + else + raise IO::Error.from_errno("write", target: file_descriptor) + end + else + size.to_i32 + end + end + + def close(file_descriptor : System::FileDescriptor) : Nil + evented_close(file_descriptor) + end + + def remove(file_descriptor : System::FileDescriptor) : Nil + Evented.arena.free(file_descriptor.fd) do |pd| + pd.value.release(file_descriptor.fd) { } # ignore system error + end + end + + # socket interface, see Crystal::EventLoop::Socket + + def read(socket : ::Socket, slice : Bytes) : Int32 + size = evented_read(socket, slice, socket.@read_timeout) + raise IO::Error.from_errno("read", target: socket) if size == -1 + size + end + + def write(socket : ::Socket, slice : Bytes) : Int32 + size = evented_write(socket, slice, socket.@write_timeout) + raise IO::Error.from_errno("write", target: socket) if size == -1 + size + end + + def accept(socket : ::Socket) : ::Socket::Handle? + loop do + client_fd = + {% if LibC.has_method?(:accept4) %} + LibC.accept4(socket.fd, nil, nil, LibC::SOCK_CLOEXEC) + {% else %} + # we may fail to set FD_CLOEXEC between `accept` and `fcntl` but we + # can't call `Crystal::System::Socket.lock_read` because the socket + # might be in blocking mode and accept would block until the socket + # receives a connection. + # + # we could lock when `socket.blocking?` is false, but another thread + # could change the socket back to blocking mode between the condition + # check and the `accept` call. + LibC.accept(socket.fd, nil, nil).tap do |fd| + System::Socket.fcntl(fd, LibC::F_SETFD, LibC::FD_CLOEXEC) unless fd == -1 + end + {% end %} + + return client_fd unless client_fd == -1 + return if socket.closed? + + if Errno.value == Errno::EAGAIN + wait_readable(socket, socket.@read_timeout) do + raise IO::TimeoutError.new("Accept timed out") + end + return if socket.closed? + else + raise ::Socket::Error.from_errno("accept") + end + end + end + + def connect(socket : ::Socket, address : ::Socket::Addrinfo | ::Socket::Address, timeout : Time::Span?) : IO::Error? + loop do + ret = LibC.connect(socket.fd, address, address.size) + return unless ret == -1 + + case Errno.value + when Errno::EISCONN + return + when Errno::EINPROGRESS, Errno::EALREADY + wait_writable(socket, timeout) do + return IO::TimeoutError.new("Connect timed out") + end + else + return ::Socket::ConnectError.from_errno("connect") + end + end + end + + def send_to(socket : ::Socket, slice : Bytes, address : ::Socket::Address) : Int32 + bytes_sent = LibC.sendto(socket.fd, slice.to_unsafe.as(Void*), slice.size, 0, address, address.size) + raise ::Socket::Error.from_errno("Error sending datagram to #{address}") if bytes_sent == -1 + bytes_sent.to_i32 + end + + def receive_from(socket : ::Socket, slice : Bytes) : {Int32, ::Socket::Address} + sockaddr = Pointer(LibC::SockaddrStorage).malloc.as(LibC::Sockaddr*) + + # initialize sockaddr with the initialized family of the socket + copy = sockaddr.value + copy.sa_family = socket.family + sockaddr.value = copy + addrlen = LibC::SocklenT.new(sizeof(LibC::SockaddrStorage)) + + loop do + size = LibC.recvfrom(socket.fd, slice, slice.size, 0, sockaddr, pointerof(addrlen)) + if size == -1 + if Errno.value == Errno::EAGAIN + wait_readable(socket, socket.@read_timeout) + check_open(socket) + else + raise IO::Error.from_errno("recvfrom", target: socket) + end + else + return {size.to_i32, ::Socket::Address.from(sockaddr, addrlen)} + end + end + end + + def close(socket : ::Socket) : Nil + evented_close(socket) + end + + def remove(socket : ::Socket) : Nil + Evented.arena.free(socket.fd) do |pd| + pd.value.release(socket.fd) { } # ignore system error + end + end + + # internals: IO + + private def evented_read(io, slice : Bytes, timeout : Time::Span?) : Int32 + loop do + ret = LibC.read(io.fd, slice, slice.size) + if ret == -1 && Errno.value == Errno::EAGAIN + wait_readable(io, timeout) + check_open(io) + else + return ret.to_i + end + end + end + + private def evented_write(io, slice : Bytes, timeout : Time::Span?) : Int32 + loop do + ret = LibC.write(io.fd, slice, slice.size) + if ret == -1 && Errno.value == Errno::EAGAIN + wait_writable(io, timeout) + check_open(io) + else + return ret.to_i + end + end + end + + protected def evented_close(io) + Evented.arena.free(io.fd) do |pd| + pd.value.@readers.consume_each do |event| + pd.value.@event_loop.try(&.resume_io(event)) + end + + pd.value.@writers.consume_each do |event| + pd.value.@event_loop.try(&.resume_io(event)) + end + + pd.value.release(io.fd) + io.__evloop_data = -1_i64 + end + end + + private def wait_readable(io, timeout = nil) : Nil + wait(:io_read, io, :readers, timeout) { raise IO::TimeoutError.new("Read timed out") } + end + + private def wait_readable(io, timeout = nil, &) : Nil + wait(:io_read, io, :readers, timeout) { yield } + end + + private def wait_writable(io, timeout = nil) : Nil + wait(:io_write, io, :writers, timeout) { raise IO::TimeoutError.new("Write timed out") } + end + + private def wait_writable(io, timeout = nil, &) : Nil + wait(:io_write, io, :writers, timeout) { yield } + end + + private macro wait(type, io, waiters, timeout, &) + # get or allocate the poll descriptor + if (%gen_index = {{io}}.__evloop_data) >= 0 + %pd = Evented.arena.get(%gen_index) + else + %pd, %gen_index = Evented.arena.allocate({{io}}.fd) do |pd, gen_index| + # register the fd with the event loop (once), it should usually merely add + # the fd to the current evloop but may "transfer" the ownership from + # another event loop: + {{io}}.__evloop_data = gen_index + pd.value.take_ownership(self, {{io}}.fd, gen_index) + end + end + + # create an event (on the stack) + %event = Evented::Event.new({{type}}, Fiber.current, %gen_index, {{timeout}}) + + # try to add the event to the waiting list + # don't wait if the waiter has already been marked ready (see Waiters#add) + return unless %pd.value.@{{waiters.id}}.add(pointerof(%event)) + + if %event.wake_at? + add_timer(pointerof(%event)) + + Fiber.suspend + + if %event.timed_out? + return {{yield}} + else + # nothing to do: either the timer triggered which means it was dequeued, + # or `#resume_io` was called to resume the IO and the timer got deleted + # from the timers before the fiber got reenqueued. + # + # TODO: consider a quick check to verify whether the event is still + # queued and panic when it happens: the event is put on the stack and we + # can't access it after this method returns! + end + else + Fiber.suspend + end + + {% if flag?(:preview_mt) %} + # we can safely reset readyness here, since we're about to retry the + # actual syscall + %pd.value.@{{waiters.id}}.@ready.set(false, :relaxed) + {% end %} + end + + private def check_open(io : IO) + raise IO::Error.new("Closed stream") if io.closed? + end + + # internals: timers + + protected def add_timer(event : Evented::Event*) + @lock.sync do + is_next_ready = @timers.add(event) + system_set_timer(event.value.wake_at) if is_next_ready + end + end + + protected def delete_timer(event : Evented::Event*) + @lock.sync do + was_next_ready = @timers.delete(event) + system_set_timer(@timers.next_ready?) if was_next_ready + end + end + + # Helper to resume the fiber associated to an IO event and remove the event + # from timers if applicable. + protected def resume_io(event : Evented::Event*) : Nil + delete_timer(event) if event.value.wake_at? + Crystal::Scheduler.enqueue(event.value.fiber) + end + + # Process ready timers. + # + # Shall be called after processing IO events. IO events with a timeout that + # have succeeded shall already have been removed from `@timers` otherwise the + # fiber could be resumed twice! + # + # OPTIMIZE: collect events with the lock then process them after releasing the + # lock, which should be thread-safe as long as @run_lock is locked. + private def process_timers(timer_triggered : Bool) : Nil + # events = PointerLinkedList(Event).new + size = 0 + + @lock.sync do + @timers.dequeue_ready do |event| + # events << event + process_timer(event) + size += 1 + end + + unless size == 0 && timer_triggered + system_set_timer(@timers.next_ready?) + end + end + + # events.each { |event| process_timer(event) } + end + + private def process_timer(event : Evented::Event*) + fiber = event.value.fiber + + case event.value.type + when .io_read? + # reached read timeout: cancel io event + event.value.timed_out! + pd = Evented.arena.get(event.value.gen_index) + pd.value.@readers.delete(event) + when .io_write? + # reached write timeout: cancel io event + event.value.timed_out! + pd = Evented.arena.get(event.value.gen_index) + pd.value.@writers.delete(event) + when .select_timeout? + # always dequeue the event but only enqueue the fiber if we win the + # atomic CAS + return unless select_action = fiber.timeout_select_action + fiber.timeout_select_action = nil + return unless select_action.time_expired? + fiber.@timeout_event.as(FiberEvent).clear + when .sleep? + # cleanup + fiber.@resume_event.as(FiberEvent).clear + else + raise RuntimeError.new("BUG: unexpected event in timers: #{event.value}%s\n") + end + + Crystal::Scheduler.enqueue(fiber) + end + + # internals: system + + # Process ready events and timers. + # + # The loop must always process ready events and timers before returning. When + # *blocking* is `true` the loop must wait for events to become ready (possibly + # indefinitely); when `false` the loop shall return immediately. + # + # The `PollDescriptor` of IO events can be retrieved using the *gen_index* + # from the system event's user data. + private abstract def system_run(blocking : Bool) : Nil + + # Add *fd* to the polling system, setting *gen_index* as user data. + protected abstract def system_add(fd : Int32, gen_index : Int64) : Nil + + # Remove *fd* from the polling system. Must raise a `RuntimeError` on error. + protected abstract def system_del(fd : Int32) : Nil + + # Remove *fd* from the polling system. Must yield on error. + protected abstract def system_del(fd : Int32, &) : Nil + + # Arm a timer to interrupt a run at *time*. Set to `nil` to disarm the timer. + private abstract def system_set_timer(time : Time::Span?) : Nil +end From ff85b3a2e2a1e756ce4351c4ead1634adb45b132 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 10 Sep 2024 17:31:42 +0200 Subject: [PATCH 12/56] Add epoll EventLoop (Linux, Android) Specific to Linux and Android. It might be working on Solaris too through their Linux compatibility layer. --- src/crystal/system/unix/epoll/event_loop.cr | 151 ++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 src/crystal/system/unix/epoll/event_loop.cr diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr new file mode 100644 index 000000000000..e7540d065ec3 --- /dev/null +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -0,0 +1,151 @@ +require "../evented/event_loop" +require "../epoll" +require "../eventfd" +require "../timerfd" + +class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop + def initialize + super + + # the epoll instance + @epoll = System::Epoll.new + + # notification to interrupt a run + @interrupted = Atomic::Flag.new + @eventfd = System::EventFD.new + @epoll.add(@eventfd.fd, LibC::EPOLLIN, u64: @eventfd.fd.to_u64!) + + # we use timerfd to go below the millisecond precision of epoll_wait; it + # also allows to avoid locking timers before every epoll_wait call + @timerfd = System::TimerFD.new + @epoll.add(@timerfd.fd, LibC::EPOLLIN, u64: @timerfd.fd.to_u64!) + end + + def after_fork_before_exec : Nil + super + + # O_CLOEXEC would close these automatically, but we don't want to mess with + # the parent process fds (it would mess the parent evloop) + @epoll.close + @eventfd.close + @timerfd.close + end + + {% unless flag?(:preview_mt) %} + def after_fork : Nil + super + + # close inherited fds + @epoll.close + @eventfd.close + @timerfd.close + + # create new fds + @epoll = System::Epoll.new + + @interrupted.clear + @eventfd = System::EventFD.new + @epoll.add(@eventfd.fd, LibC::EPOLLIN, u64: @eventfd.fd.to_u64!) + + @timerfd = System::TimerFD.new + @epoll.add(@timerfd.fd, LibC::EPOLLIN, u64: @timerfd.fd.to_u64!) + system_set_timer(@timers.next_ready?) + + # re-add all registered fds + Evented.arena.each { |fd, gen_index| system_add(fd, gen_index) } + end + {% end %} + + private def system_run(blocking : Bool) : Nil + Crystal.trace :evloop, "wait", blocking: blocking ? 1 : 0 + + # wait for events (indefinitely when blocking) + buffer = uninitialized LibC::EpollEvent[128] + epoll_events = @epoll.wait(buffer.to_slice, timeout: blocking ? -1 : 0) + + timer_triggered = false + + # process events + epoll_events.size.times do |i| + epoll_event = epoll_events.to_unsafe + i + + case epoll_event.value.data.u64 + when @eventfd.fd + # TODO: panic if epoll_event.value.events != LibC::EPOLLIN (could be EPOLLERR or EPLLHUP) + Crystal.trace :evloop, "interrupted" + @eventfd.read + # OPTIMIZE: only reset interrupted before a blocking wait + @interrupted.clear + when @timerfd.fd + # TODO: panic if epoll_event.value.events != LibC::EPOLLIN (could be EPOLLERR or EPLLHUP) + Crystal.trace :evloop, "timer" + timer_triggered = true + else + process(epoll_event) + end + end + + process_timers(timer_triggered) + end + + private def process(epoll_event : LibC::EpollEvent*) : Nil + gen_index = epoll_event.value.data.u64.unsafe_as(Int64) + events = epoll_event.value.events + + {% if flag?(:tracing) %} + fd = (gen_index >> 32).to_i32! + Crystal.trace :evloop, "event", fd: fd, gen_index: gen_index, events: events + {% end %} + + pd = Evented.arena.get(gen_index) + + if (events & (LibC::EPOLLERR | LibC::EPOLLHUP)) != 0 + pd.value.@readers.consume_each { |event| resume_io(event) } + pd.value.@writers.consume_each { |event| resume_io(event) } + return + end + + if (events & LibC::EPOLLRDHUP) == LibC::EPOLLRDHUP + pd.value.@readers.consume_each { |event| resume_io(event) } + elsif (events & LibC::EPOLLIN) == LibC::EPOLLIN + if event = pd.value.@readers.ready! + resume_io(event) + end + end + + if (events & LibC::EPOLLOUT) == LibC::EPOLLOUT + if event = pd.value.@writers.ready! + resume_io(event) + end + end + end + + def interrupt : Nil + # the atomic makes sure we only write once + @eventfd.write(1) if @interrupted.test_and_set + end + + protected def system_add(fd : Int32, gen_index : Int64) : Nil + Crystal.trace :evloop, "epoll_ctl", op: "add", fd: fd, gen_index: gen_index + events = LibC::EPOLLIN | LibC::EPOLLOUT | LibC::EPOLLRDHUP | LibC::EPOLLET + @epoll.add(fd, events, u64: gen_index.unsafe_as(UInt64)) + end + + protected def system_del(fd : Int32) : Nil + Crystal.trace :evloop, "epoll_ctl", op: "del", fd: fd + @epoll.delete(fd) + end + + protected def system_del(fd : Int32, &) : Nil + Crystal.trace :evloop, "epoll_ctl", op: "del", fd: fd + @epoll.delete(fd) { yield } + end + + private def system_set_timer(time : Time::Span?) : Nil + if time + @timerfd.set(time) + else + @timerfd.cancel + end + end +end From 20adb88d52c19f5ce14dbf562fb07065ca5f2240 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 10 Sep 2024 17:33:30 +0200 Subject: [PATCH 13/56] Add kqueue event loop (BSD) For BSD and Darwin. --- src/crystal/system/unix/kqueue/event_loop.cr | 230 +++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 src/crystal/system/unix/kqueue/event_loop.cr diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr new file mode 100644 index 000000000000..0c3b0305030f --- /dev/null +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -0,0 +1,230 @@ +require "../evented/event_loop" +require "../kqueue" + +class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop + INTERRUPT_IDENTIFIER = 9 + + {% unless LibC.has_constant?(:EVFILT_USER) %} + @pipe = uninitialized LibC::Int[2] + {% end %} + + def initialize + super + + # the kqueue instance + @kqueue = System::Kqueue.new + + # notification to interrupt a run + @interrupted = Atomic::Flag.new + {% unless LibC.has_constant?(:EVFILT_USER) %} + @pipe = System::FileDescriptor.system_pipe + @kqueue.kevent(@pipe[0], LibC::EVFILT_READ, LibC::EV_ADD) do + raise RuntimeError.from_errno("kevent") + end + {% end %} + end + + def after_fork_before_exec : Nil + super + + # O_CLOEXEC would close these automatically but we don't want to mess with + # the parent process fds (that would mess the parent evloop) + + {% unless flag?(:darwin) || flag?(:dragonfly) %} + # kqueue isn't inherited by fork on darwin/dragonfly, but is inherited on + # other BSD + @kqueue.close + {% end %} + + {% unless LibC.has_constant?(:EVFILT_USER) %} + @pipe.each { |fd| LibC.close(fd) } + {% end %} + end + + {% unless flag?(:preview_mt) %} + def after_fork : Nil + super + + {% unless flag?(:darwin) || flag?(:dragonfly) %} + # kqueue isn't inherited by fork on darwin/dragonfly, but is inherited + # on other BSD + @kqueue.close + {% end %} + @kqueue = System::Kqueue.new + + @interrupted.clear + {% unless LibC.has_constant?(:EVFILT_USER) %} + @pipe.each { |fd| LibC.close(fd) } + @pipe = System::FileDescriptor.system_pipe + @kqueue.kevent(@pipe[0], LibC::EVFILT_READ, LibC::EV_ADD) do + raise RuntimeError.from_errno("kevent") + end + {% end %} + + system_set_timer(@timers.next_ready?) + + # re-add all registered fds + Evented.arena.each { |fd, gen_index| system_add(fd, gen_index) } + end + {% end %} + + private def system_run(blocking : Bool) : Nil + buffer = uninitialized LibC::Kevent[128] + + Crystal.trace :evloop, "wait", blocking: blocking ? 1 : 0 + timeout = blocking ? nil : Time::Span.zero + kevents = @kqueue.wait(buffer.to_slice, timeout) + + timer_triggered = false + + # process events + kevents.size.times do |i| + kevent = kevents.to_unsafe + i + + if process_interrupt?(kevent) + # nothing special + elsif kevent.value.filter == LibC::EVFILT_TIMER + # nothing special + timer_triggered = true + else + process(kevent) + end + end + + process_timers(timer_triggered) + end + + private def process_interrupt?(kevent) + {% if LibC.has_constant?(:EVFILT_USER) %} + if kevent.value.filter == LibC::EVFILT_USER + @interrupted.clear if kevent.value.ident == INTERRUPT_IDENTIFIER + return true + end + {% else %} + if kevent.value.filter == LibC::EVFILT_READ && kevent.value.ident == @pipe[0] + @interrupted.clear + byte = 0_u8 + ret = LibC.read(@pipe[0], pointerof(byte), 1) + raise RuntimeError.from_errno("read") if ret == -1 + return true + end + {% end %} + false + end + + private def process(kevent : LibC::Kevent*) : Nil + gen_index = kevent.value.udata.address.to_i64! + + {% if flag?(:tracing) %} + fd = kevent.value.ident + Crystal.trace :evloop, "event", fd: fd, gen_index: gen_index, + filter: kevent.value.filter, flags: kevent.value.flags, fflags: kevent.value.fflags + {% end %} + + pd = Evented.arena.get(gen_index) + + if (kevent.value.fflags & LibC::EV_EOF) == LibC::EV_EOF + # apparently some systems may report EOF on write with EVFILT_READ instead + # of EVFILT_WRITE, so let's wake all waiters: + pd.value.@readers.consume_each { |event| resume_io(event) } + pd.value.@writers.consume_each { |event| resume_io(event) } + return + end + + case kevent.value.filter + when LibC::EVFILT_READ + if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR + # OPTIMIZE: pass errno (kevent.data) through PollDescriptor + pd.value.@readers.consume_each { |event| resume_io(event) } + elsif event = pd.value.@readers.ready! + resume_io(event) + end + when LibC::EVFILT_WRITE + if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR + # OPTIMIZE: pass errno (kevent.data) through PollDescriptor + pd.value.@writers.consume_each { |event| resume_io(event) } + elsif event = pd.value.@writers.ready! + resume_io(event) + end + end + end + + def interrupt : Nil + return unless @interrupted.test_and_set + + {% if LibC.has_constant?(:EVFILT_USER) %} + @kqueue.kevent( + INTERRUPT_IDENTIFIER, + LibC::EVFILT_USER, + LibC::EV_ADD | LibC::EV_ONESHOT, + LibC::NOTE_FFCOPY | LibC::NOTE_TRIGGER | 1_u16) + {% else %} + byte = 1_u8 + ret = LibC.write(@pipe[1], pointerof(byte), sizeof(typeof(byte))) + raise RuntimeError.from_errno("write") if ret == -1 + {% end %} + end + + protected def system_add(fd : Int32, gen_index : Int64) : Nil + Crystal.trace :evloop, "kevent", op: "add", fd: fd, gen_index: gen_index + + # register both read and write events + kevents = uninitialized LibC::Kevent[2] + 2.times do |i| + kevent = kevents.to_unsafe + i + filter = i == 0 ? LibC::EVFILT_READ : LibC::EVFILT_WRITE + System::Kqueue.set(kevent, fd, filter, LibC::EV_ADD | LibC::EV_CLEAR, udata: Pointer(Void).new(gen_index.to_u64!)) + end + + @kqueue.kevent(kevents.to_slice) do + # we broadly add file descriptors to kqueue whenever we open them, but + # sometimes the other end is closed and registration can fail (e.g. + # stdio). + # + # we can safely discard these errors since further read or write to these + # file descriptors will fail with the same error and the evloop will never + # try to wait. + unless Errno.value.in?(Errno::ENODEV, Errno::EPIPE) + raise RuntimeError.from_errno("kevent") + end + end + end + + protected def system_del(fd : Int32) : Nil + # nothing to do: close(2) will do the job + end + + protected def system_del(fd : Int32, &) : Nil + # nothing to do: close(2) will do the job + end + + private def system_set_timer(time : Time::Span?) : Nil + if time + flags = LibC::EV_ADD | LibC::EV_ONESHOT | LibC::EV_CLEAR + t = time - Time.monotonic + data = + {% if LibC.has_constant?(:NOTE_NSECONDS) %} + t.total_nanoseconds.to_i64!.clamp(0..) + {% else %} + # legacy BSD (and DragonFly) only have millisecond precision + t.positive? ? t.total_milliseconds.to_i64!.clamp(1..) : 0 + {% end %} + else + flags = LibC::EV_DELETE + data = 0_u64 + end + + fflags = + {% if LibC.has_constant?(:NOTE_NSECONDS) %} + LibC::NOTE_NSECONDS + {% else %} + 0 + {% end %} + + # use the evloop address as the unique identifier for the timer kevent + ident = LibC::SizeT.new!(self.as(Void*).address) + @kqueue.kevent(ident, LibC::EVFILT_TIMER, flags, fflags, data) do + raise RuntimeError.from_errno("kevent") unless Errno.value == Errno::ENOENT + end + end +end From 73412bed1d94eb01f639a0b8a26c4d8111ead276 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 10 Sep 2024 18:12:29 +0200 Subject: [PATCH 14/56] Conditionnaly load IO::Evented This is now only required for the libevent event loop, and the wasi pseudo event loop. --- src/crystal/system/unix/file_descriptor.cr | 5 +++-- src/crystal/system/unix/socket.cr | 5 +++-- src/io/evented.cr | 5 +++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index 60515b701136..4aa1ec580d32 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -1,5 +1,4 @@ require "c/fcntl" -require "io/evented" require "termios" {% if flag?(:android) && LibC::ANDROID_API < 28 %} require "c/sys/ioctl" @@ -7,7 +6,9 @@ require "termios" # :nodoc: module Crystal::System::FileDescriptor - include IO::Evented + {% if IO.has_constant?(:Evented) %} + include IO::Evented + {% end %} # Platform-specific type to represent a file descriptor handle to the operating # system. diff --git a/src/crystal/system/unix/socket.cr b/src/crystal/system/unix/socket.cr index 7c39e140849c..4aa0fc2f0af9 100644 --- a/src/crystal/system/unix/socket.cr +++ b/src/crystal/system/unix/socket.cr @@ -1,10 +1,11 @@ require "c/netdb" require "c/netinet/tcp" require "c/sys/socket" -require "io/evented" module Crystal::System::Socket - include IO::Evented + {% if IO.has_constant?(:Evented) %} + include IO::Evented + {% end %} alias Handle = Int32 diff --git a/src/io/evented.cr b/src/io/evented.cr index d2b3a66c336f..358192a66955 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -1,5 +1,10 @@ {% skip_file if flag?(:win32) %} +{% unless flag?(:evloop_libevent) %} + {% skip_file if flag?(:android) || flag?(:linux) || flag?(:solaris) %} + {% skip_file if flag?(:bsd) || flag?(:darwin) %} +{% end %} + require "crystal/thread_local_value" # :nodoc: From dceb184edfaebb3e5eec0f0b92131a81986192b9 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 10 Sep 2024 17:41:05 +0200 Subject: [PATCH 15/56] Enable the epoll/kqueue event loop Also includes the `:evloop_libevent` flag to fallback to libevent. --- src/crystal/system/event_loop.cr | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/crystal/system/event_loop.cr b/src/crystal/system/event_loop.cr index fe973ec8c99e..3b5e1b4543b0 100644 --- a/src/crystal/system/event_loop.cr +++ b/src/crystal/system/event_loop.cr @@ -4,7 +4,15 @@ abstract class Crystal::EventLoop {% if flag?(:wasi) %} Crystal::Wasi::EventLoop.new {% elsif flag?(:unix) %} - Crystal::LibEvent::EventLoop.new + {% if flag?(:evloop_libevent) %} + Crystal::LibEvent::EventLoop.new + {% elsif flag?(:android) || flag?(:linux) || flag?(:solaris) %} + Crystal::Epoll::EventLoop.new + {% elsif flag?(:bsd) || flag?(:darwin) %} + Crystal::Kqueue::EventLoop.new + {% else %} + Crystal::LibEvent::EventLoop.new + {% end %} {% elsif flag?(:win32) %} Crystal::IOCP::EventLoop.new {% else %} @@ -78,7 +86,15 @@ end {% if flag?(:wasi) %} require "./wasi/event_loop" {% elsif flag?(:unix) %} - require "./unix/event_loop_libevent" + {% if flag?(:evloop_libevent) %} + require "./unix/event_loop_libevent" + {% elsif flag?(:android) || flag?(:linux) || flag?(:solaris) %} + require "./unix/epoll/event_loop" + {% elsif flag?(:bsd) || flag?(:darwin) %} + require "./unix/kqueue/event_loop" + {% else %} + require "./unix/event_loop_libevent" + {% end %} {% elsif flag?(:win32) %} require "./win32/event_loop_iocp" {% else %} From cf6873b891c6a5bd7e42b6cfc05fd64bfaef3bb5 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 12 Sep 2024 11:20:44 +0200 Subject: [PATCH 16/56] Fix: cleanup evloop resources before closing pipe/reopen io We directly call `#file_descriptor_close` in `System::Signal.after_fork` and `System::Process.reopen_io` but we didn't clean the event loop resources nor cleaned the generation index (`__evloop_data`) set on the IO objects. We should clean these up, so we don't reach weird states after fork. --- src/crystal/system/unix/evented/event_loop.cr | 2 ++ src/crystal/system/unix/process.cr | 1 + src/crystal/system/unix/signal.cr | 5 ++++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index d5564a336692..093e2719c46d 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -182,6 +182,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop def remove(file_descriptor : System::FileDescriptor) : Nil Evented.arena.free(file_descriptor.fd) do |pd| pd.value.release(file_descriptor.fd) { } # ignore system error + file_descriptor.__evloop_data = -1_i64 end end @@ -287,6 +288,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop def remove(socket : ::Socket) : Nil Evented.arena.free(socket.fd) do |pd| pd.value.release(socket.fd) { } # ignore system error + socket.__evloop_data = -1_i64 end end diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 420030f8ba53..5a45f3ffdb3f 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -345,6 +345,7 @@ struct Crystal::System::Process private def self.reopen_io(src_io : IO::FileDescriptor, dst_io : IO::FileDescriptor) if src_io.closed? + Crystal::EventLoop.current.remove(dst_io) dst_io.file_descriptor_close else src_io = to_real_fd(src_io) diff --git a/src/crystal/system/unix/signal.cr b/src/crystal/system/unix/signal.cr index 1d1e885fc71d..052c10282ee1 100644 --- a/src/crystal/system/unix/signal.cr +++ b/src/crystal/system/unix/signal.cr @@ -97,7 +97,10 @@ module Crystal::System::Signal # Replaces the signal pipe so the child process won't share the file # descriptors of the parent process and send it received signals. def self.after_fork - @@pipe.each(&.file_descriptor_close) + @@pipe.each do |pipe_io| + Crystal::EventLoop.current.remove(pipe_io) + pipe_io.file_descriptor_close { } + end ensure @@pipe = IO.pipe(read_blocking: false, write_blocking: true) end From a9b48c0e98e8337d5364386d33ee4be0e94f4719 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 12 Sep 2024 12:12:53 +0200 Subject: [PATCH 17/56] Fix: always close the kqueue after fork --- src/crystal/system/unix/kqueue/event_loop.cr | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index 0c3b0305030f..d6478de431e7 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -30,11 +30,8 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop # O_CLOEXEC would close these automatically but we don't want to mess with # the parent process fds (that would mess the parent evloop) - {% unless flag?(:darwin) || flag?(:dragonfly) %} - # kqueue isn't inherited by fork on darwin/dragonfly, but is inherited on - # other BSD - @kqueue.close - {% end %} + # kqueue isn't inherited by fork on darwin/dragonfly, but we still close + @kqueue.close {% unless LibC.has_constant?(:EVFILT_USER) %} @pipe.each { |fd| LibC.close(fd) } @@ -45,11 +42,8 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop def after_fork : Nil super - {% unless flag?(:darwin) || flag?(:dragonfly) %} - # kqueue isn't inherited by fork on darwin/dragonfly, but is inherited - # on other BSD - @kqueue.close - {% end %} + # kqueue isn't inherited by fork on darwin/dragonfly, but we still close + @kqueue.close @kqueue = System::Kqueue.new @interrupted.clear From eac515b86a385e61edf06aa2e1a8aa927f3d1787 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 12 Sep 2024 15:58:04 +0200 Subject: [PATCH 18/56] Fix: remember maximum index in arena to avoid OOM We iterate the arena to re-register the fds after fork, but we iterate up to the total capacity, which is the hard limit of fds the process open, which happens to be unlimited on darwin (and thus capped to Int32::MAX) which means we iterate a billion entries *and* the kernel eventually tries to allocate 96GB of RAM and we each an OOM error. This didn't reproduce on linux or freebsd that have a reasonable hard limit (around 1 million) by default and only allocated 40MB. They still wasted some memory. --- src/crystal/system/unix/evented/arena.cr | 28 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/system/unix/evented/arena.cr index 54ac180fc3e1..fd60e7030ae5 100644 --- a/src/crystal/system/unix/evented/arena.cr +++ b/src/crystal/system/unix/evented/arena.cr @@ -55,6 +55,10 @@ class Crystal::Evented::Arena(T) @buffer : Slice(Entry(T)) + {% unless flag?(:preview_mt) %} + @maximum = 0 + {% end %} + def initialize(capacity : Int32) pointer = self.class.mmap(LibC::SizeT.new(sizeof(Entry(T))) * capacity) @buffer = Slice.new(pointer.as(Pointer(Entry(T))), capacity) @@ -111,6 +115,10 @@ class Crystal::Evented::Arena(T) gen_index = to_gen_index(index, entry) unless entry.value.allocated? + {% unless flag?(:preview_mt) %} + @maximum = index if index > @maximum + {% end %} + entry.value.allocated = true yield pointer, gen_index end @@ -153,19 +161,21 @@ class Crystal::Evented::Arena(T) end end - # Iterates all allocated objects, yields the actual index as well as the - # generation index. - def each(&) : Nil - pointer = @buffer.to_unsafe + {% unless flag?(:preview_mt) %} + # Iterates all allocated objects, yields the actual index as well as the + # generation index. + def each(&) : Nil + pointer = @buffer.to_unsafe - @buffer.size.times do |index| - entry = pointer + index + 0.upto(@maximum) do |index| + entry = pointer + index - if entry.value.allocated? - yield index, to_gen_index(index, entry) + if entry.value.allocated? + yield index, to_gen_index(index, entry) + end end end - end + {% end %} private def to_gen_index(index : Int32, entry : Pointer(Entry(T))) : Int64 (index.to_i64! << 32) | entry.value.generation.to_u64! From a97069637ef5fe3861935f416cf4ee47cd765f2a Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 12 Sep 2024 17:39:58 +0200 Subject: [PATCH 19/56] fixup! Add epoll EventLoop (Linux, Android) --- src/crystal/system/unix/epoll/event_loop.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr index e7540d065ec3..e95b6e64dcd9 100644 --- a/src/crystal/system/unix/epoll/event_loop.cr +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -57,7 +57,7 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop {% end %} private def system_run(blocking : Bool) : Nil - Crystal.trace :evloop, "wait", blocking: blocking ? 1 : 0 + Crystal.trace :evloop, "run", blocking: blocking ? 1 : 0 # wait for events (indefinitely when blocking) buffer = uninitialized LibC::EpollEvent[128] From c9e8b4a9340b5a780abb5afdc0954b5627770303 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 12 Sep 2024 17:40:26 +0200 Subject: [PATCH 20/56] fixup! Add kqueue event loop (BSD) --- src/crystal/system/unix/kqueue/event_loop.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index d6478de431e7..4fd7f3a9f464 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -65,7 +65,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop private def system_run(blocking : Bool) : Nil buffer = uninitialized LibC::Kevent[128] - Crystal.trace :evloop, "wait", blocking: blocking ? 1 : 0 + Crystal.trace :evloop, "run", blocking: blocking ? 1 : 0 timeout = blocking ? nil : Time::Span.zero kevents = @kqueue.wait(buffer.to_slice, timeout) From 3c3a801b1802ac1c1c0b560cb155686410108bef Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 13 Sep 2024 10:43:52 +0200 Subject: [PATCH 21/56] Fix: rename Arena#allocate as Arena#lazy_allocate An `#allocate` method should raise when trying to allocate an already allocated object. The lazy allocate method formulates better the intent to only allocate once and that it's fine to allocate twice (as long as it's initialized only once). --- src/crystal/system/unix/evented/arena.cr | 10 +++++++--- src/crystal/system/unix/evented/event_loop.cr | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/system/unix/evented/arena.cr index fd60e7030ae5..fdcc8021a8cc 100644 --- a/src/crystal/system/unix/evented/arena.cr +++ b/src/crystal/system/unix/evented/arena.cr @@ -102,12 +102,16 @@ class Crystal::Evented::Arena(T) entry.value.pointer end - # Yields and allocates the object at *index* unless already allocated, then - # returns a pointer to the object at *index* and the generation index. + # Yields and allocates the object at *index* unless already allocated. + # Returns a pointer to the object at *index* and the generation index. + # + # Permits two threads to allocate the same object in parallel yet only allow + # one to initialize it; the other one will silently receive the pointer and + # the generation index. # # There are no generational checks. # Raises if *index* is negative. - def allocate(index : Int32, &) : {Pointer(T), Int64} + def lazy_allocate(index : Int32, &) : {Pointer(T), Int64} entry = at(index) entry.value.@lock.sync do diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 093e2719c46d..96f841a67835 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -354,7 +354,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop if (%gen_index = {{io}}.__evloop_data) >= 0 %pd = Evented.arena.get(%gen_index) else - %pd, %gen_index = Evented.arena.allocate({{io}}.fd) do |pd, gen_index| + %pd, %gen_index = Evented.arena.lazy_allocate({{io}}.fd) do |pd, gen_index| # register the fd with the event loop (once), it should usually merely add # the fd to the current evloop but may "transfer" the ownership from # another event loop: From db6fe2065a75f50e5fccbb1444e01fe2cca3c667 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 13 Sep 2024 11:00:38 +0200 Subject: [PATCH 22/56] Fix: rename PollDescriptor#release to #remove Harmonizes with the EventLoop#remove method. --- src/crystal/system/unix/evented/event_loop.cr | 8 ++++---- src/crystal/system/unix/evented/poll_descriptor.cr | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 96f841a67835..505867fa3b56 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -54,7 +54,7 @@ end # one instance at a time. When trying to block from another loop, the fd will be # removed from its associated loop and added to the current one (this is # automatic). Trying to move a fd to another loop with pending waiters is -# unsupported and will raise an exception. See `PollDescriptor#release`. +# unsupported and will raise an exception. See `PollDescriptor#remove`. # # A timed event such as sleep or select timeout follows the following logic: # @@ -181,7 +181,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop def remove(file_descriptor : System::FileDescriptor) : Nil Evented.arena.free(file_descriptor.fd) do |pd| - pd.value.release(file_descriptor.fd) { } # ignore system error + pd.value.remove(file_descriptor.fd) { } # ignore system error file_descriptor.__evloop_data = -1_i64 end end @@ -287,7 +287,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop def remove(socket : ::Socket) : Nil Evented.arena.free(socket.fd) do |pd| - pd.value.release(socket.fd) { } # ignore system error + pd.value.remove(socket.fd) { } # ignore system error socket.__evloop_data = -1_i64 end end @@ -328,7 +328,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop pd.value.@event_loop.try(&.resume_io(event)) end - pd.value.release(io.fd) + pd.value.remove(io.fd) io.__evloop_data = -1_i64 end end diff --git a/src/crystal/system/unix/evented/poll_descriptor.cr b/src/crystal/system/unix/evented/poll_descriptor.cr index cc62a5c146ad..8cdf8c7a09f9 100644 --- a/src/crystal/system/unix/evented/poll_descriptor.cr +++ b/src/crystal/system/unix/evented/poll_descriptor.cr @@ -33,16 +33,16 @@ struct Crystal::Evented::PollDescriptor end end - # Removes *fd* from the current event loop. Raises on errors. - def release(fd : Int32) : Nil + # Removes *fd* from its owner event loop. Raises on errors. + def remove(fd : Int32) : Nil @lock.sync do current, @event_loop = @event_loop, nil current.try(&.system_del(fd)) end end - # Same as `#release` but yields on errors. - def release(fd : Int32, &) : Nil + # Same as `#remove` but yields on errors. + def remove(fd : Int32, &) : Nil @lock.sync do current, @event_loop = @event_loop, nil current.try(&.system_del(fd) { yield }) From c78c5563973b77a58247ed5c6c78adebed0485a7 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 13 Sep 2024 12:05:07 +0200 Subject: [PATCH 23/56] Fix: errno handling on add/del of fd from epoll/kqueue The fd is lazily (un)registered into epoll and kqueue so we don't need to check for errnos about incompatible fds (EPERM, ENOENT, ENODEV) since the actuel operation (read, write, connect, ...) already failed or would never block in the first place (e.g. disk files). --- src/crystal/system/unix/epoll.cr | 5 +---- src/crystal/system/unix/kqueue.cr | 4 ++-- src/crystal/system/unix/kqueue/event_loop.cr | 11 +---------- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/crystal/system/unix/epoll.cr b/src/crystal/system/unix/epoll.cr index 1a6f4ac57b87..8827c40c9934 100644 --- a/src/crystal/system/unix/epoll.cr +++ b/src/crystal/system/unix/epoll.cr @@ -31,10 +31,7 @@ struct Crystal::System::Epoll def delete(fd : Int32) : Nil delete(fd) do - # OPTIMIZE: we might be able to spare the errno checks for EPERM and ENOENT - unless Errno.value.in?(Errno::EPERM, Errno::ENOENT) - raise RuntimeError.from_errno("epoll_ctl(EPOLL_CTL_DEL)") - end + raise RuntimeError.from_errno("epoll_ctl(EPOLL_CTL_DEL)") end end diff --git a/src/crystal/system/unix/kqueue.cr b/src/crystal/system/unix/kqueue.cr index 68f95ebba36f..28145bf1ddb0 100644 --- a/src/crystal/system/unix/kqueue.cr +++ b/src/crystal/system/unix/kqueue.cr @@ -40,9 +40,9 @@ struct Crystal::System::Kqueue end # Helper to register multiple *changes*. Returns immediately. - def kevent(changes : Slice(LibC::Kevent)) : Nil + def kevent(changes : Slice(LibC::Kevent), &) : Nil ret = LibC.kevent(@kq, changes.to_unsafe, changes.size, nil, 0, nil) - yield if ret == -1 # && !Errno.value.in?(Errno::ENOENT, Errno::EPIPE, Errno::ENODEV) + yield if ret == -1 end # Helper to wait for registered events to become active. Returns a subslice to diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index 4fd7f3a9f464..d20a7fc87ed8 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -171,16 +171,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop end @kqueue.kevent(kevents.to_slice) do - # we broadly add file descriptors to kqueue whenever we open them, but - # sometimes the other end is closed and registration can fail (e.g. - # stdio). - # - # we can safely discard these errors since further read or write to these - # file descriptors will fail with the same error and the evloop will never - # try to wait. - unless Errno.value.in?(Errno::ENODEV, Errno::EPIPE) - raise RuntimeError.from_errno("kevent") - end + raise RuntimeError.from_errno("kevent") end end From fd73eb2f6cb0d5c849bc0f5e5ed1a00c8ddb8aec Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 13 Sep 2024 12:22:45 +0200 Subject: [PATCH 24/56] Fix: fd transfer on kqueue (BSD, Darwin) --- src/crystal/system/unix/epoll/event_loop.cr | 4 ++-- src/crystal/system/unix/evented/event_loop.cr | 11 ++++++--- .../system/unix/evented/poll_descriptor.cr | 2 +- src/crystal/system/unix/kqueue/event_loop.cr | 24 +++++++++++++++---- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr index e95b6e64dcd9..e22e4db855fd 100644 --- a/src/crystal/system/unix/epoll/event_loop.cr +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -131,12 +131,12 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop @epoll.add(fd, events, u64: gen_index.unsafe_as(UInt64)) end - protected def system_del(fd : Int32) : Nil + protected def system_del(fd : Int32, closing = true) : Nil Crystal.trace :evloop, "epoll_ctl", op: "del", fd: fd @epoll.delete(fd) end - protected def system_del(fd : Int32, &) : Nil + protected def system_del(fd : Int32, closing = true, &) : Nil Crystal.trace :evloop, "epoll_ctl", op: "del", fd: fd @epoll.delete(fd) { yield } end diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 505867fa3b56..ed000c707184 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -498,10 +498,15 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop protected abstract def system_add(fd : Int32, gen_index : Int64) : Nil # Remove *fd* from the polling system. Must raise a `RuntimeError` on error. - protected abstract def system_del(fd : Int32) : Nil + # + # If *closing* is true, then it preceeds a call to `close(2)`. Some + # implementations may take advantage of close doing the book keeping. + # + # If *closing* is false then the fd must be deleted from the polling system. + protected abstract def system_del(fd : Int32, closing = true) : Nil - # Remove *fd* from the polling system. Must yield on error. - protected abstract def system_del(fd : Int32, &) : Nil + # Identical to `#system_del` but yields on error. + protected abstract def system_del(fd : Int32, closing = true, &) : Nil # Arm a timer to interrupt a run at *time*. Set to `nil` to disarm the timer. private abstract def system_set_timer(time : Time::Span?) : Nil diff --git a/src/crystal/system/unix/evented/poll_descriptor.cr b/src/crystal/system/unix/evented/poll_descriptor.cr index 8cdf8c7a09f9..97a60190e54e 100644 --- a/src/crystal/system/unix/evented/poll_descriptor.cr +++ b/src/crystal/system/unix/evented/poll_descriptor.cr @@ -29,7 +29,7 @@ struct Crystal::Evented::PollDescriptor @event_loop = event_loop event_loop.system_add(fd, gen_index) - current.try(&.system_del(fd)) + current.try(&.system_del(fd, closing: false)) end end diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index d20a7fc87ed8..9e5c137c358f 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -175,12 +175,28 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop end end - protected def system_del(fd : Int32) : Nil - # nothing to do: close(2) will do the job + protected def system_del(fd : Int32, closing = true) : Nil + system_del(fd, closing) do + raise RuntimeError.from_errno("kevent") + end end - protected def system_del(fd : Int32, &) : Nil - # nothing to do: close(2) will do the job + protected def system_del(fd : Int32, closing = true, &) : Nil + return if closing # nothing to do: close(2) will do the cleanup + + Crystal.trace :evloop, "kevent", op: "del", fd: fd + + # unregister both read and write events + kevents = uninitialized LibC::Kevent[2] + 2.times do |i| + kevent = kevents.to_unsafe + i + filter = i == 0 ? LibC::EVFILT_READ : LibC::EVFILT_WRITE + System::Kqueue.set(kevent, fd, filter, LibC::EV_DELETE) + end + + @kqueue.kevent(kevents.to_slice) do + raise RuntimeError.from_errno("kevent") + end end private def system_set_timer(time : Time::Span?) : Nil From 5a0d09b0f2fe839c9ea5974f88296cff82d8f460 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 13 Sep 2024 15:06:09 +0200 Subject: [PATCH 25/56] Improve documentation a bit --- src/crystal/system/unix/evented/arena.cr | 6 +++--- src/crystal/system/unix/evented/event.cr | 20 +++++++++--------- src/crystal/system/unix/evented/event_loop.cr | 21 ++++++++++++------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/system/unix/evented/arena.cr index fdcc8021a8cc..f467b206c90d 100644 --- a/src/crystal/system/unix/evented/arena.cr +++ b/src/crystal/system/unix/evented/arena.cr @@ -1,7 +1,7 @@ # Generational Arena. # # Allocates a `Slice` of `T` through `mmap`. `T` is supposed to be a struct, so -# it can be embedded right into the memory. +# it can be embedded right into the memory region. # # The arena allocates objects `T` at a predefined index. The object iself is # uninitialized (outside of having its memory initialized to zero). The object @@ -32,7 +32,7 @@ # TODO: we could use a growing/shrinking buffer (realloc) though it would # require a rwlock to borrow accesses during which we can mutate the pointed # memory, but growing/shrinking would need exclusive write access (it -# reallocates, hence invalidate all pointers); resizing could be delayed and +# reallocates, hence invalidates all pointers); resizing could be delayed and # thus shouldn't happen often + borrowing accesses should be as quick/small as # possible. class Crystal::Evented::Arena(T) @@ -85,7 +85,7 @@ class Crystal::Evented::Arena(T) # Returns a pointer to the object allocated at *gen_idx* (generation index). # # Raises if the object isn't allocated. - # Raises if the generation has changed (i.e. the object has been freed then reallocated) + # Raises if the generation has changed (i.e. the object has been freed then reallocated). # Raises if *index* is negative. def get(gen_idx : Int64) : Pointer(T) index, generation = from_gen_index(gen_idx) diff --git a/src/crystal/system/unix/evented/event.cr b/src/crystal/system/unix/evented/event.cr index 314cf0cbd151..00b9ff403f0b 100644 --- a/src/crystal/system/unix/evented/event.cr +++ b/src/crystal/system/unix/evented/event.cr @@ -1,7 +1,7 @@ # Information about the event that a `Fiber` is waiting on. # # The event can be waiting for `IO` with or without a timeout, or be a timed -# event such as sleep or a select timeout. +# event such as sleep or a select timeout (without IO). # # The events can be found in different queues, for example `Timers` and/or # `Waiters` depending on their type. @@ -19,18 +19,18 @@ struct Crystal::Evented::Event # resume. getter fiber : Fiber - # Arena index to access the associated `PollDescriptor` when processing the - # event. + # Arena index to access the associated `PollDescriptor` when processing an IO + # event. Nil for timed events (sleep, select timeout). getter! gen_index : Int64? - # The absolute time (against the monotonic clock) at which a timed event shall - # trigger. Can be nil for IO events without a timeout. + # The absolute time, against the monotonic clock, at which a timed event shall + # trigger. Nil for IO events without a timeout. getter! wake_at : Time::Span - # True if an IO event has timed out (we're past `#wake_at`). + # True if an IO event has timed out (i.e. we're past `#wake_at`). getter? timed_out : Bool = false - # The event can be added into + # The event can be added into different lists. See `Waiters` and `Timers`. include PointerLinkedList::Node def initialize(@type : Type, @fiber, @gen_index = nil, timeout : Time::Span? = nil) @@ -43,10 +43,10 @@ struct Crystal::Evented::Event end # Manually set the absolute time (against the monotonic clock). This is meant - # for `FiberEvent` to set and cancel its inner sleep or select timeout - # (allocated once per `Fiber`). + # for `FiberEvent` to set and cancel its inner sleep or select timeout; these + # objects are allocated once per `Fiber`. # - # NOTE: musn't be changed after registering the event into timers! + # NOTE: musn't be changed after registering the event into `Timers`! def wake_at=(@wake_at) end end diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index ed000c707184..9f085366271c 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -23,10 +23,16 @@ module Crystal::Evented # together in the same region, and polluting the IO object itself with specific # evloop data (except for the generation index). # - # We assume the fd is unique (OS guarantee) and that the OS will always reuse - # the lowest fds before growing, so the memory region should never grow too - # big without a good reason (i.e. we need that many fds at that time). This - # assumption allows the arena to not have to keep a list of free indexes. + # Takes advantage of the fd being unique per process and that the operating + # system will always reuse the lowest fd (POSIX compliance) and will only grow + # when the process needs that many file descriptors, so the allocated memory + # region won't grow larger than necessary. This assumption allows the arena to + # skip maintaining a list of free indexes. + # + # Some systems may deviate from the POSIX default, but all systems seem to + # follow it, as it allows optimizations to the OS (it can reuse already + # allocated resources), and either the man page explicitly says so (Linux), or + # they don't (BSD) and they must follow the POSIX definition. protected class_getter arena = Arena(PollDescriptor).new(max_fds) private def self.max_fds : Int32 @@ -43,7 +49,8 @@ end # polling based UNIX targets, such as epoll (linux), kqueue (bsd), or poll # (posix) syscalls. This class only implements the generic parts for the # external world to interact with the loop. A specific implementation is -# required to handle the actual syscalls. +# required to handle the actual syscalls. See `Crystal::Epoll::EventLoop` and +# `Crystal::Kqueue::EventLoop`. # # The event loop registers the fd into the kernel data structures when an IO # operation would block, then keeps it there until the fd is closed. @@ -107,7 +114,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end {% end %} - # thread unsafe: must hold `@run_mutex` before calling! + # thread unsafe: must hold `@run_lock` before calling! def run(blocking : Bool) : Bool system_run(blocking) true @@ -431,7 +438,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop # fiber could be resumed twice! # # OPTIMIZE: collect events with the lock then process them after releasing the - # lock, which should be thread-safe as long as @run_lock is locked. + # lock, which should be thread-safe as long as `@run_lock` is locked. private def process_timers(timer_triggered : Bool) : Nil # events = PointerLinkedList(Event).new size = 0 From 6c45d091134fe1a871506cf700c2e8212842ae47 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 14 Sep 2024 12:11:30 +0200 Subject: [PATCH 26/56] Fix: arena doesn't free the object when the block raises --- src/crystal/system/unix/evented/arena.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/system/unix/evented/arena.cr index f467b206c90d..1b4726e733f3 100644 --- a/src/crystal/system/unix/evented/arena.cr +++ b/src/crystal/system/unix/evented/arena.cr @@ -142,6 +142,7 @@ class Crystal::Evented::Arena(T) return unless entry.value.allocated? yield entry.value.pointer + ensure entry.value.free end end From a58e8c15a2d32c3abc073199d150553d5df64c13 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 17 Sep 2024 12:15:21 +0200 Subject: [PATCH 27/56] Add Arena::Index to abstract the generation index Arena::Index now abstracts the underlying i64 representation of the generation index and allows to access the actual index and generation number independently. Arena::Index is now the sole acceptable index after the object is allocated (be it get or free). Also fixes an issue in kqueue/evloop for 32-bit systems, where udata can only be a 32 bit number (oops). Since we already have the fd/index as the kevent ident, we deconstruct the index to only pass the genration number, then reconstruct the index from the ident+udata. --- src/crystal/system/unix/epoll/event_loop.cr | 17 +- src/crystal/system/unix/evented/arena.cr | 145 ++++++++++++------ src/crystal/system/unix/evented/event.cr | 4 +- src/crystal/system/unix/evented/event_loop.cr | 51 +++--- .../system/unix/evented/poll_descriptor.cr | 4 +- src/crystal/system/unix/kqueue/event_loop.cr | 33 ++-- 6 files changed, 157 insertions(+), 97 deletions(-) diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr index e22e4db855fd..8037b9a22aba 100644 --- a/src/crystal/system/unix/epoll/event_loop.cr +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -52,7 +52,7 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop system_set_timer(@timers.next_ready?) # re-add all registered fds - Evented.arena.each { |fd, gen_index| system_add(fd, gen_index) } + Evented.arena.each { |fd, index| system_add(fd, index) } end {% end %} @@ -89,15 +89,12 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop end private def process(epoll_event : LibC::EpollEvent*) : Nil - gen_index = epoll_event.value.data.u64.unsafe_as(Int64) + index = Evented::Arena::Index.new(epoll_event.value.data.u64) events = epoll_event.value.events - {% if flag?(:tracing) %} - fd = (gen_index >> 32).to_i32! - Crystal.trace :evloop, "event", fd: fd, gen_index: gen_index, events: events - {% end %} + Crystal.trace :evloop, "event", fd: index.to_i, index: index.to_i64, events: events - pd = Evented.arena.get(gen_index) + pd = Evented.arena.get(index) if (events & (LibC::EPOLLERR | LibC::EPOLLHUP)) != 0 pd.value.@readers.consume_each { |event| resume_io(event) } @@ -125,10 +122,10 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop @eventfd.write(1) if @interrupted.test_and_set end - protected def system_add(fd : Int32, gen_index : Int64) : Nil - Crystal.trace :evloop, "epoll_ctl", op: "add", fd: fd, gen_index: gen_index + protected def system_add(fd : Int32, index : Evented::Arena::Index) : Nil + Crystal.trace :evloop, "epoll_ctl", op: "add", fd: fd, index: index.to_i64 events = LibC::EPOLLIN | LibC::EPOLLOUT | LibC::EPOLLRDHUP | LibC::EPOLLET - @epoll.add(fd, events, u64: gen_index.unsafe_as(UInt64)) + @epoll.add(fd, events, u64: index.to_u64) end protected def system_del(fd : Int32, closing = true) : Nil diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/system/unix/evented/arena.cr index 1b4726e733f3..54749bcf696a 100644 --- a/src/crystal/system/unix/evented/arena.cr +++ b/src/crystal/system/unix/evented/arena.cr @@ -5,12 +5,12 @@ # # The arena allocates objects `T` at a predefined index. The object iself is # uninitialized (outside of having its memory initialized to zero). The object -# can be allocated and later retrieved using the generation index (Int64) that -# contains both the actual index (Int32) and the generation number (UInt32). -# Deallocating the object increases the generation number, which allows the -# object to be reallocated later on. Trying to retrieve the allocation using the -# generation index will fail if the generation number changed (it's a new -# allocation). +# can be allocated and later retrieved using the generation index +# (Arena::Index) that contains both the actual index (Int32) and the generation +# number (UInt32). Deallocating the object increases the generation number, +# which allows the object to be reallocated later on. Trying to retrieve the +# allocation using the generation index will fail if the generation number +# changed (it's a new allocation). # # This arena isn't generic as it won't keep a list of free indexes. It assumes # that something else will maintain the uniqueness of indexes and reuse indexes @@ -29,13 +29,50 @@ # initialized to zero by default, then `#free` will also clear the memory, so # the next allocation shall be initialized to zero, too. # -# TODO: we could use a growing/shrinking buffer (realloc) though it would -# require a rwlock to borrow accesses during which we can mutate the pointed -# memory, but growing/shrinking would need exclusive write access (it -# reallocates, hence invalidates all pointers); resizing could be delayed and -# thus shouldn't happen often + borrowing accesses should be as quick/small as -# possible. +# TODO: instead of the mmap that must preallocate a fixed chunk of virtual +# memory, we could allocate individual blocks of memory, then access the actual +# block at `index % size`. Pointers would still be valid (as long as the block +# isn't collected). We wouldn't have to worry about maximum capacity, we could +# still allocate blocks discontinuously & collect unused blocks during GC +# collections. class Crystal::Evented::Arena(T) + INVALID_INDEX = Index.new(-1, 0) + + struct Index + def initialize(index : Int32, generation : UInt32) + @data = (index.to_i64! << 32) | generation.to_u64! + end + + def initialize(@data : Int64) + end + + def initialize(data : UInt64) + @data = data.unsafe_as(Int64) + end + + # Returns the generation number. + def generation : UInt32 + @data.to_u32! + end + + # Returns the actual index. + def to_i : Int32 + (@data >> 32).to_i32! + end + + def to_i64 : Int64 + @data + end + + def to_u64 : UInt64 + @data.unsafe_as(UInt64) + end + + def valid? : Bool + @data >= 0 + end + end + struct Entry(T) @lock = SpinLock.new # protects parallel allocate/free calls property? allocated = false @@ -82,26 +119,6 @@ class Crystal::Evented::Arena(T) LibC.munmap(@buffer.to_unsafe, @buffer.bytesize) end - # Returns a pointer to the object allocated at *gen_idx* (generation index). - # - # Raises if the object isn't allocated. - # Raises if the generation has changed (i.e. the object has been freed then reallocated). - # Raises if *index* is negative. - def get(gen_idx : Int64) : Pointer(T) - index, generation = from_gen_index(gen_idx) - entry = at(index) - - unless entry.value.allocated? - raise RuntimeError.new("#{self.class.name}: object not allocated at index #{index}") - end - - unless (actual = entry.value.generation) == generation - raise RuntimeError.new("#{self.class.name}: object generation changed at index #{index} (#{generation} => #{actual})") - end - - entry.value.pointer - end - # Yields and allocates the object at *index* unless already allocated. # Returns a pointer to the object at *index* and the generation index. # @@ -110,13 +127,13 @@ class Crystal::Evented::Arena(T) # the generation index. # # There are no generational checks. - # Raises if *index* is negative. - def lazy_allocate(index : Int32, &) : {Pointer(T), Int64} + # Raises if *index* is out of bounds. + def lazy_allocate(index : Int32, &) : {Pointer(T), Index} entry = at(index) entry.value.@lock.sync do pointer = entry.value.pointer - gen_index = to_gen_index(index, entry) + gen_index = Index.new(index, entry.value.generation) unless entry.value.allocated? {% unless flag?(:preview_mt) %} @@ -131,22 +148,60 @@ class Crystal::Evented::Arena(T) end end - # Yields the object allocated at *index* then releases it. - # Does nothing if the object wasn't allocated. + # Returns a pointer to the object previously allocated at *index*. + # + # Raises if the object isn't allocated. + # Raises if the generation has changed (i.e. the object has been freed then reallocated). + # Raises if *index* is negative. + def get(index : Index) : Pointer(T) + entry = at(index.to_i) + entry.value.pointer + end + + # Returns a pointer to the object previously allocated at *index*. + # Returns `nil` if the object isn't allocated or the generation has changed. + # + # Raises if *index* is negative. + def get?(index : Index) : Pointer(T)? + if entry = at?(index.to_i) + entry.value.pointer + end + end + + # Yields the object previously allocated at *index* then releases it. Does + # nothing if the object isn't allocated or the generation has changed. # # Raises if *index* is negative. - def free(index : Int32, &) : Nil - return unless entry = at?(index) + def free(index : Index, &) : Nil + return unless entry = at?(index.to_i) entry.value.@lock.sync do return unless entry.value.allocated? - + return unless entry.value.generation == index.generation yield entry.value.pointer ensure entry.value.free end end + private def at(index : Index) : Pointer(Entry(T)) + entry = at(index.to_i) + unless entry.value.allocated? + raise RuntimeError.new("#{self.class.name}: object not allocated at index #{index.to_i}") + end + unless entry.value.generation == generation + raise RuntimeError.new("#{self.class.name}: object generation changed at index #{index.to_i} (#{index.generation} => #{entry.value.generation})") + end + entry + end + + private def at?(index : Index) : Pointer(Entry(T)) + return unless entry = at?(index.to_i) + return unless entry.value.allocated? + return unless entry.value.generation == generation + entry + end + private def at(index : Int32) : Pointer(Entry(T)) if index.negative? raise ArgumentError.new("#{self.class.name}: negative index #{index}") @@ -176,17 +231,9 @@ class Crystal::Evented::Arena(T) entry = pointer + index if entry.value.allocated? - yield index, to_gen_index(index, entry) + yield index, Index.new(index, entry.value.generation) end end end {% end %} - - private def to_gen_index(index : Int32, entry : Pointer(Entry(T))) : Int64 - (index.to_i64! << 32) | entry.value.generation.to_u64! - end - - private def from_gen_index(gen_index : Int64) : {Int32, UInt32} - {(gen_index >> 32).to_i32!, gen_index.to_u32!} - end end diff --git a/src/crystal/system/unix/evented/event.cr b/src/crystal/system/unix/evented/event.cr index 00b9ff403f0b..1cfc4a58f04c 100644 --- a/src/crystal/system/unix/evented/event.cr +++ b/src/crystal/system/unix/evented/event.cr @@ -21,7 +21,7 @@ struct Crystal::Evented::Event # Arena index to access the associated `PollDescriptor` when processing an IO # event. Nil for timed events (sleep, select timeout). - getter! gen_index : Int64? + getter! index : Arena::Index? # The absolute time, against the monotonic clock, at which a timed event shall # trigger. Nil for IO events without a timeout. @@ -33,7 +33,7 @@ struct Crystal::Evented::Event # The event can be added into different lists. See `Waiters` and `Timers`. include PointerLinkedList::Node - def initialize(@type : Type, @fiber, @gen_index = nil, timeout : Time::Span? = nil) + def initialize(@type : Type, @fiber, @index = nil, timeout : Time::Span? = nil) @wake_at = Time.monotonic + timeout if timeout end diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 9f085366271c..541353ba2138 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -3,12 +3,12 @@ require "./arena" module Crystal::System::FileDescriptor # user data (generation index for the arena) - property __evloop_data : Int64 = -1_i64 + property __evloop_data : Evented::Arena::Index = Evented::Arena::INVALID_INDEX end module Crystal::System::Socket # user data (generation index for the arena) - property __evloop_data : Int64 = -1_i64 + property __evloop_data : Evented::Arena::Index = Evented::Arena::INVALID_INDEX end module Crystal::Evented @@ -187,10 +187,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end def remove(file_descriptor : System::FileDescriptor) : Nil - Evented.arena.free(file_descriptor.fd) do |pd| - pd.value.remove(file_descriptor.fd) { } # ignore system error - file_descriptor.__evloop_data = -1_i64 - end + internal_remove(file_descriptor) end # socket interface, see Crystal::EventLoop::Socket @@ -293,10 +290,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end def remove(socket : ::Socket) : Nil - Evented.arena.free(socket.fd) do |pd| - pd.value.remove(socket.fd) { } # ignore system error - socket.__evloop_data = -1_i64 - end + internal_remove(socket) end # internals: IO @@ -326,7 +320,10 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end protected def evented_close(io) - Evented.arena.free(io.fd) do |pd| + return unless (index = io.__evloop_data).valid? + io.__evloop_data = Arena::INVALID_INDEX + + Evented.arena.free(index) do |pd| pd.value.@readers.consume_each do |event| pd.value.@event_loop.try(&.resume_io(event)) end @@ -336,7 +333,15 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end pd.value.remove(io.fd) - io.__evloop_data = -1_i64 + end + end + + private def internal_remove(io) + return unless (index = io.__evloop_data).valid? + io.__evloop_data = Arena::INVALID_INDEX + + Evented.arena.free(index) do |pd| + pd.value.remove(io.fd) { } # ignore system error end end @@ -358,20 +363,20 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop private macro wait(type, io, waiters, timeout, &) # get or allocate the poll descriptor - if (%gen_index = {{io}}.__evloop_data) >= 0 - %pd = Evented.arena.get(%gen_index) + if (%index = {{io}}.__evloop_data).valid? + %pd = Evented.arena.get(%index) else - %pd, %gen_index = Evented.arena.lazy_allocate({{io}}.fd) do |pd, gen_index| + %pd, %index = Evented.arena.lazy_allocate({{io}}.fd) do |pd, index| # register the fd with the event loop (once), it should usually merely add # the fd to the current evloop but may "transfer" the ownership from # another event loop: - {{io}}.__evloop_data = gen_index - pd.value.take_ownership(self, {{io}}.fd, gen_index) + {{io}}.__evloop_data = index + pd.value.take_ownership(self, {{io}}.fd, index) end end # create an event (on the stack) - %event = Evented::Event.new({{type}}, Fiber.current, %gen_index, {{timeout}}) + %event = Evented::Event.new({{type}}, Fiber.current, %index, {{timeout}}) # try to add the event to the waiting list # don't wait if the waiter has already been marked ready (see Waiters#add) @@ -465,12 +470,12 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop when .io_read? # reached read timeout: cancel io event event.value.timed_out! - pd = Evented.arena.get(event.value.gen_index) + pd = Evented.arena.get(event.value.index) pd.value.@readers.delete(event) when .io_write? # reached write timeout: cancel io event event.value.timed_out! - pd = Evented.arena.get(event.value.gen_index) + pd = Evented.arena.get(event.value.index) pd.value.@writers.delete(event) when .select_timeout? # always dequeue the event but only enqueue the fiber if we win the @@ -497,12 +502,12 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop # *blocking* is `true` the loop must wait for events to become ready (possibly # indefinitely); when `false` the loop shall return immediately. # - # The `PollDescriptor` of IO events can be retrieved using the *gen_index* + # The `PollDescriptor` of IO events can be retrieved using the *index* # from the system event's user data. private abstract def system_run(blocking : Bool) : Nil - # Add *fd* to the polling system, setting *gen_index* as user data. - protected abstract def system_add(fd : Int32, gen_index : Int64) : Nil + # Add *fd* to the polling system, setting *index* as user data. + protected abstract def system_add(fd : Int32, index : Index) : Nil # Remove *fd* from the polling system. Must raise a `RuntimeError` on error. # diff --git a/src/crystal/system/unix/evented/poll_descriptor.cr b/src/crystal/system/unix/evented/poll_descriptor.cr index 97a60190e54e..53ab5313e313 100644 --- a/src/crystal/system/unix/evented/poll_descriptor.cr +++ b/src/crystal/system/unix/evented/poll_descriptor.cr @@ -12,7 +12,7 @@ struct Crystal::Evented::PollDescriptor # Makes *event_loop* the new owner of *fd*. # Removes *fd* from the current event loop (if any). - def take_ownership(event_loop : EventLoop, fd : Int32, gen_index : Int64) : Nil + def take_ownership(event_loop : EventLoop, fd : Int32, index : Arena::Index) : Nil @lock.sync do current = @event_loop @@ -28,7 +28,7 @@ struct Crystal::Evented::PollDescriptor end @event_loop = event_loop - event_loop.system_add(fd, gen_index) + event_loop.system_add(fd, index) current.try(&.system_del(fd, closing: false)) end end diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index 9e5c137c358f..7098d7ef16f3 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -58,7 +58,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop system_set_timer(@timers.next_ready?) # re-add all registered fds - Evented.arena.each { |fd, gen_index| system_add(fd, gen_index) } + Evented.arena.each { |fd, index| system_add(fd, index) } end {% end %} @@ -107,15 +107,18 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop end private def process(kevent : LibC::Kevent*) : Nil - gen_index = kevent.value.udata.address.to_i64! + index = + {% if flag?(:bits64) %} + Evented::Arena::Index.new(kevent.value.udata.address) + {% else %} + # assuming 32-bit target: rebuild the arena index + Evented::Arena::Index.new(kevent.value.ident.to_i32!, kevent.value.udata.address.to_u32!) + {% end %} - {% if flag?(:tracing) %} - fd = kevent.value.ident - Crystal.trace :evloop, "event", fd: fd, gen_index: gen_index, - filter: kevent.value.filter, flags: kevent.value.flags, fflags: kevent.value.fflags - {% end %} + Crystal.trace :evloop, "event", fd: kevent.value.ident, index: index.to_i64, + filter: kevent.value.filter, flags: kevent.value.flags, fflags: kevent.value.fflags - pd = Evented.arena.get(gen_index) + pd = Evented.arena.get(index) if (kevent.value.fflags & LibC::EV_EOF) == LibC::EV_EOF # apparently some systems may report EOF on write with EVFILT_READ instead @@ -159,15 +162,23 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop {% end %} end - protected def system_add(fd : Int32, gen_index : Int64) : Nil - Crystal.trace :evloop, "kevent", op: "add", fd: fd, gen_index: gen_index + protected def system_add(fd : Int32, index : Evented::Arena::Index) : Nil + Crystal.trace :evloop, "kevent", op: "add", fd: fd, index: index.to_i64 # register both read and write events kevents = uninitialized LibC::Kevent[2] 2.times do |i| kevent = kevents.to_unsafe + i filter = i == 0 ? LibC::EVFILT_READ : LibC::EVFILT_WRITE - System::Kqueue.set(kevent, fd, filter, LibC::EV_ADD | LibC::EV_CLEAR, udata: Pointer(Void).new(gen_index.to_u64!)) + + udata = + {% if flag?(:bits64) %} + Pointer(Void).new(index.to_u64) + {% else %} + # assuming 32-bit target: pass the generation as udata (ident is the fd/index) + Pointer(Void).new(index.generation) + {% end %} + System::Kqueue.set(kevent, fd, filter, LibC::EV_ADD | LibC::EV_CLEAR, udata: udata) end @kqueue.kevent(kevents.to_slice) do From 0393b01ecec8594890171e7e4cabf4d034d2ff5a Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 17 Sep 2024 13:14:57 +0200 Subject: [PATCH 28/56] Fix: invalid check for waiters in PollDescriptor#take_ownership --- src/crystal/system/unix/evented/poll_descriptor.cr | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/crystal/system/unix/evented/poll_descriptor.cr b/src/crystal/system/unix/evented/poll_descriptor.cr index 53ab5313e313..fdfb7a556bbe 100644 --- a/src/crystal/system/unix/evented/poll_descriptor.cr +++ b/src/crystal/system/unix/evented/poll_descriptor.cr @@ -23,7 +23,7 @@ struct Crystal::Evented::PollDescriptor # ensure we can't have cross enqueues after we transfer the fd, so we # can optimize (all enqueues are local) and we don't end up with a timer # from evloop A to cancel an event from evloop B (currently unsafe) - if current && @readers.@list.empty? && @writers.@list.empty? + if current && !empty? raise RuntimeError.new("BUG: transfering fd=#{fd} to another evloop with pending reader/writer fibers") end @@ -48,4 +48,10 @@ struct Crystal::Evented::PollDescriptor current.try(&.system_del(fd) { yield }) end end + + # Returns true when there is at least one reader or writer. Returns false + # otherwise. + def empty? : Bool + @readers.@list.empty? && @writers.@list.empty? + end end From cc6c33a7efa6245a3be679aeea4d14ae2479acf0 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 19 Sep 2024 15:01:54 +0200 Subject: [PATCH 29/56] Fix: race condition with #evented_close in parallel to #run Now either has to dequeue from both queues (waiters and timers) to be able to resume the fiber, with the rule that the timer always wins when run dequeued a timer and evented_close dequeue the IO event in parallel. --- src/crystal/system/unix/epoll/event_loop.cr | 18 ++--- src/crystal/system/unix/evented/event_loop.cr | 81 +++++++++++-------- src/crystal/system/unix/evented/timers.cr | 23 +++--- src/crystal/system/unix/evented/waiters.cr | 28 ++++--- src/crystal/system/unix/kqueue/event_loop.cr | 20 ++--- 5 files changed, 97 insertions(+), 73 deletions(-) diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr index 8037b9a22aba..93ff09a8d7b2 100644 --- a/src/crystal/system/unix/epoll/event_loop.cr +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -81,14 +81,14 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop Crystal.trace :evloop, "timer" timer_triggered = true else - process(epoll_event) + process_io(epoll_event) end end process_timers(timer_triggered) end - private def process(epoll_event : LibC::EpollEvent*) : Nil + private def process_io(epoll_event : LibC::EpollEvent*) : Nil index = Evented::Arena::Index.new(epoll_event.value.data.u64) events = epoll_event.value.events @@ -97,23 +97,19 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop pd = Evented.arena.get(index) if (events & (LibC::EPOLLERR | LibC::EPOLLHUP)) != 0 - pd.value.@readers.consume_each { |event| resume_io(event) } - pd.value.@writers.consume_each { |event| resume_io(event) } + pd.value.@readers.consume_each { |event| unsafe_resume_io(event) } + pd.value.@writers.consume_each { |event| unsafe_resume_io(event) } return end if (events & LibC::EPOLLRDHUP) == LibC::EPOLLRDHUP - pd.value.@readers.consume_each { |event| resume_io(event) } + pd.value.@readers.consume_each { |event| unsafe_resume_io(event) } elsif (events & LibC::EPOLLIN) == LibC::EPOLLIN - if event = pd.value.@readers.ready! - resume_io(event) - end + pd.value.@readers.ready { |event| unsafe_resume_io(event) } end if (events & LibC::EPOLLOUT) == LibC::EPOLLOUT - if event = pd.value.@writers.ready! - resume_io(event) - end + pd.value.@writers.ready { |event| unsafe_resume_io(event) } end end diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 541353ba2138..2fe42aee44bd 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -84,11 +84,6 @@ end # If the IO operation has a timeout, the event is also registered into `@timers` # before suspending the fiber, then after resume it will raise # `IO::TimeoutError` if the event timed out, and continue otherwise. -# -# OPTIMIZE: collect fibers & canceled timers, delete canceled timers when -# processing timers, and eventually enqueue all fibers; it would avoid repeated -# lock/unlock timers on each #resume_io and allow to replace individual fiber -# enqueues with a single batch enqueue (simpler). abstract class Crystal::Evented::EventLoop < Crystal::EventLoop {% if flag?(:preview_mt) %} @run_lock = Atomic::Flag.new # protects parallel runs @@ -325,11 +320,11 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop Evented.arena.free(index) do |pd| pd.value.@readers.consume_each do |event| - pd.value.@event_loop.try(&.resume_io(event)) + pd.value.@event_loop.try(&.unsafe_resume_io(event)) end pd.value.@writers.consume_each do |event| - pd.value.@event_loop.try(&.resume_io(event)) + pd.value.@event_loop.try(&.unsafe_resume_io(event)) end pd.value.remove(io.fd) @@ -391,12 +386,8 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop return {{yield}} else # nothing to do: either the timer triggered which means it was dequeued, - # or `#resume_io` was called to resume the IO and the timer got deleted - # from the timers before the fiber got reenqueued. - # - # TODO: consider a quick check to verify whether the event is still - # queued and panic when it happens: the event is put on the stack and we - # can't access it after this method returns! + # or `#unsafe_resume_io` was called to resume the IO and the timer got + # deleted from the timers before the fiber got reenqueued. end else Fiber.suspend @@ -422,18 +413,37 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end end - protected def delete_timer(event : Evented::Event*) + protected def delete_timer(event : Evented::Event*) : Bool @lock.sync do - was_next_ready = @timers.delete(event) - system_set_timer(@timers.next_ready?) if was_next_ready + if index = @timers.delete(event) + # update system timer if we deleted the next timer + system_set_timer(@timers.next_ready?) if index.zero? + return true + end end + false end # Helper to resume the fiber associated to an IO event and remove the event - # from timers if applicable. - protected def resume_io(event : Evented::Event*) : Nil - delete_timer(event) if event.value.wake_at? - Crystal::Scheduler.enqueue(event.value.fiber) + # from timers if applicable. Returns true if the fiber has been enqueued. + # + # Thread unsafe: we must hold the poll descriptor waiter lock for the whole + # duration of the dequeue/resume_io otherwise we might conflict with timers + # trying to cancel an IO event. + protected def unsafe_resume_io(event : Evented::Event*) : Bool + # we only partially own the poll descriptor; thanks to the lock we know that + # another thread won't dequeue it, yet it may still be in the timers queue, + # which at worst may be waiting on the lock to be released, so event* can be + # dereferenced safely. + + if !event.value.wake_at? || delete_timer(event) + # no timeout or we canceled it: we fully own the event + Crystal::Scheduler.enqueue(event.value.fiber) + true + else + # failed to cancel the timeout so the timer owns the event (by rule) + false + end end # Process ready timers. @@ -441,42 +451,47 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop # Shall be called after processing IO events. IO events with a timeout that # have succeeded shall already have been removed from `@timers` otherwise the # fiber could be resumed twice! - # - # OPTIMIZE: collect events with the lock then process them after releasing the - # lock, which should be thread-safe as long as `@run_lock` is locked. private def process_timers(timer_triggered : Bool) : Nil - # events = PointerLinkedList(Event).new + # collect ready timers before processing them —this is safe— to avoids a + # deadlock situation when another thread tries to process a ready IO event + # (in poll descriptor waiters) with a timeout (same event* in timers) + buffer = uninitialized StaticArray(Pointer(Evented::Event), 128) size = 0 @lock.sync do @timers.dequeue_ready do |event| - # events << event - process_timer(event) - size += 1 + buffer.to_unsafe[size] = event + break if (size &+= 1) == buffer.size end - unless size == 0 && timer_triggered + if size > 0 || timer_triggered system_set_timer(@timers.next_ready?) end end - # events.each { |event| process_timer(event) } + buffer.to_slice[0, size].each do |event| + process_timer(event) + end end private def process_timer(event : Evented::Event*) + # we dequeued the event from timers, and by rule we own it, so event* can + # safely be dereferenced: fiber = event.value.fiber case event.value.type when .io_read? - # reached read timeout: cancel io event - event.value.timed_out! + # reached read timeout: cancel io event; by rule the timer always wins, + # even in case of conflict with #unsafe_resume_io we must resume the fiber pd = Evented.arena.get(event.value.index) pd.value.@readers.delete(event) - when .io_write? - # reached write timeout: cancel io event event.value.timed_out! + when .io_write? + # reached write timeout: cancel io event; by rule the timer always wins, + # even in case of conflict with #unsafe_resume_io we must resume the fiber pd = Evented.arena.get(event.value.index) pd.value.@writers.delete(event) + event.value.timed_out! when .select_timeout? # always dequeue the event but only enqueue the fiber if we win the # atomic CAS diff --git a/src/crystal/system/unix/evented/timers.cr b/src/crystal/system/unix/evented/timers.cr index 0b70e7958325..098b11d522b5 100644 --- a/src/crystal/system/unix/evented/timers.cr +++ b/src/crystal/system/unix/evented/timers.cr @@ -4,10 +4,16 @@ # Thread unsafe: parallel accesses much be protected. # # NOTE: this is a struct because it only wraps a const pointer to a deque -# allocated in the heap +# allocated in the heap. # -# OPTIMIZE: consider a skiplist for quicker lookups + avoid memmove on `#add` -# and `#delete`. +# OPTIMIZE: consider a skiplist for faster lookups (add/delete). +# +# OPTIMIZE: we could avoid memmove on add/delete by allocating a buffer, putting +# entries at whatever available index in the buffer, and linking entries in +# order (using indices so we can realloc the buffer); we'd have to keep a list +# of free indexes, too. It could be a good combo of unbounded linked list while +# retaining some memory locality. It should even be compatible with a skiplist +# (e.g. make entries a fixed height tower instead of prev/next node). struct Crystal::Evented::Timers def initialize @list = Deque(Evented::Event*).new @@ -24,7 +30,7 @@ struct Crystal::Evented::Timers # Dequeues and yields each ready timer (their `#wake_at` is lower than # `Time.monotonic`) from the oldest to the most recent (i.e. time ascending). - def dequeue_ready(&) : Nil + def dequeue_ready(& : Evented::Event* -> Nil) : Nil return if @list.empty? now = Time.monotonic @@ -60,13 +66,12 @@ struct Crystal::Evented::Timers end end - # Removes a timer from the list. Returns true if it was the next ready timer. - def delete(event : Evented::Event*) : Bool + # Remove a timer from the list. Returns the index at which the event was, and + # `nil` otherwise. + def delete(event : Evented::Event*) : Int32? if index = @list.index(event) @list.delete_at(index) - index == 0 - else - false + index end end end diff --git a/src/crystal/system/unix/evented/waiters.cr b/src/crystal/system/unix/evented/waiters.cr index 0ba2e9e15aab..c98bdcb12ee0 100644 --- a/src/crystal/system/unix/evented/waiters.cr +++ b/src/crystal/system/unix/evented/waiters.cr @@ -10,9 +10,9 @@ struct Crystal::Evented::Waiters def add(event : Pointer(Event)) : Bool {% if flag?(:preview_mt) %} - # we check for readyness to avoid a race condition with another thread - # running the evloop and trying to wakeup a waiting fiber while we try to - # add a waiting fiber + # check for readiness since another thread running the evloop might be + # trying to dequeue an event while we're waiting on the lock (failure to + # notice notice the IO is ready) return false if ready? @lock.sync do @@ -40,17 +40,25 @@ struct Crystal::Evented::Waiters @ready.swap(false, :relaxed) end - def ready! : Pointer(Event)? + def ready(& : Pointer(Event) -> Bool) : Nil @lock.sync do {% if flag?(:preview_mt) %} - if event = @list.shift? - event - else - @ready.set(true, :relaxed) - nil + # loop until the block succesfully processes an event (it may have to + # dequeue the timeout from timers) + loop do + if event = @list.shift? + break if yield event + else + # no event queued but another thread may be waiting for the lock to + # add an event: set as ready to resolve the race condition + @ready.set(true, :relaxed) + return + end end {% else %} - @list.shift? + if event = @list.shift? + yield event + end {% end %} end end diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index 7098d7ef16f3..95702d56e293 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -81,7 +81,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop # nothing special timer_triggered = true else - process(kevent) + process_io(kevent) end end @@ -106,7 +106,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop false end - private def process(kevent : LibC::Kevent*) : Nil + private def process_io(kevent : LibC::Kevent*) : Nil index = {% if flag?(:bits64) %} Evented::Arena::Index.new(kevent.value.udata.address) @@ -123,8 +123,8 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop if (kevent.value.fflags & LibC::EV_EOF) == LibC::EV_EOF # apparently some systems may report EOF on write with EVFILT_READ instead # of EVFILT_WRITE, so let's wake all waiters: - pd.value.@readers.consume_each { |event| resume_io(event) } - pd.value.@writers.consume_each { |event| resume_io(event) } + pd.value.@readers.consume_each { |event| unsafe_resume_io(event) } + pd.value.@writers.consume_each { |event| unsafe_resume_io(event) } return end @@ -132,16 +132,16 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop when LibC::EVFILT_READ if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR # OPTIMIZE: pass errno (kevent.data) through PollDescriptor - pd.value.@readers.consume_each { |event| resume_io(event) } - elsif event = pd.value.@readers.ready! - resume_io(event) + pd.value.@readers.consume_each { |event| unsafe_resume_io(event) } + else + pd.value.@readers.ready { |event| unsafe_resume_io(event) } end when LibC::EVFILT_WRITE if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR # OPTIMIZE: pass errno (kevent.data) through PollDescriptor - pd.value.@writers.consume_each { |event| resume_io(event) } - elsif event = pd.value.@writers.ready! - resume_io(event) + pd.value.@writers.consume_each { |event| unsafe_resume_io(event) } + else + pd.value.@writers.ready { |event| unsafe_resume_io(event) } end end end From fa6a93539d654345dbd49e4f754ef2788a3adec7 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 24 Sep 2024 11:03:03 +0200 Subject: [PATCH 30/56] Retry epoll_wait/kevent syscall on EINTR with infinite timeout --- src/crystal/system/unix/epoll.cr | 16 +++++++++-- src/crystal/system/unix/kqueue.cr | 45 +++++++++++++++++-------------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/crystal/system/unix/epoll.cr b/src/crystal/system/unix/epoll.cr index 8827c40c9934..28a157ae3360 100644 --- a/src/crystal/system/unix/epoll.cr +++ b/src/crystal/system/unix/epoll.cr @@ -43,8 +43,20 @@ struct Crystal::System::Epoll # `timeout` is in milliseconds; -1 will wait indefinitely; 0 will never wait. def wait(events : Slice(LibC::EpollEvent), timeout : Int32) : Slice(LibC::EpollEvent) - count = LibC.epoll_wait(@epfd, events.to_unsafe, events.size, timeout) - raise RuntimeError.from_errno("epoll_wait") if count == -1 && Errno.value != Errno::EINTR + count = 0 + + loop do + count = LibC.epoll_wait(@epfd, events.to_unsafe, events.size, timeout) + break unless count == -1 + + if Errno.value == Errno::EINTR + # retry when waiting indefinitely, return otherwise + break unless timeout == -1 + else + raise RuntimeError.from_errno("epoll_wait") + end + end + events[0, count.clamp(0..)] end diff --git a/src/crystal/system/unix/kqueue.cr b/src/crystal/system/unix/kqueue.cr index 28145bf1ddb0..ff36ca734487 100644 --- a/src/crystal/system/unix/kqueue.cr +++ b/src/crystal/system/unix/kqueue.cr @@ -13,24 +13,6 @@ struct Crystal::System::Kqueue raise RuntimeError.from_errno("kqueue1") if @kq == -1 end - # Registers *changes* and returns a subslice to *events*. - # - # Timeout is relative to now; blocks indefinitely if `nil`; returns - # immediately if zero. - def kevent(changes : Slice(LibC::Kevent), events : Slice(LibC::Kevent), timeout : ::Time::Span? = nil) : Slice(LibC::Kevent) - if timeout - ts = uninitialized LibC::Timespec - ts.tv_sec = typeof(ts.tv_sec).new!(timeout.@seconds) - ts.tv_nsec = typeof(ts.tv_nsec).new!(timeout.@nanoseconds) - tsp = pointerof(ts) - else - tsp = Pointer(LibC::Timespec).null - end - count = LibC.kevent(@kq, changes.to_unsafe, changes.size, events.to_unsafe, events.size, tsp) - raise RuntimeError.from_errno("kevent") if count == -1 && !Errno.value.in?(Errno::EINTR, Errno::ENOENT) - events[0, count.clamp(0..)] - end - # Helper to register a single event. Returns immediately. def kevent(ident, filter, flags, fflags = 0, data = 0, udata = nil, &) : Nil kevent = uninitialized LibC::Kevent @@ -45,14 +27,37 @@ struct Crystal::System::Kqueue yield if ret == -1 end - # Helper to wait for registered events to become active. Returns a subslice to + # Waits for registered events to become active. Returns a subslice to # *events*. # # Timeout is relative to now; blocks indefinitely if `nil`; returns # immediately if zero. def wait(events : Slice(LibC::Kevent), timeout : ::Time::Span? = nil) : Slice(LibC::Kevent) + if timeout + ts = uninitialized LibC::Timespec + ts.tv_sec = typeof(ts.tv_sec).new!(timeout.@seconds) + ts.tv_nsec = typeof(ts.tv_nsec).new!(timeout.@nanoseconds) + tsp = pointerof(ts) + else + tsp = Pointer(LibC::Timespec).null + end + changes = uninitialized LibC::Kevent[0] - kevent(changes.to_slice, events, timeout) + count = 0 + + loop do + count = LibC.kevent(@kq, changes.to_unsafe, changes.size, events.to_unsafe, events.size, tsp) + break unless count == -1 + + if Errno.value == Errno::EINTR + # retry when waiting indefinitely, return otherwise + break if timeout + else + raise RuntimeError.from_errno("kevent") + end + end + + events[0, count.clamp(0..)] end def close : Nil From f42ea86e4aa8320a9164fb5641cdef3de6c12153 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 24 Sep 2024 11:03:56 +0200 Subject: [PATCH 31/56] Remove #try_run? and #try_lock? --- src/crystal/system/unix/evented/event_loop.cr | 30 +------------------ 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 2fe42aee44bd..896a989ff03b 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -85,10 +85,6 @@ end # before suspending the fiber, then after resume it will raise # `IO::TimeoutError` if the event timed out, and continue otherwise. abstract class Crystal::Evented::EventLoop < Crystal::EventLoop - {% if flag?(:preview_mt) %} - @run_lock = Atomic::Flag.new # protects parallel runs - {% end %} - def initialize @lock = SpinLock.new # protects parallel accesses to @timers @timers = Timers.new @@ -97,46 +93,22 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop # reset the mutexes since another thread may have acquired the lock of one # event loop, which would prevent closing file descriptors for example. def after_fork_before_exec : Nil - {% if flag?(:preview_mt) %} @run_lock.clear {% end %} @lock = SpinLock.new end {% unless flag?(:preview_mt) %} # no parallelism issues, but let's clean-up anyway def after_fork : Nil - {% if flag?(:preview_mt) %} @run_lock.clear {% end %} @lock = SpinLock.new end {% end %} - # thread unsafe: must hold `@run_lock` before calling! + # NOTE: thread unsafe def run(blocking : Bool) : Bool system_run(blocking) true end - def try_lock?(&) : Bool - {% if flag?(:preview_mt) %} - if @run_lock.test_and_set - begin - yield - true - ensure - @run_lock.clear - end - else - false - end - {% else %} - yield - true - {% end %} - end - - def try_run?(blocking : Bool) : Bool - try_lock? { run(blocking) } - end - # fiber interface, see Crystal::EventLoop def create_resume_event(fiber : Fiber) : FiberEvent From 2a84f7326ae38d1ad41f7bea8cc5feb4fa378749 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 30 Sep 2024 16:18:06 +0200 Subject: [PATCH 32/56] Fix: use 'open files' soft limit to preallocate arena The hard limit can sometimes be be very high (e.g. 1 billion) and trying to preallocate that much virtual memory is failing. Let's use the soft limit for starters, then we can revisit the problem by allocating the arena into blocks instead (no upper limit at the expense of a bounds check). --- src/crystal/system/unix/evented/event_loop.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 896a989ff03b..1bb552de2972 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -39,7 +39,7 @@ module Crystal::Evented if LibC.getrlimit(LibC::RLIMIT_NOFILE, out rlimit) == -1 raise RuntimeError.from_errno("getrlimit(RLIMIT_NOFILE)") end - rlimit.rlim_max.clamp(..Int32::MAX).to_i32! + rlimit.rlim_cur.clamp(..Int32::MAX).to_i32! end end From 589993b8f762026a5c76ce26dd448ea3afe61bff Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 3 Oct 2024 15:54:08 +0200 Subject: [PATCH 33/56] Fix: race condition in Waiters#add vs Waiters#consume_each One thread may close a file descriptor or run the evloop and receive an error or hup event, which will wakeup all pending fibers. Further attempts should fail since the fd is closed, but another thread might be trying to add a waiter, which may go unnoticed by `#consume_each` that doesn't manipulate the `@ready` variable. This patch resolves this by distinguishing between ready (we want to wakeup one waiter) and closed (the fd will never go to block anymore). Also simplifies the implementation to rely on the lock, instead of the lock + atomics. We'll see later if the lock is an issue and how to optimize in that case. --- src/crystal/system/unix/epoll/event_loop.cr | 10 ++-- src/crystal/system/unix/evented/event.cr | 2 +- src/crystal/system/unix/evented/event_loop.cr | 14 ++--- src/crystal/system/unix/evented/waiters.cr | 56 ++++++++++++------- src/crystal/system/unix/kqueue/event_loop.cr | 12 ++-- 5 files changed, 52 insertions(+), 42 deletions(-) diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr index 93ff09a8d7b2..58e5ac880384 100644 --- a/src/crystal/system/unix/epoll/event_loop.cr +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -97,19 +97,19 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop pd = Evented.arena.get(index) if (events & (LibC::EPOLLERR | LibC::EPOLLHUP)) != 0 - pd.value.@readers.consume_each { |event| unsafe_resume_io(event) } - pd.value.@writers.consume_each { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } + pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } return end if (events & LibC::EPOLLRDHUP) == LibC::EPOLLRDHUP - pd.value.@readers.consume_each { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } elsif (events & LibC::EPOLLIN) == LibC::EPOLLIN - pd.value.@readers.ready { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_one { |event| unsafe_resume_io(event) } end if (events & LibC::EPOLLOUT) == LibC::EPOLLOUT - pd.value.@writers.ready { |event| unsafe_resume_io(event) } + pd.value.@writers.ready_one { |event| unsafe_resume_io(event) } end end diff --git a/src/crystal/system/unix/evented/event.cr b/src/crystal/system/unix/evented/event.cr index 1cfc4a58f04c..ad8fbd5a8276 100644 --- a/src/crystal/system/unix/evented/event.cr +++ b/src/crystal/system/unix/evented/event.cr @@ -30,7 +30,7 @@ struct Crystal::Evented::Event # True if an IO event has timed out (i.e. we're past `#wake_at`). getter? timed_out : Bool = false - # The event can be added into different lists. See `Waiters` and `Timers`. + # The event can be added to `Waiters` lists. include PointerLinkedList::Node def initialize(@type : Type, @fiber, @index = nil, timeout : Time::Span? = nil) diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 1bb552de2972..8a5594ff4af4 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -291,11 +291,11 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop io.__evloop_data = Arena::INVALID_INDEX Evented.arena.free(index) do |pd| - pd.value.@readers.consume_each do |event| + pd.value.@readers.ready_all do |event| pd.value.@event_loop.try(&.unsafe_resume_io(event)) end - pd.value.@writers.consume_each do |event| + pd.value.@writers.ready_all do |event| pd.value.@event_loop.try(&.unsafe_resume_io(event)) end @@ -313,7 +313,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end private def wait_readable(io, timeout = nil) : Nil - wait(:io_read, io, :readers, timeout) { raise IO::TimeoutError.new("Read timed out") } + wait_readable(io, timeout) { raise IO::TimeoutError.new("Read timed out") } end private def wait_readable(io, timeout = nil, &) : Nil @@ -321,7 +321,7 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end private def wait_writable(io, timeout = nil) : Nil - wait(:io_write, io, :writers, timeout) { raise IO::TimeoutError.new("Write timed out") } + wait_writable(io, timeout) { raise IO::TimeoutError.new("Write timed out") } end private def wait_writable(io, timeout = nil, &) : Nil @@ -364,12 +364,6 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop else Fiber.suspend end - - {% if flag?(:preview_mt) %} - # we can safely reset readyness here, since we're about to retry the - # actual syscall - %pd.value.@{{waiters.id}}.@ready.set(false, :relaxed) - {% end %} end private def check_open(io : IO) diff --git a/src/crystal/system/unix/evented/waiters.cr b/src/crystal/system/unix/evented/waiters.cr index c98bdcb12ee0..25a0125670c8 100644 --- a/src/crystal/system/unix/evented/waiters.cr +++ b/src/crystal/system/unix/evented/waiters.cr @@ -4,19 +4,31 @@ # Thread safe: mutations are protected with a lock, and race conditions are # handled through the ready atomic. struct Crystal::Evented::Waiters - @ready = Atomic(Bool).new(false) + {% if flag?(:preview_mt) %} + @ready = false + @closed = false + {% end %} @lock = SpinLock.new @list = PointerLinkedList(Event).new + # Adds an event to the waiting list. May return false immediately if another + # thread marked the list as ready in parallel, returns true otherwise. def add(event : Pointer(Event)) : Bool {% if flag?(:preview_mt) %} - # check for readiness since another thread running the evloop might be - # trying to dequeue an event while we're waiting on the lock (failure to - # notice notice the IO is ready) - return false if ready? - @lock.sync do - return false if ready? + if @closed + # another thread closed the fd or we received a fd error or hup event: + # the fd will never block again + return false + end + + if @ready + # another thread readied the fd before the current thread got to add + # the event: don't block and resets @ready for the next loop + @ready = false + return false + end + @list.push(event) end {% else %} @@ -26,21 +38,13 @@ struct Crystal::Evented::Waiters true end - def delete(event) : Nil + def delete(event : Pointer(Event)) : Nil @lock.sync { @list.delete(event) } end - def consume_each(&) : Nil - @lock.sync do - @list.consume_each { |event| yield event } - end - end - - def ready? : Bool - @ready.swap(false, :relaxed) - end - - def ready(& : Pointer(Event) -> Bool) : Nil + # Removes one pending event or marks the list as ready when there are no + # pending events (we got notified of readiness before a thread enqueued). + def ready_one(& : Pointer(Event) -> Bool) : Nil @lock.sync do {% if flag?(:preview_mt) %} # loop until the block succesfully processes an event (it may have to @@ -51,7 +55,7 @@ struct Crystal::Evented::Waiters else # no event queued but another thread may be waiting for the lock to # add an event: set as ready to resolve the race condition - @ready.set(true, :relaxed) + @ready = true return end end @@ -62,4 +66,16 @@ struct Crystal::Evented::Waiters {% end %} end end + + # Dequeues all pending events and marks the list as closed. This must be + # called when a fd is closed or an error or hup event occurred. + def ready_all(& : Pointer(Event) ->) : Nil + @lock.sync do + @list.consume_each { |event| yield event } + + {% if flag?(:preview_mt) %} + @closed = true + {% end %} + end + end end diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index 95702d56e293..5c04d47b128e 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -123,8 +123,8 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop if (kevent.value.fflags & LibC::EV_EOF) == LibC::EV_EOF # apparently some systems may report EOF on write with EVFILT_READ instead # of EVFILT_WRITE, so let's wake all waiters: - pd.value.@readers.consume_each { |event| unsafe_resume_io(event) } - pd.value.@writers.consume_each { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } + pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } return end @@ -132,16 +132,16 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop when LibC::EVFILT_READ if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR # OPTIMIZE: pass errno (kevent.data) through PollDescriptor - pd.value.@readers.consume_each { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } else - pd.value.@readers.ready { |event| unsafe_resume_io(event) } + pd.value.@readers.ready_one { |event| unsafe_resume_io(event) } end when LibC::EVFILT_WRITE if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR # OPTIMIZE: pass errno (kevent.data) through PollDescriptor - pd.value.@writers.consume_each { |event| unsafe_resume_io(event) } + pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } else - pd.value.@writers.ready { |event| unsafe_resume_io(event) } + pd.value.@writers.ready_one { |event| unsafe_resume_io(event) } end end end From 23eb8e93efeb1d42fbb4e1a4cbed5c332d29a334 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 3 Oct 2024 16:21:45 +0200 Subject: [PATCH 34/56] fixup! Add C bindings for getrlimit(RLIMIT_NOFILE) --- src/lib_c/aarch64-linux-musl/c/sys/resource.cr | 13 ++----------- src/lib_c/i386-linux-musl/c/sys/resource.cr | 13 ++----------- src/lib_c/x86_64-linux-musl/c/sys/resource.cr | 13 ++----------- 3 files changed, 6 insertions(+), 33 deletions(-) diff --git a/src/lib_c/aarch64-linux-musl/c/sys/resource.cr b/src/lib_c/aarch64-linux-musl/c/sys/resource.cr index b0b1dc6ec2b2..656e43cb0379 100644 --- a/src/lib_c/aarch64-linux-musl/c/sys/resource.cr +++ b/src/lib_c/aarch64-linux-musl/c/sys/resource.cr @@ -6,6 +6,8 @@ lib LibC rlim_max : RlimT end + RLIMIT_NOFILE = 7 + fun getrlimit(Int, Rlimit*) : Int RLIMIT_STACK = 3 @@ -33,15 +35,4 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 - - alias RlimT = ULongLong - - struct Rlimit - rlim_cur : RlimT - rlim_max : RlimT - end - - RLIMIT_NOFILE = 7 - - fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/i386-linux-musl/c/sys/resource.cr b/src/lib_c/i386-linux-musl/c/sys/resource.cr index b0b1dc6ec2b2..656e43cb0379 100644 --- a/src/lib_c/i386-linux-musl/c/sys/resource.cr +++ b/src/lib_c/i386-linux-musl/c/sys/resource.cr @@ -6,6 +6,8 @@ lib LibC rlim_max : RlimT end + RLIMIT_NOFILE = 7 + fun getrlimit(Int, Rlimit*) : Int RLIMIT_STACK = 3 @@ -33,15 +35,4 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 - - alias RlimT = ULongLong - - struct Rlimit - rlim_cur : RlimT - rlim_max : RlimT - end - - RLIMIT_NOFILE = 7 - - fun getrlimit(resource : Int, rlim : Rlimit*) : Int end diff --git a/src/lib_c/x86_64-linux-musl/c/sys/resource.cr b/src/lib_c/x86_64-linux-musl/c/sys/resource.cr index b0b1dc6ec2b2..656e43cb0379 100644 --- a/src/lib_c/x86_64-linux-musl/c/sys/resource.cr +++ b/src/lib_c/x86_64-linux-musl/c/sys/resource.cr @@ -6,6 +6,8 @@ lib LibC rlim_max : RlimT end + RLIMIT_NOFILE = 7 + fun getrlimit(Int, Rlimit*) : Int RLIMIT_STACK = 3 @@ -33,15 +35,4 @@ lib LibC RUSAGE_CHILDREN = -1 fun getrusage(who : Int, usage : RUsage*) : Int16 - - alias RlimT = ULongLong - - struct Rlimit - rlim_cur : RlimT - rlim_max : RlimT - end - - RLIMIT_NOFILE = 7 - - fun getrlimit(resource : Int, rlim : Rlimit*) : Int end From eb35831ac073a07af72f39769e21459d45f0e374 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 10 Oct 2024 11:08:22 +0200 Subject: [PATCH 35/56] Arena: tests + fixes + use IndexError Adds tests for the arena behavior and of course fixes obvious issues (e.g. generation wasn't actually checked). Relies on Slice's bound checks to raise IndexError as proposed by @straight-shoota during the review. --- spec/std/crystal/evented/arena_spec.cr | 173 ++++++++++++++++++ src/crystal/system/unix/epoll/event_loop.cr | 2 +- src/crystal/system/unix/evented/arena.cr | 55 +++--- src/crystal/system/unix/evented/event_loop.cr | 41 +++-- 4 files changed, 225 insertions(+), 46 deletions(-) create mode 100644 spec/std/crystal/evented/arena_spec.cr diff --git a/spec/std/crystal/evented/arena_spec.cr b/spec/std/crystal/evented/arena_spec.cr new file mode 100644 index 000000000000..bb2b967acb96 --- /dev/null +++ b/spec/std/crystal/evented/arena_spec.cr @@ -0,0 +1,173 @@ +{% skip_file unless flag?(:unix) %} + +require "spec" +require "../../../../src/crystal/system/unix/evented/arena" + +describe Crystal::Evented::Arena do + describe "#lazy_allocate" do + it "yields block once" do + arena = Crystal::Evented::Arena(Int32).new(32) + pointer = nil + index = nil + called = 0 + + ptr1, idx1 = arena.lazy_allocate(0) do |ptr, idx| + pointer = ptr + index = idx + called += 1 + end + called.should eq(1) + + ptr2, idx2 = arena.lazy_allocate(0) do |ptr, idx| + called += 1 + end + called.should eq(1) + + pointer.should_not be_nil + index.should_not be_nil + + ptr1.should eq(pointer) + idx1.should eq(index) + + ptr2.should eq(pointer) + idx2.should eq(index) + end + + it "allocates up to capacity" do + arena = Crystal::Evented::Arena(Int32).new(32) + + objects = 32.times.map do |i| + arena.lazy_allocate(i) { |pointer| pointer.value = i } + end + objects.each do |(pointer, index)| + arena.get(index).should eq(pointer) + pointer.value.should eq(index.index) + end + end + + it "checks bounds" do + arena = Crystal::Evented::Arena(Int32).new(32) + expect_raises(IndexError) { arena.lazy_allocate(-1) {} } + expect_raises(IndexError) { arena.lazy_allocate(33) {} } + end + end + + describe "#get" do + it "returns previously allocated object" do + arena = Crystal::Evented::Arena(Int32).new(32) + pointer, index = arena.lazy_allocate(30) { |ptr| ptr.value = 654321 } + + 2.times do + ptr = arena.get(index) + ptr.should eq(pointer) + ptr.value.should eq(654321) + end + + # not allocated: + expect_raises(RuntimeError) do + arena.get(Crystal::Evented::Arena::Index.new(10, 0)) + end + end + + it "checks generation" do + arena = Crystal::Evented::Arena(Int32).new(32) + called = 0 + + _, index1 = arena.lazy_allocate(2) { called += 1 } + called.should eq(1) + + arena.free(index1) { } + expect_raises(RuntimeError) { arena.get(index1) } + + _, index2 = arena.lazy_allocate(2) { called += 1 } + called.should eq(2) + expect_raises(RuntimeError) { arena.get(index1) } + + # doesn't raise: + arena.get(index2) + end + + it "checks out of bounds" do + arena = Crystal::Evented::Arena(Int32).new(32) + expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(-1, 0)) } + expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(33, 0)) } + end + end + + describe "#get?" do + it "returns previously allocated object" do + arena = Crystal::Evented::Arena(Int32).new(32) + pointer, index = arena.lazy_allocate(30) { |ptr| ptr.value = 654321 } + + 2.times do + ptr = arena.get?(index) + ptr.should eq(pointer) + ptr.not_nil!.value.should eq(654321) + end + + arena.get?(Crystal::Evented::Arena::Index.new(10, 0)).should be_nil + end + + it "checks generation" do + arena = Crystal::Evented::Arena(Int32).new(32) + called = 0 + + pointer1, index1 = arena.lazy_allocate(2) { called += 1 } + called.should eq(1) + + arena.free(index1) { } + arena.get?(index1).should be_nil + + pointer2, index2 = arena.lazy_allocate(2) { called += 1 } + called.should eq(2) + arena.get?(index1).should be_nil + arena.get?(index2).should eq(pointer2) + end + + it "checks out of bounds" do + arena = Crystal::Evented::Arena(Int32).new(32) + arena.get?(Crystal::Evented::Arena::Index.new(-1, 0)).should be_nil + arena.get?(Crystal::Evented::Arena::Index.new(33, 0)).should be_nil + end + end + + describe "#free" do + it "deallocates the object" do + arena = Crystal::Evented::Arena(Int32).new(32) + pointer, index1 = arena.lazy_allocate(3) { } + pointer.value = 123 + + arena.free(index1) { } + + pointer, index2 = arena.lazy_allocate(3) { } + index2.should_not eq(index1) + pointer.value.should eq(0) + end + + it "checks generation" do + arena = Crystal::Evented::Arena(Int32).new(32) + called = 0 + + _, index1 = arena.lazy_allocate(1) { } + arena.free(index1) { called += 1} + called.should eq(1) + + _, index2 = arena.lazy_allocate(1) { } + arena.free(index1) { called += 1 } + called.should eq(1) + + arena.free(index2) { called += 1 } + called.should eq(2) + end + + it "checks out of bounds" do + arena = Crystal::Evented::Arena(Int32).new(32) + called = 0 + + arena.free(Crystal::Evented::Arena::Index.new(-1, 0)) { called += 1 } + arena.free(Crystal::Evented::Arena::Index.new(33, 0)) { called += 1 } + + called.should eq(0) + end + end +end diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr index 58e5ac880384..cd83517ea356 100644 --- a/src/crystal/system/unix/epoll/event_loop.cr +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -92,7 +92,7 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop index = Evented::Arena::Index.new(epoll_event.value.data.u64) events = epoll_event.value.events - Crystal.trace :evloop, "event", fd: index.to_i, index: index.to_i64, events: events + Crystal.trace :evloop, "event", fd: index.index, index: index.to_i64, events: events pd = Evented.arena.get(index) diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/system/unix/evented/arena.cr index 54749bcf696a..ac254e23de9e 100644 --- a/src/crystal/system/unix/evented/arena.cr +++ b/src/crystal/system/unix/evented/arena.cr @@ -47,7 +47,7 @@ class Crystal::Evented::Arena(T) end def initialize(data : UInt64) - @data = data.unsafe_as(Int64) + @data = data.to_i64! end # Returns the generation number. @@ -56,7 +56,7 @@ class Crystal::Evented::Arena(T) end # Returns the actual index. - def to_i : Int32 + def index : Int32 (@data >> 32).to_i32! end @@ -65,7 +65,7 @@ class Crystal::Evented::Arena(T) end def to_u64 : UInt64 - @data.unsafe_as(UInt64) + @data.to_u64! end def valid? : Bool @@ -93,6 +93,14 @@ class Crystal::Evented::Arena(T) @buffer : Slice(Entry(T)) {% unless flag?(:preview_mt) %} + # Remember the maximum allocated fd ever; + # + # This is specific to `EventLoop#after_fork` that needs to iterate the arena + # for registered fds in epoll/kqueue to re-add them to the new epoll/kqueue + # instances. Without this upper limit we'd iterate the whole arena which + # would lead the kernel to try and allocate the whole mmap in physical + # memory (instead of virtual memory) which would at best be a waste, and a + # worst fill the memory (e.g. unlimited open files). @maximum = 0 {% end %} @@ -154,7 +162,7 @@ class Crystal::Evented::Arena(T) # Raises if the generation has changed (i.e. the object has been freed then reallocated). # Raises if *index* is negative. def get(index : Index) : Pointer(T) - entry = at(index.to_i) + entry = at(index) entry.value.pointer end @@ -163,7 +171,7 @@ class Crystal::Evented::Arena(T) # # Raises if *index* is negative. def get?(index : Index) : Pointer(T)? - if entry = at?(index.to_i) + if entry = at?(index) entry.value.pointer end end @@ -173,50 +181,43 @@ class Crystal::Evented::Arena(T) # # Raises if *index* is negative. def free(index : Index, &) : Nil - return unless entry = at?(index.to_i) + return unless entry = at?(index.index) entry.value.@lock.sync do return unless entry.value.allocated? return unless entry.value.generation == index.generation - yield entry.value.pointer - ensure - entry.value.free + begin + yield entry.value.pointer + ensure + entry.value.free + end end end private def at(index : Index) : Pointer(Entry(T)) - entry = at(index.to_i) + entry = at(index.index) unless entry.value.allocated? - raise RuntimeError.new("#{self.class.name}: object not allocated at index #{index.to_i}") + raise RuntimeError.new("#{self.class.name}: object not allocated at index #{index.index}") end - unless entry.value.generation == generation - raise RuntimeError.new("#{self.class.name}: object generation changed at index #{index.to_i} (#{index.generation} => #{entry.value.generation})") + unless entry.value.generation == index.generation + raise RuntimeError.new("#{self.class.name}: object generation changed at index #{index.index} (#{index.generation} => #{entry.value.generation})") end entry end - private def at?(index : Index) : Pointer(Entry(T)) - return unless entry = at?(index.to_i) + private def at?(index : Index) : Pointer(Entry(T))? + return unless entry = at?(index.index) return unless entry.value.allocated? - return unless entry.value.generation == generation + return unless entry.value.generation == index.generation entry end private def at(index : Int32) : Pointer(Entry(T)) - if index.negative? - raise ArgumentError.new("#{self.class.name}: negative index #{index}") - end - if index >= @buffer.size - raise ArgumentError.new("#{self.class.name}: out of bounds index #{index} >= #{@buffer.size}") - end - @buffer.to_unsafe + index + (@buffer + index).to_unsafe end private def at?(index : Int32) : Pointer(Entry(T))? - if index.negative? - raise ArgumentError.new("#{self.class.name}: negative index #{index}") - end - if index < @buffer.size + if 0 <= index < @buffer.size @buffer.to_unsafe + index end end diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 8a5594ff4af4..279ee568089a 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -12,27 +12,32 @@ module Crystal::System::Socket end module Crystal::Evented - # The choice of a generational arena permits to avoid pushing raw pointers into - # IO objects into kernel data structures that are unknown to the GC, and to - # safely check whether the allocation is still valid before trying to - # dereference the pointer. Since `PollDescriptor` also doesn't have pointers to - # the actual IO object, it won't prevent the GC from collecting lost IO objects - # (and spares us from using + # The generational arena: # - # To a lesser extent, it also allows to keep the `PollDescriptor` allocated - # together in the same region, and polluting the IO object itself with specific - # evloop data (except for the generation index). + # 1. decorrelates the fd from the IO since the evloop only really cares about + # the fd state and to resume pending fibers (it could monitor a fd without + # an IO object); # - # Takes advantage of the fd being unique per process and that the operating - # system will always reuse the lowest fd (POSIX compliance) and will only grow - # when the process needs that many file descriptors, so the allocated memory - # region won't grow larger than necessary. This assumption allows the arena to - # skip maintaining a list of free indexes. + # 2. permits to avoid pushing raw pointers to IO objects into kernel data + # structures that are unknown to the GC, and to safely check whether the + # allocation is still valid before trying to dereference the pointer. Since + # `PollDescriptor` also doesn't have pointers to the actual IO object, it + # won't prevent the GC from collecting lost IO objects (and spares us from + # using weak references). # - # Some systems may deviate from the POSIX default, but all systems seem to - # follow it, as it allows optimizations to the OS (it can reuse already - # allocated resources), and either the man page explicitly says so (Linux), or - # they don't (BSD) and they must follow the POSIX definition. + # 3. to a lesser extent, it also allows to keep the `PollDescriptor` allocated + # together in the same region, and polluting the IO object itself with + # specific evloop data (except for the generation index). + # + # The implementation takes advantage of the fd being unique per process and + # that the operating system will always reuse the lowest fd (POSIX compliance) + # and will only grow when the process needs that many file descriptors, so the + # allocated memory region won't grow larger than necessary. This assumption + # allows the arena to skip maintaining a list of free indexes. Some systems + # may deviate from the POSIX default, but all systems seem to follow it, as it + # allows optimizations to the OS (it can reuse already allocated resources), + # and either the man page explicitly says so (Linux), or they don't (BSD) and + # they must follow the POSIX definition. protected class_getter arena = Arena(PollDescriptor).new(max_fds) private def self.max_fds : Int32 From 7e9409797c97ef686a7ca1cd39edb9f07fa37e73 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 10 Oct 2024 18:54:05 +0200 Subject: [PATCH 36/56] EventLoop: follow suggestions by @straight-shoota Mostly improves readability. --- src/crystal/system/unix/epoll/event_loop.cr | 2 -- src/crystal/system/unix/evented/event_loop.cr | 6 ++--- src/crystal/system/unix/kqueue.cr | 7 ++++-- src/crystal/system/unix/kqueue/event_loop.cr | 25 ++++++++----------- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr index cd83517ea356..b209d2706aff 100644 --- a/src/crystal/system/unix/epoll/event_loop.cr +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -5,8 +5,6 @@ require "../timerfd" class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop def initialize - super - # the epoll instance @epoll = System::Epoll.new diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 279ee568089a..de20a4d49cd7 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -90,10 +90,8 @@ end # before suspending the fiber, then after resume it will raise # `IO::TimeoutError` if the event timed out, and continue otherwise. abstract class Crystal::Evented::EventLoop < Crystal::EventLoop - def initialize - @lock = SpinLock.new # protects parallel accesses to @timers - @timers = Timers.new - end + @lock = SpinLock.new # protects parallel accesses to @timers + @timers = Timers.new # reset the mutexes since another thread may have acquired the lock of one # event loop, which would prevent closing file descriptors for example. diff --git a/src/crystal/system/unix/kqueue.cr b/src/crystal/system/unix/kqueue.cr index ff36ca734487..3c3f825f68bf 100644 --- a/src/crystal/system/unix/kqueue.cr +++ b/src/crystal/system/unix/kqueue.cr @@ -10,7 +10,10 @@ struct Crystal::System::Kqueue {% else %} LibC.kqueue {% end %} - raise RuntimeError.from_errno("kqueue1") if @kq == -1 + if @kq == -1 + function_name = {% if LibC.has_method?(:kqueue1) %} "kqueue1" {% else %} "kqueue" {% end %} + raise RuntimeError.from_errno(function_name) + end end # Helper to register a single event. Returns immediately. @@ -42,7 +45,7 @@ struct Crystal::System::Kqueue tsp = Pointer(LibC::Timespec).null end - changes = uninitialized LibC::Kevent[0] + changes = Slice(LibC::Kevent).empty count = 0 loop do diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index 5c04d47b128e..46a215fcc5d0 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -2,15 +2,15 @@ require "../evented/event_loop" require "../kqueue" class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop + # the following are arbitrary numbers to identify specific events INTERRUPT_IDENTIFIER = 9 + TIMER_IDENTIFIER = 10 {% unless LibC.has_constant?(:EVFILT_USER) %} @pipe = uninitialized LibC::Int[2] {% end %} def initialize - super - # the kqueue instance @kqueue = System::Kqueue.new @@ -96,10 +96,10 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop end {% else %} if kevent.value.filter == LibC::EVFILT_READ && kevent.value.ident == @pipe[0] - @interrupted.clear - byte = 0_u8 - ret = LibC.read(@pipe[0], pointerof(byte), 1) + ident = 0 + ret = LibC.read(@pipe[0], pointerof(ident), sizeof(Int32)) raise RuntimeError.from_errno("read") if ret == -1 + @interrupted.clear if ident == INTERRUPT_IDENTIFIER return true end {% end %} @@ -156,8 +156,8 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop LibC::EV_ADD | LibC::EV_ONESHOT, LibC::NOTE_FFCOPY | LibC::NOTE_TRIGGER | 1_u16) {% else %} - byte = 1_u8 - ret = LibC.write(@pipe[1], pointerof(byte), sizeof(typeof(byte))) + ident = INTERRUPT_IDENTIFIER + ret = LibC.write(@pipe[1], pointerof(ident), sizeof(Int32)) raise RuntimeError.from_errno("write") if ret == -1 {% end %} end @@ -167,10 +167,8 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop # register both read and write events kevents = uninitialized LibC::Kevent[2] - 2.times do |i| + {LibC::EVFILT_READ, LibC::EVFILT_WRITE}.each_with_index do |filter, i| kevent = kevents.to_unsafe + i - filter = i == 0 ? LibC::EVFILT_READ : LibC::EVFILT_WRITE - udata = {% if flag?(:bits64) %} Pointer(Void).new(index.to_u64) @@ -199,9 +197,8 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop # unregister both read and write events kevents = uninitialized LibC::Kevent[2] - 2.times do |i| + {LibC::EVFILT_READ, LibC::EVFILT_WRITE}.each_with_index do |filter, i| kevent = kevents.to_unsafe + i - filter = i == 0 ? LibC::EVFILT_READ : LibC::EVFILT_WRITE System::Kqueue.set(kevent, fd, filter, LibC::EV_DELETE) end @@ -233,9 +230,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop 0 {% end %} - # use the evloop address as the unique identifier for the timer kevent - ident = LibC::SizeT.new!(self.as(Void*).address) - @kqueue.kevent(ident, LibC::EVFILT_TIMER, flags, fflags, data) do + @kqueue.kevent(TIMER_IDENTIFIER, LibC::EVFILT_TIMER, flags, fflags, data) do raise RuntimeError.from_errno("kevent") unless Errno.value == Errno::ENOENT end end From 08214a8ee97438699e24c60c1b705d2571b5d9d1 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 10 Oct 2024 19:05:18 +0200 Subject: [PATCH 37/56] EventLoop: get rid of the macro --- src/crystal/system/unix/evented/event_loop.cr | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index de20a4d49cd7..8402b18a29fa 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -316,57 +316,64 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end private def wait_readable(io, timeout = nil) : Nil - wait_readable(io, timeout) { raise IO::TimeoutError.new("Read timed out") } + wait_readable(io, timeout) do + raise IO::TimeoutError.new("Read timed out") + end end - private def wait_readable(io, timeout = nil, &) : Nil - wait(:io_read, io, :readers, timeout) { yield } + private def wait_writable(io, timeout = nil) : Nil + wait_writable(io, timeout) do + raise IO::TimeoutError.new("Write timed out") + end end - private def wait_writable(io, timeout = nil) : Nil - wait_writable(io, timeout) { raise IO::TimeoutError.new("Write timed out") } + private def wait_readable(io, timeout = nil, &) : Nil + yield if wait(:io_read, io, timeout) do |pd, event| + # don't wait if the waiter has already been marked ready (see Waiters#add) + return unless pd.value.@readers.add(event) + end end private def wait_writable(io, timeout = nil, &) : Nil - wait(:io_write, io, :writers, timeout) { yield } + yield if wait(:io_write, io, timeout) do |pd, event| + # don't wait if the waiter has already been marked ready (see Waiters#add) + return unless pd.value.@writers.add(event) + end end - private macro wait(type, io, waiters, timeout, &) + private def wait(type : Evented::Event::Type, io, timeout, &) # get or allocate the poll descriptor - if (%index = {{io}}.__evloop_data).valid? - %pd = Evented.arena.get(%index) + if (index = io.__evloop_data).valid? + pd = Evented.arena.get(index) else - %pd, %index = Evented.arena.lazy_allocate({{io}}.fd) do |pd, index| + pd, index = Evented.arena.lazy_allocate(io.fd) do |pd, index| # register the fd with the event loop (once), it should usually merely add # the fd to the current evloop but may "transfer" the ownership from # another event loop: - {{io}}.__evloop_data = index - pd.value.take_ownership(self, {{io}}.fd, index) + io.__evloop_data = index + pd.value.take_ownership(self, io.fd, index) end end # create an event (on the stack) - %event = Evented::Event.new({{type}}, Fiber.current, %index, {{timeout}}) + event = Evented::Event.new(type, Fiber.current, index, timeout) - # try to add the event to the waiting list - # don't wait if the waiter has already been marked ready (see Waiters#add) - return unless %pd.value.@{{waiters.id}}.add(pointerof(%event)) + # add the event to the waiting list + yield pd, pointerof(event) - if %event.wake_at? - add_timer(pointerof(%event)) + if event.wake_at? + add_timer(pointerof(event)) Fiber.suspend - if %event.timed_out? - return {{yield}} - else - # nothing to do: either the timer triggered which means it was dequeued, - # or `#unsafe_resume_io` was called to resume the IO and the timer got - # deleted from the timers before the fiber got reenqueued. - end - else - Fiber.suspend + # no need to delete the timer: either it triggered which means it was + # dequeued, or `#unsafe_resume_io` was called to resume the IO and the + # timer got deleted from the timers before the fiber got reenqueued. + return event.timed_out? end + + Fiber.suspend + false end private def check_open(io : IO) From 762f9b41f227ddb652adab7ad6456c48868be751 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 11 Oct 2024 12:39:17 +0200 Subject: [PATCH 38/56] Waiters: tests + fix segfault on delete + mt safe by default --- spec/std/crystal/evented/waiters_spec.cr | 169 +++++++++++++++++++++ src/crystal/system/unix/evented/waiters.cr | 78 ++++------ 2 files changed, 202 insertions(+), 45 deletions(-) create mode 100644 spec/std/crystal/evented/waiters_spec.cr diff --git a/spec/std/crystal/evented/waiters_spec.cr b/spec/std/crystal/evented/waiters_spec.cr new file mode 100644 index 000000000000..c2f154dcf631 --- /dev/null +++ b/spec/std/crystal/evented/waiters_spec.cr @@ -0,0 +1,169 @@ +{% skip_file unless flag?(:unix) %} + +require "spec" +require "../../../../src/crystal/system/unix/evented/waiters" + +describe Crystal::Evented::Waiters do + describe "#add" do + it "adds event to list" do + waiters = Crystal::Evented::Waiters.new + + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + ret = waiters.add(pointerof(event)) + ret.should be_true + end + + it "doesn't add the event when the list is ready (race condition)" do + waiters = Crystal::Evented::Waiters.new + waiters.ready_one { true } + + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + ret = waiters.add(pointerof(event)) + ret.should be_false + waiters.@ready.should be_false + end + + it "doesn't add the event when the list is always ready" do + waiters = Crystal::Evented::Waiters.new + waiters.ready_all { } + + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + ret = waiters.add(pointerof(event)) + ret.should be_false + waiters.@always_ready.should be_true + end + end + + describe "#delete" do + it "removes the event from the list" do + waiters = Crystal::Evented::Waiters.new + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + + waiters.add(pointerof(event)) + waiters.delete(pointerof(event)) + + called = false + waiters.ready_one { called = true } + called.should be_false + end + + it "does nothing when the event isn't in the list" do + waiters = Crystal::Evented::Waiters.new + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + waiters.delete(pointerof(event)) + end + end + + describe "#ready_one" do + it "marks the list as ready when empty (race condition)" do + waiters = Crystal::Evented::Waiters.new + called = false + + waiters.ready_one { called = true } + + called.should be_false + waiters.@ready.should be_true + end + + it "dequeues events in FIFO order" do + waiters = Crystal::Evented::Waiters.new + event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event3 = Crystal::Evented::Event.new(:io_read, Fiber.current) + called = 0 + + waiters.add(pointerof(event1)) + waiters.add(pointerof(event2)) + waiters.add(pointerof(event3)) + + 3.times do + waiters.ready_one do |event| + case called += 1 + when 1 then event.should eq(pointerof(event1)) + when 2 then event.should eq(pointerof(event2)) + when 3 then event.should eq(pointerof(event3)) + end + true + end + end + called.should eq(3) + waiters.@ready.should be_false + + waiters.ready_one do + called += 1 + true + end + called.should eq(3) + waiters.@ready.should be_true + end + + it "dequeues events until the block returns true" do + waiters = Crystal::Evented::Waiters.new + event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event3 = Crystal::Evented::Event.new(:io_read, Fiber.current) + called = 0 + + waiters.add(pointerof(event1)) + waiters.add(pointerof(event2)) + waiters.add(pointerof(event3)) + + waiters.ready_one do |event| + (called += 1) == 2 + end + called.should eq(2) + waiters.@ready.should be_false + end + + it "dequeues events until empty and marks the list as ready" do + waiters = Crystal::Evented::Waiters.new + event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) + called = 0 + + waiters.add(pointerof(event1)) + waiters.add(pointerof(event2)) + + waiters.ready_one do |event| + called += 1 + false + end + called.should eq(2) + waiters.@ready.should be_true + end + end + + describe "#ready_all" do + it "marks the list as always ready" do + waiters = Crystal::Evented::Waiters.new + called = false + + waiters.ready_all { called = true } + + called.should be_false + waiters.@always_ready.should be_true + end + + it "dequeues all events" do + waiters = Crystal::Evented::Waiters.new + event1 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event2 = Crystal::Evented::Event.new(:io_read, Fiber.current) + event3 = Crystal::Evented::Event.new(:io_read, Fiber.current) + called = 0 + + waiters.add(pointerof(event1)) + waiters.add(pointerof(event2)) + waiters.add(pointerof(event3)) + + waiters.ready_all do |event| + case called += 1 + when 1 then event.should eq(pointerof(event1)) + when 2 then event.should eq(pointerof(event2)) + when 3 then event.should eq(pointerof(event3)) + end + end + called.should eq(3) + waiters.@always_ready.should be_true + end + end +end diff --git a/src/crystal/system/unix/evented/waiters.cr b/src/crystal/system/unix/evented/waiters.cr index 25a0125670c8..bc79c5dc32c1 100644 --- a/src/crystal/system/unix/evented/waiters.cr +++ b/src/crystal/system/unix/evented/waiters.cr @@ -1,81 +1,69 @@ +require "./event" + # A FIFO queue of `Event` waiting on the same operation (either read or write) # for a fd. See `PollDescriptor`. # # Thread safe: mutations are protected with a lock, and race conditions are # handled through the ready atomic. struct Crystal::Evented::Waiters - {% if flag?(:preview_mt) %} - @ready = false - @closed = false - {% end %} - @lock = SpinLock.new @list = PointerLinkedList(Event).new + @lock = SpinLock.new + @ready = false + @always_ready = false # Adds an event to the waiting list. May return false immediately if another # thread marked the list as ready in parallel, returns true otherwise. def add(event : Pointer(Event)) : Bool - {% if flag?(:preview_mt) %} - @lock.sync do - if @closed - # another thread closed the fd or we received a fd error or hup event: - # the fd will never block again - return false - end - - if @ready - # another thread readied the fd before the current thread got to add - # the event: don't block and resets @ready for the next loop - @ready = false - return false - end + @lock.sync do + if @always_ready + # another thread closed the fd or we received a fd error or hup event: + # the fd will never block again + return false + end - @list.push(event) + if @ready + # another thread readied the fd before the current thread got to add + # the event: don't block and resets @ready for the next loop + @ready = false + return false end - {% else %} - @list.push(event) - {% end %} + @list.push(event) + end true end def delete(event : Pointer(Event)) : Nil - @lock.sync { @list.delete(event) } + @lock.sync do + @list.delete(event) if event.value.next + end end # Removes one pending event or marks the list as ready when there are no # pending events (we got notified of readiness before a thread enqueued). def ready_one(& : Pointer(Event) -> Bool) : Nil @lock.sync do - {% if flag?(:preview_mt) %} - # loop until the block succesfully processes an event (it may have to - # dequeue the timeout from timers) - loop do - if event = @list.shift? - break if yield event - else - # no event queued but another thread may be waiting for the lock to - # add an event: set as ready to resolve the race condition - @ready = true - return - end - end - {% else %} + # loop until the block succesfully processes an event (it may have to + # dequeue the timeout from timers) + loop do if event = @list.shift? - yield event + break if yield event + else + # no event queued but another thread may be waiting for the lock to + # add an event: set as ready to resolve the race condition + @ready = true + return end - {% end %} + end end end - # Dequeues all pending events and marks the list as closed. This must be + # Dequeues all pending events and marks the list as always ready. This must be # called when a fd is closed or an error or hup event occurred. def ready_all(& : Pointer(Event) ->) : Nil @lock.sync do @list.consume_each { |event| yield event } - - {% if flag?(:preview_mt) %} - @closed = true - {% end %} + @always_ready = true end end end From a5b8ae041bc679d309d632013358b6094a690d1c Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 11 Oct 2024 13:59:31 +0200 Subject: [PATCH 39/56] PollDescriptor: tests --- .../crystal/evented/poll_descriptor_spec.cr | 101 ++++++++++++++++++ .../system/unix/evented/poll_descriptor.cr | 2 +- 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 spec/std/crystal/evented/poll_descriptor_spec.cr diff --git a/spec/std/crystal/evented/poll_descriptor_spec.cr b/spec/std/crystal/evented/poll_descriptor_spec.cr new file mode 100644 index 000000000000..a2c1c98bf96f --- /dev/null +++ b/spec/std/crystal/evented/poll_descriptor_spec.cr @@ -0,0 +1,101 @@ +{% skip_file unless flag?(:unix) %} + +require "spec" +require "../../../../src/crystal/system/unix/evented/poll_descriptor" + +class Crystal::Evented::FakeLoop < Crystal::Evented::EventLoop + getter operations = [] of {Symbol, Int32, Crystal::Evented::Arena::Index | Bool} + + private def system_run(blocking : Bool) : Nil + end + + private def interrupt : Nil + end + + protected def system_add(fd : Int32, index : Arena::Index) : Nil + operations << {:add, fd, index} + end + + protected def system_del(fd : Int32, closing = true) : Nil + operations << {:del, fd, closing} + end + + protected def system_del(fd : Int32, closing = true, &) : Nil + operations << {:del, fd, closing} + end + + private def system_set_timer(time : Time::Span?) : Nil + end +end + +describe Crystal::Evented::Waiters do + describe "#take_ownership" do + it "associates a poll descriptor to an evloop instance" do + fd = Int32::MAX + pd = Crystal::Evented::PollDescriptor.new + index = Crystal::Evented::Arena::Index.new(fd, 0) + evloop = Crystal::Evented::FakeLoop.new + + pd.take_ownership(evloop, fd, index) + pd.@event_loop.should be(evloop) + + evloop.operations.should eq([ + {:add, fd, index}, + ]) + end + + it "moves a poll descriptor to another evloop instance" do + fd = Int32::MAX + pd = Crystal::Evented::PollDescriptor.new + index = Crystal::Evented::Arena::Index.new(fd, 0) + + evloop1 = Crystal::Evented::FakeLoop.new + evloop2 = Crystal::Evented::FakeLoop.new + + pd.take_ownership(evloop1, fd, index) + pd.take_ownership(evloop2, fd, index) + + pd.@event_loop.should be(evloop2) + + evloop1.operations.should eq([ + {:add, fd, index}, + {:del, fd, false}, + ]) + evloop2.operations.should eq([ + {:add, fd, index}, + ]) + end + + it "can't move to the current evloop" do + fd = Int32::MAX + pd = Crystal::Evented::PollDescriptor.new + index = Crystal::Evented::Arena::Index.new(fd, 0) + + evloop = Crystal::Evented::FakeLoop.new + + pd.take_ownership(evloop, fd, index) + expect_raises(Exception) { pd.take_ownership(evloop, fd, index) } + end + + it "can't move with pending waiters" do + fd = Int32::MAX + pd = Crystal::Evented::PollDescriptor.new + index = Crystal::Evented::Arena::Index.new(fd, 0) + event = Crystal::Evented::Event.new(:io_read, Fiber.current) + + evloop1 = Crystal::Evented::FakeLoop.new + pd.take_ownership(evloop1, fd, index) + pd.@readers.add(pointerof(event)) + + evloop2 = Crystal::Evented::FakeLoop.new + expect_raises(RuntimeError) { pd.take_ownership(evloop2, fd, index) } + + pd.@event_loop.should be(evloop1) + + evloop1.operations.should eq([ + {:add, fd, index}, + ]) + evloop2.operations.should be_empty + end + end +end diff --git a/src/crystal/system/unix/evented/poll_descriptor.cr b/src/crystal/system/unix/evented/poll_descriptor.cr index fdfb7a556bbe..e9779a71d6ab 100644 --- a/src/crystal/system/unix/evented/poll_descriptor.cr +++ b/src/crystal/system/unix/evented/poll_descriptor.cr @@ -1,4 +1,4 @@ -require "./waiters" +require "./event_loop" # Information related to the evloop for a fd, such as the read and write queues # (waiting `Event`), as well as which evloop instance currently owns the fd. From 1c56ca3fdc2a9fb8cc93f85e663f14fb6c2c99b8 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 11 Oct 2024 15:59:27 +0200 Subject: [PATCH 40/56] Timers: tests + fix #delete + add #each --- spec/std/crystal/evented/timers_spec.cr | 101 ++++++++++++++++++ src/crystal/system/unix/evented/event_loop.cr | 7 +- src/crystal/system/unix/evented/timers.cr | 15 ++- 3 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 spec/std/crystal/evented/timers_spec.cr diff --git a/spec/std/crystal/evented/timers_spec.cr b/spec/std/crystal/evented/timers_spec.cr new file mode 100644 index 000000000000..5a75588e2ea0 --- /dev/null +++ b/spec/std/crystal/evented/timers_spec.cr @@ -0,0 +1,101 @@ +{% skip_file unless flag?(:unix) %} + +require "spec" +require "../../../../src/crystal/system/unix/evented/timers" + +describe Crystal::Evented::Timers do + it "#empty?" do + timers = Crystal::Evented::Timers.new + timers.empty?.should be_true + + event = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 7.seconds) + timers.add(pointerof(event)) + timers.empty?.should be_false + end + + it "#next_ready?" do + # empty + timers = Crystal::Evented::Timers.new + timers.next_ready?.should be_nil + + # with events + event = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 5.seconds) + timers.add(pointerof(event)) + timers.next_ready?.should eq(event.wake_at?) + end + + it "#dequeue_ready" do + timers = Crystal::Evented::Timers.new + + event1 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event2 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event3 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 1.minute) + + # empty + called = 0 + timers.dequeue_ready { called += 1 } + called.should eq(0) + + # add events in non chronological order + timers = Crystal::Evented::Timers.new + timers.add(pointerof(event1)) + timers.add(pointerof(event3)) + timers.add(pointerof(event2)) + + events = [] of Crystal::Evented::Event* + timers.dequeue_ready { |event|events << event } + + events.should eq([ + pointerof(event1), + pointerof(event2), + ]) + timers.empty?.should be_false + end + + it "#add" do + timers = Crystal::Evented::Timers.new + + event0 = Crystal::Evented::Event.new(:sleep, Fiber.current) + event1 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event2 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 2.minutes) + event3 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 1.minute) + + # add events in non chronological order + timers.add(pointerof(event1)).should be_true # added to the head (next ready) + timers.add(pointerof(event2)).should be_false + timers.add(pointerof(event3)).should be_false + + event0.wake_at = -1.minute + timers.add(pointerof(event0)).should be_true # added new head (next ready) + + events = [] of Crystal::Evented::Event* + timers.each { |event| events << event } + events.should eq([ + pointerof(event0), + pointerof(event1), + pointerof(event3), + pointerof(event2), + ]) + timers.empty?.should be_false + end + + it "#delete" do + event1 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event2 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 0.seconds) + event3 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 1.minute) + event4 = Crystal::Evented::Event.new(:sleep, Fiber.current, timeout: 4.minutes) + + # add events in non chronological order + timers = Crystal::Evented::Timers.new + timers.add(pointerof(event1)) + timers.add(pointerof(event3)) + timers.add(pointerof(event2)) + + timers.delete(pointerof(event1)).should eq({true, true}) # dequeued+removed head (next ready) + timers.delete(pointerof(event3)).should eq({true, false}) # dequeued + timers.delete(pointerof(event2)).should eq({true, true}) # dequeued+removed new head (next ready) + timers.empty?.should be_true + timers.delete(pointerof(event2)).should eq({false, false}) # not dequeued + timers.delete(pointerof(event4)).should eq({false, false}) # not dequeued + end +end diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 8402b18a29fa..3e5756dc3913 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -391,10 +391,11 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop protected def delete_timer(event : Evented::Event*) : Bool @lock.sync do - if index = @timers.delete(event) + dequeued, was_next_ready = @timers.delete(event) + if dequeued # update system timer if we deleted the next timer - system_set_timer(@timers.next_ready?) if index.zero? - return true + system_set_timer(@timers.next_ready?) if was_next_ready + return true if dequeued end end false diff --git a/src/crystal/system/unix/evented/timers.cr b/src/crystal/system/unix/evented/timers.cr index 098b11d522b5..f90280b4f734 100644 --- a/src/crystal/system/unix/evented/timers.cr +++ b/src/crystal/system/unix/evented/timers.cr @@ -66,12 +66,19 @@ struct Crystal::Evented::Timers end end - # Remove a timer from the list. Returns the index at which the event was, and - # `nil` otherwise. - def delete(event : Evented::Event*) : Int32? + # Remove a timer from the list. Returns a tuple(dequeued, was_next_ready) of + # booleans. The first bool tells whether the event was dequeued, in which case + # the second one tells if it was the next ready event. + def delete(event : Evented::Event*) : {Bool, Bool} if index = @list.index(event) @list.delete_at(index) - index + {true, index.zero?} + else + {false, false} end end + + def each(&) : Nil + @list.each { |event| yield event } + end end From d20e2cfb4d3ca88e89af3129ecad78b6134f3148 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 11 Oct 2024 19:49:55 +0200 Subject: [PATCH 41/56] Fix: crystal tool format --- spec/std/crystal/evented/arena_spec.cr | 6 +++--- spec/std/crystal/evented/timers_spec.cr | 2 +- src/crystal/system/unix/evented/event_loop.cr | 12 ++++++------ src/crystal/system/unix/kqueue/event_loop.cr | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/spec/std/crystal/evented/arena_spec.cr b/spec/std/crystal/evented/arena_spec.cr index bb2b967acb96..47b8f9ecdb3e 100644 --- a/spec/std/crystal/evented/arena_spec.cr +++ b/spec/std/crystal/evented/arena_spec.cr @@ -47,8 +47,8 @@ describe Crystal::Evented::Arena do it "checks bounds" do arena = Crystal::Evented::Arena(Int32).new(32) - expect_raises(IndexError) { arena.lazy_allocate(-1) {} } - expect_raises(IndexError) { arena.lazy_allocate(33) {} } + expect_raises(IndexError) { arena.lazy_allocate(-1) { } } + expect_raises(IndexError) { arena.lazy_allocate(33) { } } end end @@ -149,7 +149,7 @@ describe Crystal::Evented::Arena do called = 0 _, index1 = arena.lazy_allocate(1) { } - arena.free(index1) { called += 1} + arena.free(index1) { called += 1 } called.should eq(1) _, index2 = arena.lazy_allocate(1) { } diff --git a/spec/std/crystal/evented/timers_spec.cr b/spec/std/crystal/evented/timers_spec.cr index 5a75588e2ea0..a05572c27779 100644 --- a/spec/std/crystal/evented/timers_spec.cr +++ b/spec/std/crystal/evented/timers_spec.cr @@ -43,7 +43,7 @@ describe Crystal::Evented::Timers do timers.add(pointerof(event2)) events = [] of Crystal::Evented::Event* - timers.dequeue_ready { |event|events << event } + timers.dequeue_ready { |event| events << event } events.should eq([ pointerof(event1), diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 3e5756dc3913..27c7ed6bd994 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -329,16 +329,16 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop private def wait_readable(io, timeout = nil, &) : Nil yield if wait(:io_read, io, timeout) do |pd, event| - # don't wait if the waiter has already been marked ready (see Waiters#add) - return unless pd.value.@readers.add(event) - end + # don't wait if the waiter has already been marked ready (see Waiters#add) + return unless pd.value.@readers.add(event) + end end private def wait_writable(io, timeout = nil, &) : Nil yield if wait(:io_write, io, timeout) do |pd, event| - # don't wait if the waiter has already been marked ready (see Waiters#add) - return unless pd.value.@writers.add(event) - end + # don't wait if the waiter has already been marked ready (see Waiters#add) + return unless pd.value.@writers.add(event) + end end private def wait(type : Evented::Event::Type, io, timeout, &) diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index 46a215fcc5d0..f359fb83418c 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -3,8 +3,8 @@ require "../kqueue" class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop # the following are arbitrary numbers to identify specific events - INTERRUPT_IDENTIFIER = 9 - TIMER_IDENTIFIER = 10 + INTERRUPT_IDENTIFIER = 9 + TIMER_IDENTIFIER = 10 {% unless LibC.has_constant?(:EVFILT_USER) %} @pipe = uninitialized LibC::Int[2] From a01398d48502e6c2e01e05cd18db2b9f8e932e95 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 14 Oct 2024 09:19:45 +0200 Subject: [PATCH 42/56] Fix: simplify Evented::EventLoop#delete_timer We can merely return the dequeued variable, which is the intent of the method: to report whether the timer was dequeued (and thus own) or not. --- src/crystal/system/unix/evented/event_loop.cr | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 27c7ed6bd994..6f37c305e1f6 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -392,13 +392,10 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop protected def delete_timer(event : Evented::Event*) : Bool @lock.sync do dequeued, was_next_ready = @timers.delete(event) - if dequeued - # update system timer if we deleted the next timer - system_set_timer(@timers.next_ready?) if was_next_ready - return true if dequeued - end + # update system timer if we deleted the next timer + system_set_timer(@timers.next_ready?) if was_next_ready + dequeued end - false end # Helper to resume the fiber associated to an IO event and remove the event From 5e7f1aaa35305ed7dbd903b4d0182cbd29559e90 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 14 Oct 2024 09:25:42 +0200 Subject: [PATCH 43/56] Fix: simplify skip_file for loading io/evented --- src/io/evented.cr | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/io/evented.cr b/src/io/evented.cr index 358192a66955..59666691e8e7 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -1,9 +1,4 @@ -{% skip_file if flag?(:win32) %} - -{% unless flag?(:evloop_libevent) %} - {% skip_file if flag?(:android) || flag?(:linux) || flag?(:solaris) %} - {% skip_file if flag?(:bsd) || flag?(:darwin) %} -{% end %} +{% skip_file unless flag?(:evloop_libevent) || flag?(:wasi) %} require "crystal/thread_local_value" From 375253c64d94e11d4bebf6b47fad7a9938b1f2aa Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 15 Oct 2024 10:23:00 +0200 Subject: [PATCH 44/56] Fix: EventLoop#interrupt for kqueue evloop --- src/crystal/system/unix/kqueue.cr | 7 +++++ src/crystal/system/unix/kqueue/event_loop.cr | 30 +++++++++++--------- src/lib_c/aarch64-darwin/c/sys/event.cr | 2 +- src/lib_c/x86_64-darwin/c/sys/event.cr | 2 +- src/lib_c/x86_64-dragonfly/c/sys/event.cr | 2 +- src/lib_c/x86_64-freebsd/c/sys/event.cr | 2 +- src/lib_c/x86_64-netbsd/c/sys/event.cr | 2 +- 7 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/crystal/system/unix/kqueue.cr b/src/crystal/system/unix/kqueue.cr index 3c3f825f68bf..9f7cb1f414b9 100644 --- a/src/crystal/system/unix/kqueue.cr +++ b/src/crystal/system/unix/kqueue.cr @@ -24,6 +24,13 @@ struct Crystal::System::Kqueue yield if ret == -1 end + # Helper to register a single event. Returns immediately. + def kevent(ident, filter, flags, fflags = 0, data = 0, udata = nil) : Nil + kevent(ident, filter, flags, fflags, data, udata) do + raise RuntimeError.from_errno("kevent") + end + end + # Helper to register multiple *changes*. Returns immediately. def kevent(changes : Slice(LibC::Kevent), &) : Nil ret = LibC.kevent(@kq, changes.to_unsafe, changes.size, nil, 0, nil) diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index f359fb83418c..ef5d738b99c5 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -16,11 +16,15 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop # notification to interrupt a run @interrupted = Atomic::Flag.new - {% unless LibC.has_constant?(:EVFILT_USER) %} + + {% if LibC.has_constant?(:EVFILT_USER) %} + @kqueue.kevent( + INTERRUPT_IDENTIFIER, + LibC::EVFILT_USER, + LibC::EV_ADD | LibC::EV_ENABLE | LibC::EV_CLEAR) + {% else %} @pipe = System::FileDescriptor.system_pipe - @kqueue.kevent(@pipe[0], LibC::EVFILT_READ, LibC::EV_ADD) do - raise RuntimeError.from_errno("kevent") - end + @kqueue.kevent(@pipe[0], LibC::EVFILT_READ, LibC::EV_ADD) {% end %} end @@ -47,12 +51,16 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop @kqueue = System::Kqueue.new @interrupted.clear - {% unless LibC.has_constant?(:EVFILT_USER) %} + + {% if LibC.has_constant?(:EVFILT_USER) %} + @kqueue.kevent( + INTERRUPT_IDENTIFIER, + LibC::EVFILT_USER, + LibC::EV_ADD | LibC::EV_ENABLE | LibC::EV_CLEAR) + {% else %} @pipe.each { |fd| LibC.close(fd) } @pipe = System::FileDescriptor.system_pipe - @kqueue.kevent(@pipe[0], LibC::EVFILT_READ, LibC::EV_ADD) do - raise RuntimeError.from_errno("kevent") - end + @kqueue.kevent(@pipe[0], LibC::EVFILT_READ, LibC::EV_ADD) {% end %} system_set_timer(@timers.next_ready?) @@ -150,11 +158,7 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop return unless @interrupted.test_and_set {% if LibC.has_constant?(:EVFILT_USER) %} - @kqueue.kevent( - INTERRUPT_IDENTIFIER, - LibC::EVFILT_USER, - LibC::EV_ADD | LibC::EV_ONESHOT, - LibC::NOTE_FFCOPY | LibC::NOTE_TRIGGER | 1_u16) + @kqueue.kevent(INTERRUPT_IDENTIFIER, LibC::EVFILT_USER, 0, LibC::NOTE_TRIGGER) {% else %} ident = INTERRUPT_IDENTIFIER ret = LibC.write(@pipe[1], pointerof(ident), sizeof(Int32)) diff --git a/src/lib_c/aarch64-darwin/c/sys/event.cr b/src/lib_c/aarch64-darwin/c/sys/event.cr index 6b80c8d2ccb9..1fd68b6d1975 100644 --- a/src/lib_c/aarch64-darwin/c/sys/event.cr +++ b/src/lib_c/aarch64-darwin/c/sys/event.cr @@ -8,13 +8,13 @@ lib LibC EV_ADD = 0x0001_u16 EV_DELETE = 0x0002_u16 + EV_ENABLE = 0x0004_u16 EV_ONESHOT = 0x0010_u16 EV_CLEAR = 0x0020_u16 EV_EOF = 0x8000_u16 EV_ERROR = 0x4000_u16 NOTE_NSECONDS = 0x00000004_u32 - NOTE_FFCOPY = 0xc0000000_u32 NOTE_TRIGGER = 0x01000000_u32 struct Kevent diff --git a/src/lib_c/x86_64-darwin/c/sys/event.cr b/src/lib_c/x86_64-darwin/c/sys/event.cr index 6b80c8d2ccb9..1fd68b6d1975 100644 --- a/src/lib_c/x86_64-darwin/c/sys/event.cr +++ b/src/lib_c/x86_64-darwin/c/sys/event.cr @@ -8,13 +8,13 @@ lib LibC EV_ADD = 0x0001_u16 EV_DELETE = 0x0002_u16 + EV_ENABLE = 0x0004_u16 EV_ONESHOT = 0x0010_u16 EV_CLEAR = 0x0020_u16 EV_EOF = 0x8000_u16 EV_ERROR = 0x4000_u16 NOTE_NSECONDS = 0x00000004_u32 - NOTE_FFCOPY = 0xc0000000_u32 NOTE_TRIGGER = 0x01000000_u32 struct Kevent diff --git a/src/lib_c/x86_64-dragonfly/c/sys/event.cr b/src/lib_c/x86_64-dragonfly/c/sys/event.cr index 274a0da7791e..aff6274b8fd1 100644 --- a/src/lib_c/x86_64-dragonfly/c/sys/event.cr +++ b/src/lib_c/x86_64-dragonfly/c/sys/event.cr @@ -8,12 +8,12 @@ lib LibC EV_ADD = 0x0001_u16 EV_DELETE = 0x0002_u16 + EV_ENABLE = 0x0004_u16 EV_ONESHOT = 0x0010_u16 EV_CLEAR = 0x0020_u16 EV_EOF = 0x8000_u16 EV_ERROR = 0x4000_u16 - NOTE_FFCOPY = 0xc0000000_u32 NOTE_TRIGGER = 0x01000000_u32 struct Kevent diff --git a/src/lib_c/x86_64-freebsd/c/sys/event.cr b/src/lib_c/x86_64-freebsd/c/sys/event.cr index ac1a1580a704..0abe0686aba0 100644 --- a/src/lib_c/x86_64-freebsd/c/sys/event.cr +++ b/src/lib_c/x86_64-freebsd/c/sys/event.cr @@ -8,13 +8,13 @@ lib LibC EV_ADD = 0x0001_u16 EV_DELETE = 0x0002_u16 + EV_ENABLE = 0x0004_u16 EV_ONESHOT = 0x0010_u16 EV_CLEAR = 0x0020_u16 EV_EOF = 0x8000_u16 EV_ERROR = 0x4000_u16 NOTE_NSECONDS = 0x00000008_u32 - NOTE_FFCOPY = 0xc0000000_u32 NOTE_TRIGGER = 0x01000000_u32 struct Kevent diff --git a/src/lib_c/x86_64-netbsd/c/sys/event.cr b/src/lib_c/x86_64-netbsd/c/sys/event.cr index ce84b00267af..9ba5f0cf2548 100644 --- a/src/lib_c/x86_64-netbsd/c/sys/event.cr +++ b/src/lib_c/x86_64-netbsd/c/sys/event.cr @@ -8,13 +8,13 @@ lib LibC EV_ADD = 0x0001_u32 EV_DELETE = 0x0002_u32 + EV_ENABLE = 0x0004_u16 EV_ONESHOT = 0x0010_u32 EV_CLEAR = 0x0020_u32 EV_EOF = 0x8000_u32 EV_ERROR = 0x4000_u32 NOTE_NSECONDS = 0x00000003_u32 - NOTE_FFCOPY = 0xc0000000_u32 NOTE_TRIGGER = 0x01000000_u32 struct Kevent From 69850802c0fc47d9f060031ed2ef48a73c2b2e6a Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 17 Oct 2024 20:08:36 +0200 Subject: [PATCH 45/56] Fix: prohibit parallel access to arena objects Fixes race conditions in the evloop when another thread closes the fd, and thus frees the poll descriptor (waitlists): - After waiting for events and processing the events: processing events and timers would fail to get the index and raise an exception. - After trying to read or write but before we could actually get the poll descriptor: we'd fail to get the index and raise an exception. - After trying to read/write and after we allocated the poll descriptor, but before we could wait: we'd be adding to a freed poll descriptor. - Resetting the arena index on IO (__evloop_data) allowed a parallel wait to re-allocate the poll descriptor (despite the IO being closed). - ... Preventing parallel accesses to an arena object may be a bit drastic, but at least we know for sure that we don't have any parallelism issue. It also allows to drop the lock on poll descriptor as well as on each waiting list: we don't need them anymore. A disadvantage is that parallel read/write/close and evloop process event/timeout may conflict... though they already competed for the 3 aforementioned locks. --- spec/std/crystal/evented/arena_spec.cr | 168 ++++++++++++------ src/crystal/system/unix/epoll/event_loop.cr | 28 +-- src/crystal/system/unix/evented/arena.cr | 103 ++++++----- src/crystal/system/unix/evented/event_loop.cr | 37 ++-- .../system/unix/evented/poll_descriptor.cr | 47 +++-- src/crystal/system/unix/evented/waiters.cr | 65 +++---- src/crystal/system/unix/kqueue/event_loop.cr | 44 ++--- src/crystal/system/unix/socket.cr | 3 + 8 files changed, 281 insertions(+), 214 deletions(-) diff --git a/spec/std/crystal/evented/arena_spec.cr b/spec/std/crystal/evented/arena_spec.cr index 47b8f9ecdb3e..1b8f4bd50534 100644 --- a/spec/std/crystal/evented/arena_spec.cr +++ b/spec/std/crystal/evented/arena_spec.cr @@ -4,68 +4,84 @@ require "spec" require "../../../../src/crystal/system/unix/evented/arena" describe Crystal::Evented::Arena do - describe "#lazy_allocate" do - it "yields block once" do + describe "#allocate_at?" do + it "yields block when not allocated" do arena = Crystal::Evented::Arena(Int32).new(32) pointer = nil index = nil called = 0 - ptr1, idx1 = arena.lazy_allocate(0) do |ptr, idx| + ret = arena.allocate_at?(0) do |ptr, idx| pointer = ptr index = idx called += 1 end + ret.should eq(index) called.should eq(1) - ptr2, idx2 = arena.lazy_allocate(0) do |ptr, idx| - called += 1 - end + ret = arena.allocate_at?(0) { called += 1 } + ret.should be_nil called.should eq(1) pointer.should_not be_nil index.should_not be_nil - ptr1.should eq(pointer) - idx1.should eq(index) - - ptr2.should eq(pointer) - idx2.should eq(index) + arena.get(index.not_nil!) do |ptr| + ptr.should eq(pointer) + end end it "allocates up to capacity" do arena = Crystal::Evented::Arena(Int32).new(32) + indexes = [] of Crystal::Evented::Arena::Index - objects = 32.times.map do |i| - arena.lazy_allocate(i) { |pointer| pointer.value = i } - end - objects.each do |(pointer, index)| - arena.get(index).should eq(pointer) - pointer.value.should eq(index.index) + indexes = 32.times.map do |i| + arena.allocate_at?(i) { |ptr, _| ptr.value = i } + end.to_a + + indexes.size.should eq(32) + + indexes.each do |index| + arena.get(index.not_nil!) do |pointer| + pointer.should eq(pointer) + pointer.value.should eq(index.not_nil!.index) + end end end it "checks bounds" do arena = Crystal::Evented::Arena(Int32).new(32) - expect_raises(IndexError) { arena.lazy_allocate(-1) { } } - expect_raises(IndexError) { arena.lazy_allocate(33) { } } + expect_raises(IndexError) { arena.allocate_at?(-1) { } } + expect_raises(IndexError) { arena.allocate_at?(33) { } } end end describe "#get" do it "returns previously allocated object" do arena = Crystal::Evented::Arena(Int32).new(32) - pointer, index = arena.lazy_allocate(30) { |ptr| ptr.value = 654321 } + pointer = nil + + index = arena.allocate_at(30) do |ptr| + pointer = ptr + ptr.value = 654321 + end + called = 0 2.times do - ptr = arena.get(index) - ptr.should eq(pointer) - ptr.value.should eq(654321) + arena.get(index.not_nil!) do |ptr| + ptr.should eq(pointer) + ptr.value.should eq(654321) + called += 1 + end end + called.should eq(2) + end + + it "can't access unallocated object" do + arena = Crystal::Evented::Arena(Int32).new(32) - # not allocated: expect_raises(RuntimeError) do - arena.get(Crystal::Evented::Arena::Index.new(10, 0)) + arena.get(Crystal::Evented::Arena::Index.new(10, 0)) { } end end @@ -73,90 +89,128 @@ describe Crystal::Evented::Arena do arena = Crystal::Evented::Arena(Int32).new(32) called = 0 - _, index1 = arena.lazy_allocate(2) { called += 1 } + index1 = arena.allocate_at(2) { called += 1 } called.should eq(1) arena.free(index1) { } - expect_raises(RuntimeError) { arena.get(index1) } + expect_raises(RuntimeError) { arena.get(index1) { } } - _, index2 = arena.lazy_allocate(2) { called += 1 } + index2 = arena.allocate_at(2) { called += 1 } called.should eq(2) - expect_raises(RuntimeError) { arena.get(index1) } + expect_raises(RuntimeError) { arena.get(index1) { } } - # doesn't raise: - arena.get(index2) + arena.get(index2) { } end it "checks out of bounds" do arena = Crystal::Evented::Arena(Int32).new(32) - expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(-1, 0)) } - expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(33, 0)) } + expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(-1, 0)) { } } + expect_raises(IndexError) { arena.get(Crystal::Evented::Arena::Index.new(33, 0)) { } } end end describe "#get?" do it "returns previously allocated object" do arena = Crystal::Evented::Arena(Int32).new(32) - pointer, index = arena.lazy_allocate(30) { |ptr| ptr.value = 654321 } + pointer = nil + index = arena.allocate_at(30) do |ptr| + pointer = ptr + ptr.value = 654321 + end + + called = 0 2.times do - ptr = arena.get?(index) - ptr.should eq(pointer) - ptr.not_nil!.value.should eq(654321) + ret = arena.get?(index) do |ptr| + ptr.should eq(pointer) + ptr.not_nil!.value.should eq(654321) + called += 1 + end + ret.should be_true end + called.should eq(2) + end - arena.get?(Crystal::Evented::Arena::Index.new(10, 0)).should be_nil + it "can't access unallocated index" do + arena = Crystal::Evented::Arena(Int32).new(32) + + called = 0 + ret = arena.get?(Crystal::Evented::Arena::Index.new(10, 0)) { called += 1 } + ret.should be_false + called.should eq(0) end it "checks generation" do arena = Crystal::Evented::Arena(Int32).new(32) called = 0 - pointer1, index1 = arena.lazy_allocate(2) { called += 1 } - called.should eq(1) + old_index = arena.allocate_at(2) { } + arena.free(old_index) { } - arena.free(index1) { } - arena.get?(index1).should be_nil + # not accessible after free: + ret = arena.get?(old_index) { called += 1 } + ret.should be_false + called.should eq(0) - pointer2, index2 = arena.lazy_allocate(2) { called += 1 } - called.should eq(2) - arena.get?(index1).should be_nil - arena.get?(index2).should eq(pointer2) + # can be reallocated: + new_index = arena.allocate_at(2) { } + + # still not accessible after reallocate: + ret = arena.get?(old_index) { called += 1 } + ret.should be_false + called.should eq(0) + + # accessible after reallocate (new index): + ret = arena.get?(new_index) { called += 1 } + ret.should be_true + called.should eq(1) end it "checks out of bounds" do arena = Crystal::Evented::Arena(Int32).new(32) - arena.get?(Crystal::Evented::Arena::Index.new(-1, 0)).should be_nil - arena.get?(Crystal::Evented::Arena::Index.new(33, 0)).should be_nil + called = 0 + + arena.get?(Crystal::Evented::Arena::Index.new(-1, 0)) { called += 1 }.should be_false + arena.get?(Crystal::Evented::Arena::Index.new(33, 0)) { called += 1 }.should be_false + + called.should eq(0) end end describe "#free" do it "deallocates the object" do arena = Crystal::Evented::Arena(Int32).new(32) - pointer, index1 = arena.lazy_allocate(3) { } - pointer.value = 123 + index1 = arena.allocate_at(3) { |ptr| ptr.value = 123 } arena.free(index1) { } - pointer, index2 = arena.lazy_allocate(3) { } + index2 = arena.allocate_at(3) { } index2.should_not eq(index1) - pointer.value.should eq(0) + + value = nil + arena.get(index2) { |ptr| value = ptr.value } + value.should eq(0) end it "checks generation" do arena = Crystal::Evented::Arena(Int32).new(32) + called = 0 + old_index = arena.allocate_at(1) { } - _, index1 = arena.lazy_allocate(1) { } - arena.free(index1) { called += 1 } + # can free: + arena.free(old_index) { called += 1 } called.should eq(1) - _, index2 = arena.lazy_allocate(1) { } - arena.free(index1) { called += 1 } + # can reallocate: + new_index = arena.allocate_at(1) { } + + # can't free with invalid index: + arena.free(old_index) { called += 1 } called.should eq(1) - arena.free(index2) { called += 1 } + # but new index can: + arena.free(new_index) { called += 1 } called.should eq(2) end diff --git a/src/crystal/system/unix/epoll/event_loop.cr b/src/crystal/system/unix/epoll/event_loop.cr index b209d2706aff..dc2f2052dfa2 100644 --- a/src/crystal/system/unix/epoll/event_loop.cr +++ b/src/crystal/system/unix/epoll/event_loop.cr @@ -92,22 +92,22 @@ class Crystal::Epoll::EventLoop < Crystal::Evented::EventLoop Crystal.trace :evloop, "event", fd: index.index, index: index.to_i64, events: events - pd = Evented.arena.get(index) - - if (events & (LibC::EPOLLERR | LibC::EPOLLHUP)) != 0 - pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } - pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } - return - end + Evented.arena.get?(index) do |pd| + if (events & (LibC::EPOLLERR | LibC::EPOLLHUP)) != 0 + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } + pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } + return + end - if (events & LibC::EPOLLRDHUP) == LibC::EPOLLRDHUP - pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } - elsif (events & LibC::EPOLLIN) == LibC::EPOLLIN - pd.value.@readers.ready_one { |event| unsafe_resume_io(event) } - end + if (events & LibC::EPOLLRDHUP) == LibC::EPOLLRDHUP + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } + elsif (events & LibC::EPOLLIN) == LibC::EPOLLIN + pd.value.@readers.ready_one { |event| unsafe_resume_io(event) } + end - if (events & LibC::EPOLLOUT) == LibC::EPOLLOUT - pd.value.@writers.ready_one { |event| unsafe_resume_io(event) } + if (events & LibC::EPOLLOUT) == LibC::EPOLLOUT + pd.value.@writers.ready_one { |event| unsafe_resume_io(event) } + end end end diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/system/unix/evented/arena.cr index ac254e23de9e..5dcb5c5c2c8c 100644 --- a/src/crystal/system/unix/evented/arena.cr +++ b/src/crystal/system/unix/evented/arena.cr @@ -23,7 +23,10 @@ # # Thread safety: the memory region is pre-allocated (up to capacity) using mmap # (virtual allocation) and pointers are never invalidated. Individual -# (de)allocations of objects are protected with a fine grained lock. +# allocation, deallocation and regular accesses are protected by a fine grained +# lock over each object: parallel accesses to the memory region are prohibited, +# and pointers are expected to not outlive the block that yielded them (don't +# capture them). # # Guarantees: `mmap` initializes the memory to zero, which means `T` objects are # initialized to zero by default, then `#free` will also clear the memory, so @@ -127,65 +130,70 @@ class Crystal::Evented::Arena(T) LibC.munmap(@buffer.to_unsafe, @buffer.bytesize) end - # Yields and allocates the object at *index* unless already allocated. - # Returns a pointer to the object at *index* and the generation index. + # Allocates the object at *index* unless already allocated, then yields a + # pointer to the object at *index* and the current generation index to later + # retrieve and free the allocated object. Eventually returns the generation + # index. # - # Permits two threads to allocate the same object in parallel yet only allow - # one to initialize it; the other one will silently receive the pointer and - # the generation index. + # Does nothing if the object has already been allocated and returns `nil`. # # There are no generational checks. # Raises if *index* is out of bounds. - def lazy_allocate(index : Int32, &) : {Pointer(T), Index} + def allocate_at?(index : Int32, & : (Pointer(T), Index) ->) : Index? entry = at(index) entry.value.@lock.sync do - pointer = entry.value.pointer - gen_index = Index.new(index, entry.value.generation) + return if entry.value.allocated? - unless entry.value.allocated? - {% unless flag?(:preview_mt) %} - @maximum = index if index > @maximum - {% end %} + {% unless flag?(:preview_mt) %} + @maximum = index if index > @maximum + {% end %} + entry.value.allocated = true - entry.value.allocated = true - yield pointer, gen_index - end + gen_index = Index.new(index, entry.value.generation) + yield entry.value.pointer, gen_index - {pointer, gen_index} + gen_index end end - # Returns a pointer to the object previously allocated at *index*. + # Same as `#allocate_at?` but raises when already allocated. + def allocate_at(index : Int32, & : (Pointer(T), Index) ->) : Index? + allocate_at?(index) { |ptr, idx| yield ptr, idx } || + raise RuntimeError.new("#{self.class.name}: already allocated index=#{index}") + end + + # Yields a pointer to the object previously allocated at *index*. # # Raises if the object isn't allocated. # Raises if the generation has changed (i.e. the object has been freed then reallocated). # Raises if *index* is negative. - def get(index : Index) : Pointer(T) - entry = at(index) - entry.value.pointer + def get(index : Index, &) : Nil + at(index) do |entry| + yield entry.value.pointer + end end - # Returns a pointer to the object previously allocated at *index*. - # Returns `nil` if the object isn't allocated or the generation has changed. + # Yields a pointer to the object previously allocated at *index* and returns + # true. + # Does nothing if the object isn't allocated or the generation has changed, + # and returns false. # # Raises if *index* is negative. - def get?(index : Index) : Pointer(T)? - if entry = at?(index) - entry.value.pointer + def get?(index : Index) : Bool + at?(index) do |entry| + yield entry.value.pointer + return true end + false end - # Yields the object previously allocated at *index* then releases it. Does - # nothing if the object isn't allocated or the generation has changed. + # Yields the object previously allocated at *index* then releases it. + # Does nothing if the object isn't allocated or the generation has changed. # # Raises if *index* is negative. def free(index : Index, &) : Nil - return unless entry = at?(index.index) - - entry.value.@lock.sync do - return unless entry.value.allocated? - return unless entry.value.generation == index.generation + at?(index) do |entry| begin yield entry.value.pointer ensure @@ -194,22 +202,31 @@ class Crystal::Evented::Arena(T) end end - private def at(index : Index) : Pointer(Entry(T)) + private def at(index : Index, &) : Nil entry = at(index.index) - unless entry.value.allocated? - raise RuntimeError.new("#{self.class.name}: object not allocated at index #{index.index}") + entry.value.@lock.lock + + unless entry.value.allocated? && entry.value.generation == index.generation + entry.value.@lock.unlock + raise RuntimeError.new("#{self.class.name}: invalid reference index=#{index.index}:#{index.generation} current=#{index.index}:#{entry.value.generation}") end - unless entry.value.generation == index.generation - raise RuntimeError.new("#{self.class.name}: object generation changed at index #{index.index} (#{index.generation} => #{entry.value.generation})") + + begin + yield entry + ensure + entry.value.@lock.unlock end - entry end - private def at?(index : Index) : Pointer(Entry(T))? + private def at?(index : Index, &) : Nil return unless entry = at?(index.index) - return unless entry.value.allocated? - return unless entry.value.generation == index.generation - entry + + entry.value.@lock.sync do + return unless entry.value.allocated? + return unless entry.value.generation == index.generation + + yield entry + end end private def at(index : Int32) : Pointer(Entry(T)) diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 6f37c305e1f6..fc7e78d23da5 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -291,7 +291,6 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop protected def evented_close(io) return unless (index = io.__evloop_data).valid? - io.__evloop_data = Arena::INVALID_INDEX Evented.arena.free(index) do |pd| pd.value.@readers.ready_all do |event| @@ -308,7 +307,6 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop private def internal_remove(io) return unless (index = io.__evloop_data).valid? - io.__evloop_data = Arena::INVALID_INDEX Evented.arena.free(index) do |pd| pd.value.remove(io.fd) { } # ignore system error @@ -342,25 +340,36 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop end private def wait(type : Evented::Event::Type, io, timeout, &) - # get or allocate the poll descriptor + # prepare event (on the stack); we can't initialize it properly until we get + # the arena index below; we also can't use a nilable since `pointerof` would + # point to the union, not the event + event = uninitialized Evented::Event + + # add the event to the waiting list; in case we can't access or allocate the + # poll descriptor into the arena, we merely return to let the caller handle + # the situation (maybe the IO got closed?) if (index = io.__evloop_data).valid? - pd = Evented.arena.get(index) + event = Evented::Event.new(type, Fiber.current, index, timeout) + + return false unless Evented.arena.get?(index) do |pd| + yield pd, pointerof(event) + end else - pd, index = Evented.arena.lazy_allocate(io.fd) do |pd, index| + # OPTIMIZE: failing to allocate may be a simple conflict with 2 fibers + # starting to read or write on the same fd, we may want to detect any + # error situation instead of returning and retrying a syscall + return false unless Evented.arena.allocate_at?(io.fd) do |pd, index| # register the fd with the event loop (once), it should usually merely add # the fd to the current evloop but may "transfer" the ownership from # another event loop: io.__evloop_data = index pd.value.take_ownership(self, io.fd, index) + + event = Evented::Event.new(type, Fiber.current, index, timeout) + yield pd, pointerof(event) end end - # create an event (on the stack) - event = Evented::Event.new(type, Fiber.current, index, timeout) - - # add the event to the waiting list - yield pd, pointerof(event) - if event.wake_at? add_timer(pointerof(event)) @@ -457,14 +466,12 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop when .io_read? # reached read timeout: cancel io event; by rule the timer always wins, # even in case of conflict with #unsafe_resume_io we must resume the fiber - pd = Evented.arena.get(event.value.index) - pd.value.@readers.delete(event) + Evented.arena.get?(event.value.index, &.value.@readers.delete(event)) event.value.timed_out! when .io_write? # reached write timeout: cancel io event; by rule the timer always wins, # even in case of conflict with #unsafe_resume_io we must resume the fiber - pd = Evented.arena.get(event.value.index) - pd.value.@writers.delete(event) + Evented.arena.get?(event.value.index, &.value.@writers.delete(event)) event.value.timed_out! when .select_timeout? # always dequeue the event but only enqueue the fiber if we win the diff --git a/src/crystal/system/unix/evented/poll_descriptor.cr b/src/crystal/system/unix/evented/poll_descriptor.cr index e9779a71d6ab..1ef318e454bb 100644 --- a/src/crystal/system/unix/evented/poll_descriptor.cr +++ b/src/crystal/system/unix/evented/poll_descriptor.cr @@ -3,50 +3,43 @@ require "./event_loop" # Information related to the evloop for a fd, such as the read and write queues # (waiting `Event`), as well as which evloop instance currently owns the fd. # -# Thread-safe: mutations are protected with a lock. +# Thread-unsafe: parallel mutations must be protected with a lock. struct Crystal::Evented::PollDescriptor @event_loop : Evented::EventLoop? - @lock = SpinLock.new @readers = Waiters.new @writers = Waiters.new # Makes *event_loop* the new owner of *fd*. # Removes *fd* from the current event loop (if any). def take_ownership(event_loop : EventLoop, fd : Int32, index : Arena::Index) : Nil - @lock.sync do - current = @event_loop - - if event_loop == current - raise "BUG: evloop already owns the poll-descriptor for fd=#{fd}" - end - - # ensure we can't have cross enqueues after we transfer the fd, so we - # can optimize (all enqueues are local) and we don't end up with a timer - # from evloop A to cancel an event from evloop B (currently unsafe) - if current && !empty? - raise RuntimeError.new("BUG: transfering fd=#{fd} to another evloop with pending reader/writer fibers") - end - - @event_loop = event_loop - event_loop.system_add(fd, index) - current.try(&.system_del(fd, closing: false)) + current = @event_loop + + if event_loop == current + raise "BUG: evloop already owns the poll-descriptor for fd=#{fd}" + end + + # ensure we can't have cross enqueues after we transfer the fd, so we + # can optimize (all enqueues are local) and we don't end up with a timer + # from evloop A to cancel an event from evloop B (currently unsafe) + if current && !empty? + raise RuntimeError.new("BUG: transfering fd=#{fd} to another evloop with pending reader/writer fibers") end + + @event_loop = event_loop + event_loop.system_add(fd, index) + current.try(&.system_del(fd, closing: false)) end # Removes *fd* from its owner event loop. Raises on errors. def remove(fd : Int32) : Nil - @lock.sync do - current, @event_loop = @event_loop, nil - current.try(&.system_del(fd)) - end + current, @event_loop = @event_loop, nil + current.try(&.system_del(fd)) end # Same as `#remove` but yields on errors. def remove(fd : Int32, &) : Nil - @lock.sync do - current, @event_loop = @event_loop, nil - current.try(&.system_del(fd) { yield }) - end + current, @event_loop = @event_loop, nil + current.try(&.system_del(fd) { yield }) end # Returns true when there is at least one reader or writer. Returns false diff --git a/src/crystal/system/unix/evented/waiters.cr b/src/crystal/system/unix/evented/waiters.cr index bc79c5dc32c1..2d052718bae9 100644 --- a/src/crystal/system/unix/evented/waiters.cr +++ b/src/crystal/system/unix/evented/waiters.cr @@ -3,57 +3,52 @@ require "./event" # A FIFO queue of `Event` waiting on the same operation (either read or write) # for a fd. See `PollDescriptor`. # -# Thread safe: mutations are protected with a lock, and race conditions are -# handled through the ready atomic. +# Race conditions on the state of the waiting list are handled through the ready +# always ready variables. +# +# Thread unsafe: parallel mutations must be protected with a lock. struct Crystal::Evented::Waiters @list = PointerLinkedList(Event).new - @lock = SpinLock.new @ready = false @always_ready = false # Adds an event to the waiting list. May return false immediately if another # thread marked the list as ready in parallel, returns true otherwise. def add(event : Pointer(Event)) : Bool - @lock.sync do - if @always_ready - # another thread closed the fd or we received a fd error or hup event: - # the fd will never block again - return false - end - - if @ready - # another thread readied the fd before the current thread got to add - # the event: don't block and resets @ready for the next loop - @ready = false - return false - end + if @always_ready + # another thread closed the fd or we received a fd error or hup event: + # the fd will never block again + return false + end - @list.push(event) + if @ready + # another thread readied the fd before the current thread got to add + # the event: don't block and resets @ready for the next loop + @ready = false + return false end + + @list.push(event) true end def delete(event : Pointer(Event)) : Nil - @lock.sync do - @list.delete(event) if event.value.next - end + @list.delete(event) if event.value.next end # Removes one pending event or marks the list as ready when there are no # pending events (we got notified of readiness before a thread enqueued). def ready_one(& : Pointer(Event) -> Bool) : Nil - @lock.sync do - # loop until the block succesfully processes an event (it may have to - # dequeue the timeout from timers) - loop do - if event = @list.shift? - break if yield event - else - # no event queued but another thread may be waiting for the lock to - # add an event: set as ready to resolve the race condition - @ready = true - return - end + # loop until the block succesfully processes an event (it may have to + # dequeue the timeout from timers) + loop do + if event = @list.shift? + break if yield event + else + # no event queued but another thread may be waiting for the lock to + # add an event: set as ready to resolve the race condition + @ready = true + return end end end @@ -61,9 +56,7 @@ struct Crystal::Evented::Waiters # Dequeues all pending events and marks the list as always ready. This must be # called when a fd is closed or an error or hup event occurred. def ready_all(& : Pointer(Event) ->) : Nil - @lock.sync do - @list.consume_each { |event| yield event } - @always_ready = true - end + @list.consume_each { |event| yield event } + @always_ready = true end end diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index ef5d738b99c5..bf864b3f1542 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -126,30 +126,30 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop Crystal.trace :evloop, "event", fd: kevent.value.ident, index: index.to_i64, filter: kevent.value.filter, flags: kevent.value.flags, fflags: kevent.value.fflags - pd = Evented.arena.get(index) - - if (kevent.value.fflags & LibC::EV_EOF) == LibC::EV_EOF - # apparently some systems may report EOF on write with EVFILT_READ instead - # of EVFILT_WRITE, so let's wake all waiters: - pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } - pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } - return - end - - case kevent.value.filter - when LibC::EVFILT_READ - if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR - # OPTIMIZE: pass errno (kevent.data) through PollDescriptor + Evented.arena.get?(index) do |pd| + if (kevent.value.fflags & LibC::EV_EOF) == LibC::EV_EOF + # apparently some systems may report EOF on write with EVFILT_READ instead + # of EVFILT_WRITE, so let's wake all waiters: pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } - else - pd.value.@readers.ready_one { |event| unsafe_resume_io(event) } - end - when LibC::EVFILT_WRITE - if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR - # OPTIMIZE: pass errno (kevent.data) through PollDescriptor pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } - else - pd.value.@writers.ready_one { |event| unsafe_resume_io(event) } + return + end + + case kevent.value.filter + when LibC::EVFILT_READ + if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR + # OPTIMIZE: pass errno (kevent.data) through PollDescriptor + pd.value.@readers.ready_all { |event| unsafe_resume_io(event) } + else + pd.value.@readers.ready_one { |event| unsafe_resume_io(event) } + end + when LibC::EVFILT_WRITE + if (kevent.value.fflags & LibC::EV_ERROR) == LibC::EV_ERROR + # OPTIMIZE: pass errno (kevent.data) through PollDescriptor + pd.value.@writers.ready_all { |event| unsafe_resume_io(event) } + else + pd.value.@writers.ready_one { |event| unsafe_resume_io(event) } + end end end end diff --git a/src/crystal/system/unix/socket.cr b/src/crystal/system/unix/socket.cr index 4aa0fc2f0af9..535f37f386c0 100644 --- a/src/crystal/system/unix/socket.cr +++ b/src/crystal/system/unix/socket.cr @@ -25,6 +25,9 @@ module Crystal::System::Socket end private def initialize_handle(fd) + {% if Crystal.has_constant?(:Evented) %} + @__evloop_data = Crystal::Evented::Arena::INVALID_INDEX + {% end %} end # Tries to bind the socket to a local address. From f2a1efe580849f7e8c9858c18f0e2d9fd238814d Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 22 Oct 2024 11:01:15 +0200 Subject: [PATCH 46/56] NetBSD: fix libc binding for kevent --- src/lib_c/x86_64-netbsd/c/sys/event.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib_c/x86_64-netbsd/c/sys/event.cr b/src/lib_c/x86_64-netbsd/c/sys/event.cr index 9ba5f0cf2548..91da3cea1a04 100644 --- a/src/lib_c/x86_64-netbsd/c/sys/event.cr +++ b/src/lib_c/x86_64-netbsd/c/sys/event.cr @@ -28,5 +28,5 @@ lib LibC end fun kqueue1(flags : Int) : Int - fun kevent = __kevent100(kq : Int, changelist : Kevent*, nchanges : SizeT, eventlist : Kevent*, nevents : SizeT, timeout : Timespec*) : Int + fun kevent = __kevent50(kq : Int, changelist : Kevent*, nchanges : SizeT, eventlist : Kevent*, nevents : SizeT, timeout : Timespec*) : Int end From b14fa3c9ff983700d6c445df21f1e7a5f901113b Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 22 Oct 2024 11:05:55 +0200 Subject: [PATCH 47/56] Fix: prefer System::Time.monotonic over Time.monotonic Avoids an issue with the timecop shard that overrides Time.monotonic. --- src/crystal/system/unix/evented/event.cr | 6 +++++- src/crystal/system/unix/evented/fiber_event.cr | 4 +++- src/crystal/system/unix/evented/timers.cr | 6 ++++-- src/crystal/system/unix/kqueue/event_loop.cr | 6 +++++- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/crystal/system/unix/evented/event.cr b/src/crystal/system/unix/evented/event.cr index ad8fbd5a8276..146427777be4 100644 --- a/src/crystal/system/unix/evented/event.cr +++ b/src/crystal/system/unix/evented/event.cr @@ -34,7 +34,11 @@ struct Crystal::Evented::Event include PointerLinkedList::Node def initialize(@type : Type, @fiber, @index = nil, timeout : Time::Span? = nil) - @wake_at = Time.monotonic + timeout if timeout + if timeout + seconds, nanoseconds = System::Time.monotonic + now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) + @wake_at = now + timeout + end end # Mark the IO event as timed out. diff --git a/src/crystal/system/unix/evented/fiber_event.cr b/src/crystal/system/unix/evented/fiber_event.cr index 640606a04e93..074dd67e926f 100644 --- a/src/crystal/system/unix/evented/fiber_event.cr +++ b/src/crystal/system/unix/evented/fiber_event.cr @@ -7,7 +7,9 @@ class Crystal::Evented::FiberEvent # sleep or select timeout def add(timeout : Time::Span) : Nil - @event.wake_at = Time.monotonic + timeout + seconds, nanoseconds = System::Time.monotonic + now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) + @event.wake_at = now + timeout @event_loop.add_timer(pointerof(@event)) end diff --git a/src/crystal/system/unix/evented/timers.cr b/src/crystal/system/unix/evented/timers.cr index f90280b4f734..ace4fefcf09b 100644 --- a/src/crystal/system/unix/evented/timers.cr +++ b/src/crystal/system/unix/evented/timers.cr @@ -29,11 +29,13 @@ struct Crystal::Evented::Timers end # Dequeues and yields each ready timer (their `#wake_at` is lower than - # `Time.monotonic`) from the oldest to the most recent (i.e. time ascending). + # `System::Time.monotonic`) from the oldest to the most recent (i.e. time + # ascending). def dequeue_ready(& : Evented::Event* -> Nil) : Nil return if @list.empty? - now = Time.monotonic + seconds, nanoseconds = System::Time.monotonic + now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) n = 0 @list.each do |event| diff --git a/src/crystal/system/unix/kqueue/event_loop.cr b/src/crystal/system/unix/kqueue/event_loop.cr index bf864b3f1542..6eb98a7dc948 100644 --- a/src/crystal/system/unix/kqueue/event_loop.cr +++ b/src/crystal/system/unix/kqueue/event_loop.cr @@ -214,7 +214,11 @@ class Crystal::Kqueue::EventLoop < Crystal::Evented::EventLoop private def system_set_timer(time : Time::Span?) : Nil if time flags = LibC::EV_ADD | LibC::EV_ONESHOT | LibC::EV_CLEAR - t = time - Time.monotonic + + seconds, nanoseconds = System::Time.monotonic + now = Time::Span.new(seconds: seconds, nanoseconds: nanoseconds) + t = time - now + data = {% if LibC.has_constant?(:NOTE_NSECONDS) %} t.total_nanoseconds.to_i64!.clamp(0..) From b4c192e37964d7e4040ddf90b90086c7c82f86fb Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 4 Nov 2024 14:50:21 +0100 Subject: [PATCH 48/56] Add :evloop_epoll and :evloop_kqueue flags + opt-in on some targets --- src/crystal/system/event_loop.cr | 20 ++++++++++++-------- src/io/evented.cr | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/crystal/system/event_loop.cr b/src/crystal/system/event_loop.cr index 3b5e1b4543b0..356ebe5c3c8a 100644 --- a/src/crystal/system/event_loop.cr +++ b/src/crystal/system/event_loop.cr @@ -4,14 +4,18 @@ abstract class Crystal::EventLoop {% if flag?(:wasi) %} Crystal::Wasi::EventLoop.new {% elsif flag?(:unix) %} - {% if flag?(:evloop_libevent) %} + # TODO: dragonfly: the kqueue evloop hasn't been tested + # TODO: netbsd: the kqueue evloop doesn't work (needs investigation) + # TODO: openbsd: the kqueue evloop is magnitudes slower (needs investigation) + # TODO: solaris: the epoll evloop hasn't been tested (better: use event-ports) + {% if flag?(:evloop_libevent) || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} Crystal::LibEvent::EventLoop.new - {% elsif flag?(:android) || flag?(:linux) || flag?(:solaris) %} + {% elsif flag?(:evloop_epoll) || flag?(:android) || flag?(:linux) %} Crystal::Epoll::EventLoop.new - {% elsif flag?(:bsd) || flag?(:darwin) %} + {% elsif flag?(:evloop_kqueue) || flag?(:darwin) || flag?(:freebsd) %} Crystal::Kqueue::EventLoop.new {% else %} - Crystal::LibEvent::EventLoop.new + {% raise "Event loop not supported" %} {% end %} {% elsif flag?(:win32) %} Crystal::IOCP::EventLoop.new @@ -86,14 +90,14 @@ end {% if flag?(:wasi) %} require "./wasi/event_loop" {% elsif flag?(:unix) %} - {% if flag?(:evloop_libevent) %} + {% if flag?(:evloop_libevent) || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} require "./unix/event_loop_libevent" - {% elsif flag?(:android) || flag?(:linux) || flag?(:solaris) %} + {% elsif flag?(:evloop_epoll) || flag?(:android) || flag?(:linux) %} require "./unix/epoll/event_loop" - {% elsif flag?(:bsd) || flag?(:darwin) %} + {% elsif flag?(:evloop_kqueue) || flag?(:darwin) || flag?(:freebsd) %} require "./unix/kqueue/event_loop" {% else %} - require "./unix/event_loop_libevent" + {% raise "Event loop not supported" %} {% end %} {% elsif flag?(:win32) %} require "./win32/event_loop_iocp" diff --git a/src/io/evented.cr b/src/io/evented.cr index 59666691e8e7..f0f518b27bff 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:evloop_libevent) || flag?(:wasi) %} +{% skip_file unless flag?(:evloop_libevent) || flag?(:wasi) || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} require "crystal/thread_local_value" From 36ef33b2f2ad0107d5f660410189434542624f67 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 4 Nov 2024 15:02:50 +0100 Subject: [PATCH 49/56] Fix: compilation with -Devloop_libevent We must cast the actual LibEvent types because the type signatures return the abstract Crystal::EventLoop interface that don't implement the necessary methods. --- src/io/evented.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/io/evented.cr b/src/io/evented.cr index f0f518b27bff..0eb493b79208 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -52,8 +52,8 @@ module IO::Evented end private def add_read_event(timeout = @read_timeout) : Nil - event = @read_event.get { Crystal::EventLoop.current.create_fd_read_event(self) } - event.add timeout + event = @read_event.get { Crystal::EventLoop.current.as(Crystal::LibEvent::EventLoop).create_fd_read_event(self) } + event.as(Crystal::LibEvent::Event).add timeout end # :nodoc: @@ -77,8 +77,8 @@ module IO::Evented end private def add_write_event(timeout = @write_timeout) : Nil - event = @write_event.get { Crystal::EventLoop.current.create_fd_write_event(self) } - event.add timeout + event = @write_event.get { Crystal::EventLoop.current.as(Crystal::LibEvent::EventLoop).create_fd_write_event(self) } + event.as(Crystal::LibEvent::Event).add timeout end def evented_close : Nil From 3777d73eb9bc5d3d44d5e22c4161b13454a887f2 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 4 Nov 2024 15:20:48 +0100 Subject: [PATCH 50/56] fixup! Fix: compilation with -Devloop_libevent --- spec/std/crystal/evented/arena_spec.cr | 2 +- spec/std/crystal/evented/poll_descriptor_spec.cr | 2 +- spec/std/crystal/evented/timers_spec.cr | 2 +- spec/std/crystal/evented/waiters_spec.cr | 2 +- src/io/evented.cr | 8 ++++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/spec/std/crystal/evented/arena_spec.cr b/spec/std/crystal/evented/arena_spec.cr index 1b8f4bd50534..76667041b1ea 100644 --- a/spec/std/crystal/evented/arena_spec.cr +++ b/spec/std/crystal/evented/arena_spec.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:unix) %} +{% skip_file if !flag?(:unix) || flag?(:evloop_libevent) %} require "spec" require "../../../../src/crystal/system/unix/evented/arena" diff --git a/spec/std/crystal/evented/poll_descriptor_spec.cr b/spec/std/crystal/evented/poll_descriptor_spec.cr index a2c1c98bf96f..0f582568ab0c 100644 --- a/spec/std/crystal/evented/poll_descriptor_spec.cr +++ b/spec/std/crystal/evented/poll_descriptor_spec.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:unix) %} +{% skip_file if !flag?(:unix) || flag?(:evloop_libevent) %} require "spec" require "../../../../src/crystal/system/unix/evented/poll_descriptor" diff --git a/spec/std/crystal/evented/timers_spec.cr b/spec/std/crystal/evented/timers_spec.cr index a05572c27779..2bcd491b2903 100644 --- a/spec/std/crystal/evented/timers_spec.cr +++ b/spec/std/crystal/evented/timers_spec.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:unix) %} +{% skip_file if !flag?(:unix) || flag?(:evloop_libevent) %} require "spec" require "../../../../src/crystal/system/unix/evented/timers" diff --git a/spec/std/crystal/evented/waiters_spec.cr b/spec/std/crystal/evented/waiters_spec.cr index c2f154dcf631..f61722b17b5c 100644 --- a/spec/std/crystal/evented/waiters_spec.cr +++ b/spec/std/crystal/evented/waiters_spec.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:unix) %} +{% skip_file if !flag?(:unix) || flag?(:evloop_libevent) %} require "spec" require "../../../../src/crystal/system/unix/evented/waiters" diff --git a/src/io/evented.cr b/src/io/evented.cr index 0eb493b79208..f0f518b27bff 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -52,8 +52,8 @@ module IO::Evented end private def add_read_event(timeout = @read_timeout) : Nil - event = @read_event.get { Crystal::EventLoop.current.as(Crystal::LibEvent::EventLoop).create_fd_read_event(self) } - event.as(Crystal::LibEvent::Event).add timeout + event = @read_event.get { Crystal::EventLoop.current.create_fd_read_event(self) } + event.add timeout end # :nodoc: @@ -77,8 +77,8 @@ module IO::Evented end private def add_write_event(timeout = @write_timeout) : Nil - event = @write_event.get { Crystal::EventLoop.current.as(Crystal::LibEvent::EventLoop).create_fd_write_event(self) } - event.as(Crystal::LibEvent::Event).add timeout + event = @write_event.get { Crystal::EventLoop.current.create_fd_write_event(self) } + event.add timeout end def evented_close : Nil From cf6f508abfd4da18a97056dca75c5dea8e372dc1 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 4 Nov 2024 18:20:48 +0100 Subject: [PATCH 51/56] Use -Devloop=[libevent|epoll|kqueue] flag(s) --- spec/std/crystal/evented/arena_spec.cr | 2 +- spec/std/crystal/evented/poll_descriptor_spec.cr | 2 +- spec/std/crystal/evented/timers_spec.cr | 2 +- spec/std/crystal/evented/waiters_spec.cr | 2 +- src/crystal/system/event_loop.cr | 12 ++++++------ src/io/evented.cr | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/spec/std/crystal/evented/arena_spec.cr b/spec/std/crystal/evented/arena_spec.cr index 76667041b1ea..f07306070818 100644 --- a/spec/std/crystal/evented/arena_spec.cr +++ b/spec/std/crystal/evented/arena_spec.cr @@ -1,4 +1,4 @@ -{% skip_file if !flag?(:unix) || flag?(:evloop_libevent) %} +{% skip_file if !flag?(:unix) || flag?("evloop=libevent") %} require "spec" require "../../../../src/crystal/system/unix/evented/arena" diff --git a/spec/std/crystal/evented/poll_descriptor_spec.cr b/spec/std/crystal/evented/poll_descriptor_spec.cr index 0f582568ab0c..b15177478523 100644 --- a/spec/std/crystal/evented/poll_descriptor_spec.cr +++ b/spec/std/crystal/evented/poll_descriptor_spec.cr @@ -1,4 +1,4 @@ -{% skip_file if !flag?(:unix) || flag?(:evloop_libevent) %} +{% skip_file if !flag?(:unix) || flag?("evloop=libevent") %} require "spec" require "../../../../src/crystal/system/unix/evented/poll_descriptor" diff --git a/spec/std/crystal/evented/timers_spec.cr b/spec/std/crystal/evented/timers_spec.cr index 2bcd491b2903..8d41aa938172 100644 --- a/spec/std/crystal/evented/timers_spec.cr +++ b/spec/std/crystal/evented/timers_spec.cr @@ -1,4 +1,4 @@ -{% skip_file if !flag?(:unix) || flag?(:evloop_libevent) %} +{% skip_file if !flag?(:unix) || flag?("evloop=libevent") %} require "spec" require "../../../../src/crystal/system/unix/evented/timers" diff --git a/spec/std/crystal/evented/waiters_spec.cr b/spec/std/crystal/evented/waiters_spec.cr index f61722b17b5c..5caba42993bf 100644 --- a/spec/std/crystal/evented/waiters_spec.cr +++ b/spec/std/crystal/evented/waiters_spec.cr @@ -1,4 +1,4 @@ -{% skip_file if !flag?(:unix) || flag?(:evloop_libevent) %} +{% skip_file if !flag?(:unix) || flag?("evloop=libevent") %} require "spec" require "../../../../src/crystal/system/unix/evented/waiters" diff --git a/src/crystal/system/event_loop.cr b/src/crystal/system/event_loop.cr index 356ebe5c3c8a..bd6c3a697362 100644 --- a/src/crystal/system/event_loop.cr +++ b/src/crystal/system/event_loop.cr @@ -8,11 +8,11 @@ abstract class Crystal::EventLoop # TODO: netbsd: the kqueue evloop doesn't work (needs investigation) # TODO: openbsd: the kqueue evloop is magnitudes slower (needs investigation) # TODO: solaris: the epoll evloop hasn't been tested (better: use event-ports) - {% if flag?(:evloop_libevent) || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} + {% if flag?("evloop=libevent") || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} Crystal::LibEvent::EventLoop.new - {% elsif flag?(:evloop_epoll) || flag?(:android) || flag?(:linux) %} + {% elsif flag?("evloop=epoll") || flag?(:android) || flag?(:linux) %} Crystal::Epoll::EventLoop.new - {% elsif flag?(:evloop_kqueue) || flag?(:darwin) || flag?(:freebsd) %} + {% elsif flag?("evloop=kqueue") || flag?(:darwin) || flag?(:freebsd) %} Crystal::Kqueue::EventLoop.new {% else %} {% raise "Event loop not supported" %} @@ -90,11 +90,11 @@ end {% if flag?(:wasi) %} require "./wasi/event_loop" {% elsif flag?(:unix) %} - {% if flag?(:evloop_libevent) || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} + {% if flag?("evloop=libevent") || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} require "./unix/event_loop_libevent" - {% elsif flag?(:evloop_epoll) || flag?(:android) || flag?(:linux) %} + {% elsif flag?("evloop=epoll") || flag?(:android) || flag?(:linux) %} require "./unix/epoll/event_loop" - {% elsif flag?(:evloop_kqueue) || flag?(:darwin) || flag?(:freebsd) %} + {% elsif flag?("evloop=kqueue") || flag?(:darwin) || flag?(:freebsd) %} require "./unix/kqueue/event_loop" {% else %} {% raise "Event loop not supported" %} diff --git a/src/io/evented.cr b/src/io/evented.cr index f0f518b27bff..54e3de500f8f 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:evloop_libevent) || flag?(:wasi) || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} +{% skip_file unless flag?("evloop=libevent") || flag?(:wasi) || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} require "crystal/thread_local_value" From 406d7d6abf286286be5cc1cdb431247078b12e14 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 4 Nov 2024 18:32:14 +0100 Subject: [PATCH 52/56] Format + avoid formatter bug (#15112) --- src/crystal/system/unix/evented/event_loop.cr | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index fc7e78d23da5..943885cba9ac 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -352,22 +352,22 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop event = Evented::Event.new(type, Fiber.current, index, timeout) return false unless Evented.arena.get?(index) do |pd| - yield pd, pointerof(event) - end + yield pd, pointerof(event) + end else # OPTIMIZE: failing to allocate may be a simple conflict with 2 fibers # starting to read or write on the same fd, we may want to detect any # error situation instead of returning and retrying a syscall return false unless Evented.arena.allocate_at?(io.fd) do |pd, index| - # register the fd with the event loop (once), it should usually merely add - # the fd to the current evloop but may "transfer" the ownership from - # another event loop: - io.__evloop_data = index - pd.value.take_ownership(self, io.fd, index) - - event = Evented::Event.new(type, Fiber.current, index, timeout) - yield pd, pointerof(event) - end + # register the fd with the event loop (once), it should usually merely add + # the fd to the current evloop but may "transfer" the ownership from + # another event loop: + io.__evloop_data = index + pd.value.take_ownership(self, io.fd, index) + + event = Evented::Event.new(type, Fiber.current, index, timeout) + yield pd, pointerof(event) + end end if event.wake_at? @@ -466,12 +466,12 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop when .io_read? # reached read timeout: cancel io event; by rule the timer always wins, # even in case of conflict with #unsafe_resume_io we must resume the fiber - Evented.arena.get?(event.value.index, &.value.@readers.delete(event)) + Evented.arena.get?(event.value.index) { |event| event.value.@readers.delete(event) } event.value.timed_out! when .io_write? # reached write timeout: cancel io event; by rule the timer always wins, # even in case of conflict with #unsafe_resume_io we must resume the fiber - Evented.arena.get?(event.value.index, &.value.@writers.delete(event)) + Evented.arena.get?(event.value.index) { |event| event.value.@writers.delete(event) } event.value.timed_out! when .select_timeout? # always dequeue the event but only enqueue the fiber if we win the From 4a66600ebe25c59573d34524943087571afbb1ad Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 4 Nov 2024 18:55:20 +0100 Subject: [PATCH 53/56] fixup! Format + avoid formatter bug (#15112) --- src/crystal/system/unix/evented/event_loop.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crystal/system/unix/evented/event_loop.cr b/src/crystal/system/unix/evented/event_loop.cr index 943885cba9ac..65b9e746b9b2 100644 --- a/src/crystal/system/unix/evented/event_loop.cr +++ b/src/crystal/system/unix/evented/event_loop.cr @@ -466,12 +466,12 @@ abstract class Crystal::Evented::EventLoop < Crystal::EventLoop when .io_read? # reached read timeout: cancel io event; by rule the timer always wins, # even in case of conflict with #unsafe_resume_io we must resume the fiber - Evented.arena.get?(event.value.index) { |event| event.value.@readers.delete(event) } + Evented.arena.get?(event.value.index) { |pd| pd.value.@readers.delete(event) } event.value.timed_out! when .io_write? # reached write timeout: cancel io event; by rule the timer always wins, # even in case of conflict with #unsafe_resume_io we must resume the fiber - Evented.arena.get?(event.value.index) { |event| event.value.@writers.delete(event) } + Evented.arena.get?(event.value.index) { |pd| pd.value.@writers.delete(event) } event.value.timed_out! when .select_timeout? # always dequeue the event but only enqueue the fiber if we win the From f782784e0b97b25982884df1e66910292a5c996c Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 4 Nov 2024 21:33:35 +0100 Subject: [PATCH 54/56] fixup! Use -Devloop=[libevent|epoll|kqueue] flag(s) --- spec/std/crystal/evented/arena_spec.cr | 3 +-- spec/std/crystal/evented/poll_descriptor_spec.cr | 3 +-- spec/std/crystal/evented/timers_spec.cr | 3 +-- spec/std/crystal/evented/waiters_spec.cr | 3 +-- src/crystal/system/event_loop.cr | 14 +++++++------- src/crystal/system/unix/evented/event.cr | 2 ++ src/io/evented.cr | 4 +++- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/spec/std/crystal/evented/arena_spec.cr b/spec/std/crystal/evented/arena_spec.cr index f07306070818..c25bb9ec1adc 100644 --- a/spec/std/crystal/evented/arena_spec.cr +++ b/spec/std/crystal/evented/arena_spec.cr @@ -1,7 +1,6 @@ -{% skip_file if !flag?(:unix) || flag?("evloop=libevent") %} +{% skip_file unless Crystal.has_constant?(:Evented) %} require "spec" -require "../../../../src/crystal/system/unix/evented/arena" describe Crystal::Evented::Arena do describe "#allocate_at?" do diff --git a/spec/std/crystal/evented/poll_descriptor_spec.cr b/spec/std/crystal/evented/poll_descriptor_spec.cr index b15177478523..d50ecd1036b9 100644 --- a/spec/std/crystal/evented/poll_descriptor_spec.cr +++ b/spec/std/crystal/evented/poll_descriptor_spec.cr @@ -1,7 +1,6 @@ -{% skip_file if !flag?(:unix) || flag?("evloop=libevent") %} +{% skip_file unless Crystal.has_constant?(:Evented) %} require "spec" -require "../../../../src/crystal/system/unix/evented/poll_descriptor" class Crystal::Evented::FakeLoop < Crystal::Evented::EventLoop getter operations = [] of {Symbol, Int32, Crystal::Evented::Arena::Index | Bool} diff --git a/spec/std/crystal/evented/timers_spec.cr b/spec/std/crystal/evented/timers_spec.cr index 8d41aa938172..d40917910d1d 100644 --- a/spec/std/crystal/evented/timers_spec.cr +++ b/spec/std/crystal/evented/timers_spec.cr @@ -1,7 +1,6 @@ -{% skip_file if !flag?(:unix) || flag?("evloop=libevent") %} +{% skip_file unless Crystal.has_constant?(:Evented) %} require "spec" -require "../../../../src/crystal/system/unix/evented/timers" describe Crystal::Evented::Timers do it "#empty?" do diff --git a/spec/std/crystal/evented/waiters_spec.cr b/spec/std/crystal/evented/waiters_spec.cr index 5caba42993bf..91e145f6f811 100644 --- a/spec/std/crystal/evented/waiters_spec.cr +++ b/spec/std/crystal/evented/waiters_spec.cr @@ -1,7 +1,6 @@ -{% skip_file if !flag?(:unix) || flag?("evloop=libevent") %} +{% skip_file unless Crystal.has_constant?(:Evented) %} require "spec" -require "../../../../src/crystal/system/unix/evented/waiters" describe Crystal::Evented::Waiters do describe "#add" do diff --git a/src/crystal/system/event_loop.cr b/src/crystal/system/event_loop.cr index bd6c3a697362..1a5978e6c00f 100644 --- a/src/crystal/system/event_loop.cr +++ b/src/crystal/system/event_loop.cr @@ -4,18 +4,18 @@ abstract class Crystal::EventLoop {% if flag?(:wasi) %} Crystal::Wasi::EventLoop.new {% elsif flag?(:unix) %} - # TODO: dragonfly: the kqueue evloop hasn't been tested + # TODO: dragonfly: the kqueue evloop doesn't work properly (regular hangs on evloop.run) # TODO: netbsd: the kqueue evloop doesn't work (needs investigation) - # TODO: openbsd: the kqueue evloop is magnitudes slower (needs investigation) - # TODO: solaris: the epoll evloop hasn't been tested (better: use event-ports) - {% if flag?("evloop=libevent") || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} + # TODO: openbsd: the kqueue evloop is sluggish when running std specs (the main fiber keeps reenqueueing itself from the evloop run) + # TODO: solaris: the epoll evloop hasn't been tested (better alternative: use event-ports) + {% if flag?("evloop=libevent") %} Crystal::LibEvent::EventLoop.new {% elsif flag?("evloop=epoll") || flag?(:android) || flag?(:linux) %} Crystal::Epoll::EventLoop.new {% elsif flag?("evloop=kqueue") || flag?(:darwin) || flag?(:freebsd) %} Crystal::Kqueue::EventLoop.new {% else %} - {% raise "Event loop not supported" %} + Crystal::LibEvent::EventLoop.new {% end %} {% elsif flag?(:win32) %} Crystal::IOCP::EventLoop.new @@ -90,14 +90,14 @@ end {% if flag?(:wasi) %} require "./wasi/event_loop" {% elsif flag?(:unix) %} - {% if flag?("evloop=libevent") || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} + {% if flag?("evloop=libevent") %} require "./unix/event_loop_libevent" {% elsif flag?("evloop=epoll") || flag?(:android) || flag?(:linux) %} require "./unix/epoll/event_loop" {% elsif flag?("evloop=kqueue") || flag?(:darwin) || flag?(:freebsd) %} require "./unix/kqueue/event_loop" {% else %} - {% raise "Event loop not supported" %} + require "./unix/event_loop_libevent" {% end %} {% elsif flag?(:win32) %} require "./win32/event_loop_iocp" diff --git a/src/crystal/system/unix/evented/event.cr b/src/crystal/system/unix/evented/event.cr index 146427777be4..b33130df53c2 100644 --- a/src/crystal/system/unix/evented/event.cr +++ b/src/crystal/system/unix/evented/event.cr @@ -1,3 +1,5 @@ +require "crystal/pointer_linked_list" + # Information about the event that a `Fiber` is waiting on. # # The event can be waiting for `IO` with or without a timeout, or be a timed diff --git a/src/io/evented.cr b/src/io/evented.cr index 54e3de500f8f..f59aa205c543 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -1,4 +1,6 @@ -{% skip_file unless flag?("evloop=libevent") || flag?(:wasi) || flag?(:dragonfly) || flag?(:netbsd) || flag?(:openbsd) || flag?(:solaris) %} +require "crystal/system/event_loop" + +{% skip_file unless flag?(:wasi) || Crystal.has_constant?(:LibEvent) %} require "crystal/thread_local_value" From a3a320c6f2df57c8405c4f97cc9edfc8418adc12 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 5 Nov 2024 13:08:10 +0100 Subject: [PATCH 55/56] Fix: crystal tool format --- src/crystal/system/unix/evented/arena.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/system/unix/evented/arena.cr b/src/crystal/system/unix/evented/arena.cr index 5dcb5c5c2c8c..818b80b83c41 100644 --- a/src/crystal/system/unix/evented/arena.cr +++ b/src/crystal/system/unix/evented/arena.cr @@ -180,7 +180,7 @@ class Crystal::Evented::Arena(T) # and returns false. # # Raises if *index* is negative. - def get?(index : Index) : Bool + def get?(index : Index, &) : Bool at?(index) do |entry| yield entry.value.pointer return true From 79bc334fea47f89761ddb63cd337b057eed881a2 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 7 Nov 2024 13:24:02 +0100 Subject: [PATCH 56/56] Fix: update todo --- src/crystal/system/event_loop.cr | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/crystal/system/event_loop.cr b/src/crystal/system/event_loop.cr index 1a5978e6c00f..33ff4f9dac85 100644 --- a/src/crystal/system/event_loop.cr +++ b/src/crystal/system/event_loop.cr @@ -4,10 +4,7 @@ abstract class Crystal::EventLoop {% if flag?(:wasi) %} Crystal::Wasi::EventLoop.new {% elsif flag?(:unix) %} - # TODO: dragonfly: the kqueue evloop doesn't work properly (regular hangs on evloop.run) - # TODO: netbsd: the kqueue evloop doesn't work (needs investigation) - # TODO: openbsd: the kqueue evloop is sluggish when running std specs (the main fiber keeps reenqueueing itself from the evloop run) - # TODO: solaris: the epoll evloop hasn't been tested (better alternative: use event-ports) + # TODO: enable more targets by default (need manual tests or fixes) {% if flag?("evloop=libevent") %} Crystal::LibEvent::EventLoop.new {% elsif flag?("evloop=epoll") || flag?(:android) || flag?(:linux) %}