diff --git a/engine/native/CMakeLists.txt b/engine/native/CMakeLists.txt index 50951f21..5aad2579 100644 --- a/engine/native/CMakeLists.txt +++ b/engine/native/CMakeLists.txt @@ -4,4 +4,4 @@ include(Modules) add_subdirectory(thirdparty SYSTEM) add_modules_library(core SHARED) -target_link_libraries(core PUBLIC definitions math) +target_link_libraries(core PUBLIC definitions math memory) diff --git a/engine/native/core/CMakeLists.txt b/engine/native/core/CMakeLists.txt index e941544c..43e4bae0 100644 --- a/engine/native/core/CMakeLists.txt +++ b/engine/native/core/CMakeLists.txt @@ -1,3 +1,4 @@ add_modules_library(definitions PIC) add_modules_library(math) +add_modules_library(memory) target_link_libraries(math PUBLIC definitions) \ No newline at end of file diff --git a/engine/native/core/core.cppm b/engine/native/core/core.cppm index 3b208b31..54d3ebc3 100644 --- a/engine/native/core/core.cppm +++ b/engine/native/core/core.cppm @@ -1,3 +1,4 @@ export module core; export import core.defs; export import core.math; +export import core.memory; diff --git a/engine/native/core/memory/allocator.cpp b/engine/native/core/memory/allocator.cpp new file mode 100644 index 00000000..9d74a648 --- /dev/null +++ b/engine/native/core/memory/allocator.cpp @@ -0,0 +1,34 @@ +module; + +#include + +module core.memory.allocator; + +namespace draco::memory +{ + Error nilAlloc( + Allocator alloc, + Slice *dst, + size_t size, + size_t align + ) + { + return Error::NotImplemented; + } + + Error nilFree(Allocator alloc, Slice block) + { + return Error::NotImplemented; + } + + Error nilFreeAll(Allocator alloc) + { + return Error::NotImplemented; + } + + void asAllocatorVoid(Allocator *dst, void *alloc, AllocatorVTbl *vtbl) + { + dst->allocatorData = (void*)alloc; + dst->vtbl = vtbl; + } +} diff --git a/engine/native/core/memory/allocator.cppm b/engine/native/core/memory/allocator.cppm new file mode 100644 index 00000000..2298aef7 --- /dev/null +++ b/engine/native/core/memory/allocator.cppm @@ -0,0 +1,55 @@ +module; + +#include + +export module core.memory.allocator; +export import core.memory.slice; + +export namespace draco::memory +{ + enum class Error + { + Okay, + OutOfMemory, + NotImplemented, + IllegalAddressRange, + Other, // This one shouldn't be needed. If you see it returned, make a + // new error. + }; + + struct AllocatorVTbl; + + struct Allocator + { + AllocatorVTbl *vtbl; + void *allocatorData; + }; + + struct AllocatorVTbl + { + using AllocFn = Error (*)( + Allocator alloc, + Slice *dst, + size_t size, + size_t align + ); + using FreeFn = Error (*)(Allocator alloc, Slice block); + using FreeAllFn = Error (*)(Allocator alloc); + AllocFn alloc; + FreeFn free; + FreeAllFn freeAll; + }; + + Error nilAlloc( + Allocator alloc, + Slice *dst, + size_t size, + size_t align + ); + + Error nilFree(Allocator alloc, Slice block); + + Error nilFreeAll(Allocator alloc); + + void asAllocatorVoid(Allocator *dst, void *alloc, AllocatorVTbl *vtbl); +} diff --git a/engine/native/core/memory/bumpAllocator.cpp b/engine/native/core/memory/bumpAllocator.cpp new file mode 100644 index 00000000..b88012fa --- /dev/null +++ b/engine/native/core/memory/bumpAllocator.cpp @@ -0,0 +1,117 @@ +module; + +#include +#include +#include +#include +#include + +module core.memory.bumpAllocator; + +namespace draco::memory::bump +{ + void init( + BumpAllocator *alloc, + Allocator baseAlloc, + // one page by default on unix-like systems + size_t minAllocRequest + ) + { + memset(alloc, 0, sizeof(BumpAllocator)); + alloc->base = baseAlloc; + alloc->minAllocRequest = minAllocRequest; + } + + void deinit(BumpAllocator *alloc) + { + Node *lastNode; + Node *node = alloc->first; + while (node != nullptr) + { + lastNode = node; + node = node->next; + alloc->base.vtbl->free( + alloc->base, + { + .data = (void*)lastNode, + .size = lastNode->size + sizeof(Node), + } + ); + } + } + + Error alloc(Allocator alloc, Slice *dst, size_t size, size_t align) + { + Error err; + BumpAllocator *allocData = (BumpAllocator *)alloc.allocatorData; + uintptr_t alignMask = align - 1; + Node **lastNode; + Node **node = &(allocData->first); + size_t pos = allocData->allocated; + size_t oldPos = pos; + size_t reqSize = size; + size_t spillover = 0; + Slice newBlock; + uintptr_t currentPtr; + assert(std::popcount(align) == 1); + lastNode = node; + while (((*node) != nullptr) & (pos > 0)) + { + oldPos = pos; + pos -= std::min((*node)->size, pos); + lastNode = node; + node = &((*node)->next); + } + assert(pos == 0); // fraudulent mark provided + currentPtr = ((uintptr_t)(*lastNode)) + sizeof(Node) + oldPos; + reqSize = size + ((align - (currentPtr & alignMask)) & alignMask); + if (!(*lastNode) || (reqSize > ((*lastNode)->size - oldPos))) + { + if (*lastNode) + { + spillover = ((*lastNode)->size - oldPos); + } + reqSize = (sizeof(Node) + size + alignMask) & ~alignMask; + err = allocData->base.vtbl->alloc( + allocData->base, + &newBlock, + std::max(allocData->minAllocRequest, reqSize), + std::max(alignof(Node), align) + ); + if (err != Error::Okay) + { + return Error::OutOfMemory; + } + (*node) = (Node *)newBlock.data; + (*node)->next = nullptr; + (*node)->size = newBlock.size - sizeof(Node); + pos = 0; + lastNode = node; + oldPos = 0; + } + currentPtr = ((uintptr_t)&((*lastNode)->data[oldPos])); + reqSize = size + ((align - (currentPtr & alignMask)) & alignMask); + currentPtr = (currentPtr + alignMask) & ~alignMask; + allocData->allocated += reqSize + spillover; + dst->data = (void*)currentPtr; + dst->size = size; + return Error::Okay; + } + + Error freeAll(Allocator alloc) + { + BumpAllocator *allocData = (BumpAllocator *)alloc.allocatorData; + allocData->allocated = 0; + return Error::Okay; + } + + size_t saveMark(BumpAllocator *self) + { + return self->allocated; + } + + void resumeMark(BumpAllocator *self, size_t mark) + { + self->allocated = mark; + } +} diff --git a/engine/native/core/memory/bumpAllocator.cppm b/engine/native/core/memory/bumpAllocator.cppm new file mode 100644 index 00000000..79237682 --- /dev/null +++ b/engine/native/core/memory/bumpAllocator.cppm @@ -0,0 +1,59 @@ +module; + +#include +#include +#include +#include + +export module core.memory.bumpAllocator; +export import core.memory.allocator; +export import core.memory.slice; + +export namespace draco::memory +{ + namespace bump + { + struct Node + { + Node *next; + size_t size; + uint8_t data[]; + }; + + struct BumpAllocator + { + Allocator base; + Node *first; + size_t minAllocRequest; + size_t allocated; + }; + + void init( + BumpAllocator *alloc, + Allocator baseAlloc, + // one page by default on unix-like systems + size_t minAllocRequest = (1 << 12) + ); + + void deinit(BumpAllocator *alloc); + + Error alloc(Allocator alloc, Slice *dst, size_t size, size_t align); + + Error freeAll(Allocator alloc); + + AllocatorVTbl bumpAllocatorVtbl = { + .alloc = alloc, + .free = nilFree, + .freeAll = freeAll, + }; + + size_t saveMark(BumpAllocator *self); + + void resumeMark(BumpAllocator *self, size_t mark); + + inline void asAllocator(Allocator *dst, BumpAllocator *alloc) + { + asAllocatorVoid(dst, (void*)alloc, &bumpAllocatorVtbl); + } + } +} diff --git a/engine/native/core/memory/bumpAllocator.test.cpp b/engine/native/core/memory/bumpAllocator.test.cpp new file mode 100644 index 00000000..39670d8c --- /dev/null +++ b/engine/native/core/memory/bumpAllocator.test.cpp @@ -0,0 +1,122 @@ +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include + +import core.memory; + +TEST_CASE("Bump allocator provides distinct pointers on allocation") +{ + using namespace draco::memory; + bump::BumpAllocator bumpAlloc; + Allocator alloc; + Slice a; + Slice b; + Error err; + bump::init(&bumpAlloc, page::pageAllocator); + bump::asAllocator(&alloc, &bumpAlloc); + err = alloc.vtbl->alloc(alloc, &a, sizeof(int), alignof(int)); + REQUIRE(err == Error::Okay); + REQUIRE(bumpAlloc.first != nullptr); + // REQUIRE(bumpAlloc.first->next == nullptr); + REQUIRE(bumpAlloc.first->size > bumpAlloc.allocated); + err = alloc.vtbl->alloc(alloc, &b, sizeof(int), alignof(int)); + REQUIRE(err == Error::Okay); + REQUIRE((((ptrdiff_t)b.data) - ((ptrdiff_t)a.data)) >= sizeof(int)); + bump::deinit(&bumpAlloc); +} + +TEST_CASE("Bump allocator aligns pointers correctly") +{ + using namespace draco::memory; + bump::BumpAllocator bumpAlloc; + Allocator alloc; + Slice a; + Slice b; + Error err; + bump::init(&bumpAlloc, page::pageAllocator); + bump::asAllocator(&alloc, &bumpAlloc); + err = alloc.vtbl->alloc(alloc, &a, sizeof(uint8_t), 2); + REQUIRE(err == Error::Okay); + REQUIRE(bumpAlloc.first != nullptr); + // REQUIRE(bumpAlloc.first->next == nullptr); + REQUIRE(bumpAlloc.first->size > bumpAlloc.allocated); + err = alloc.vtbl->alloc(alloc, &b, sizeof(uint8_t), 4); + REQUIRE(err == Error::Okay); + REQUIRE((((uintptr_t)a.data) & (2 - 1)) == 0); + REQUIRE((((uintptr_t)b.data) & (4 - 1)) == 0); + bump::deinit(&bumpAlloc); +} + +TEST_CASE("Bump allocator data is well packed") +{ + struct Foo + { + uint32_t a; + uint64_t b; + }; + using namespace draco::memory; + bump::BumpAllocator bumpAlloc; + Allocator alloc; + Slice aSlice; + Slice bSlice; + uint32_t *a; + uint64_t *b; + Error err; + bump::init(&bumpAlloc, page::pageAllocator); + bump::asAllocator(&alloc, &bumpAlloc); + err = alloc.vtbl->alloc(alloc, &aSlice, sizeof(uint32_t), alignof(Foo)); + REQUIRE(err == Error::Okay); + REQUIRE(bumpAlloc.first != nullptr); + // REQUIRE(bumpAlloc.first->next == nullptr); + REQUIRE(bumpAlloc.first->size > bumpAlloc.allocated); + err = alloc.vtbl->alloc(alloc, &bSlice, sizeof(uint64_t), alignof(Foo)); + REQUIRE(err == Error::Okay); + a = (uint32_t*)aSlice.data; + b = (uint64_t*)bSlice.data; + *a = 69; + *b = 420; + REQUIRE(((Foo*)bumpAlloc.first->data)->a == 69); + REQUIRE(((Foo*)bumpAlloc.first->data)->b == 420); + bump::deinit(&bumpAlloc); +} + +TEST_CASE("Bump allocator allocates second page when available") +{ + using namespace draco::memory; + bump::BumpAllocator bumpAlloc; + Allocator alloc; + Slice aSlice; + Slice bSlice; + Error err; + bump::init(&bumpAlloc, page::pageAllocator); + bump::asAllocator(&alloc, &bumpAlloc); + err = alloc.vtbl->alloc(alloc, &aSlice, 8192, 1); + REQUIRE(err == Error::Okay); + err = alloc.vtbl->alloc(alloc, &bSlice, 8192, 1); + REQUIRE(err == Error::Okay); + REQUIRE(aSlice.data != bSlice.data); + REQUIRE(aSlice.size == 8192); + REQUIRE(bSlice.size == 8192); + REQUIRE(bumpAlloc.first->next != nullptr); + bump::deinit(&bumpAlloc); +} + +TEST_CASE("Exact alignment") +{ + using namespace draco::memory; + bump::BumpAllocator bumpAlloc; + Allocator alloc; + Slice aSlice; + Slice bSlice; + Slice cSlice; + Error err; + bump::init(&bumpAlloc, page::pageAllocator); + bump::asAllocator(&alloc, &bumpAlloc); + err = alloc.vtbl->alloc(alloc, &aSlice, 5, 1); + REQUIRE(err == Error::Okay); + err = alloc.vtbl->alloc(alloc, &bSlice, 8, 8); + REQUIRE(err == Error::Okay); + err = alloc.vtbl->alloc(alloc, &cSlice, 4, 4); + REQUIRE(err == Error::Okay); + REQUIRE(bumpAlloc.allocated == 20); + bump::deinit(&bumpAlloc); +} diff --git a/engine/native/core/memory/fixedAllocator.cpp b/engine/native/core/memory/fixedAllocator.cpp new file mode 100644 index 00000000..b2768950 --- /dev/null +++ b/engine/native/core/memory/fixedAllocator.cpp @@ -0,0 +1,51 @@ +module; + +#include +#include +#include +#include + +module core.memory.fixedAllocator; + +namespace draco::memory::fixed +{ + void init(FixedAllocator *alloc, Slice block) + { + alloc->buffer = (uint8_t *)block.data; + alloc->size = block.size; + alloc->allocated = false; + } + + Error alloc( + Allocator alloc, + Slice *dst, + size_t size, + size_t align + ) + { + FixedAllocator *allocData = (FixedAllocator*)alloc.allocatorData; + size_t alignMask = align - 1; + size_t alignedSize = allocData->size - ( + (align - (((uintptr_t)allocData->buffer) & alignMask)) + & alignMask + ); + assert(std::popcount(align) == 1); + if (allocData->allocated | (alignedSize < size)) + { + return Error::OutOfMemory; + } + dst->data = (void *)( + ((uintptr_t)&(allocData->buffer[alignMask])) & ~alignMask + ); + dst->size = alignedSize; + allocData->allocated = true; + return Error::Okay; + } + + Error freeAll(Allocator alloc) + { + FixedAllocator *allocData = (FixedAllocator*)alloc.allocatorData; + allocData->allocated = false; + return Error::Okay; + } +} diff --git a/engine/native/core/memory/fixedAllocator.cppm b/engine/native/core/memory/fixedAllocator.cppm new file mode 100644 index 00000000..f6658b01 --- /dev/null +++ b/engine/native/core/memory/fixedAllocator.cppm @@ -0,0 +1,39 @@ +module; + +#include +#include +#include + +export module core.memory.fixedAllocator; +export import core.memory.allocator; +export import core.memory.slice; + +export namespace draco::memory +{ + namespace fixed + { + struct FixedAllocator + { + uint8_t *buffer; + size_t size; + bool allocated; + }; + + void init(FixedAllocator *alloc, Slice block); + + Error alloc(Allocator alloc, Slice *dst, size_t size, size_t align); + + Error freeAll(Allocator alloc); + + AllocatorVTbl fixedAllocatorVtbl = { + .alloc = alloc, + .free = nilFree, + .freeAll = freeAll, + }; + + inline void asAllocator(Allocator *dst, FixedAllocator *alloc) + { + asAllocatorVoid(dst, (void*)alloc, &fixedAllocatorVtbl); + } + } +} diff --git a/engine/native/core/memory/fixedAllocator.test.cpp b/engine/native/core/memory/fixedAllocator.test.cpp new file mode 100644 index 00000000..319eb548 --- /dev/null +++ b/engine/native/core/memory/fixedAllocator.test.cpp @@ -0,0 +1,17 @@ +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include + +import core.memory; + +TEST_CASE("Fixed allocator is correctly aligned") +{ + using namespace draco::memory; + alignas(8) uint8_t buffer[1024]; + fixed::FixedAllocator fixedAlloc; + Allocator alloc; + Slice block; + fixed::init(&fixedAlloc, { .data = buffer, .size = 1024 }); + fixed::asAllocator(&alloc, &fixedAlloc); + alloc.vtbl->alloc(alloc, &block, 512, 16); + REQUIRE((((uintptr_t)block.data) & 15) == 0); +} diff --git a/engine/native/core/memory/pageAllocator.cpp b/engine/native/core/memory/pageAllocator.cpp new file mode 100644 index 00000000..1423845b --- /dev/null +++ b/engine/native/core/memory/pageAllocator.cpp @@ -0,0 +1,123 @@ +module; + +#include +#ifdef __unix__ +#include +#include +#endif +#ifdef _WIN32 +#include +#include +#endif + +module core.memory.pageAllocator; + +namespace draco::memory::page +{ +#ifdef __unix__ + Error alloc( + Allocator alloc, + Slice *dst, + size_t size, + size_t align + ) + { + int pageSizeSub1 = getpagesize() - 1; + // Coderabbit, this is for a 64-bit machine with 48-bit addressing, + // if this overflows, the request was never going to fit into + // memory to begin with. + size_t reqSize = (size + (pageSizeSub1)) & (~pageSizeSub1); + void *ptr = mmap( + nullptr, + reqSize, + PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, + -1, + 0 + ); + if (((ptrdiff_t)ptr) == -1) + { + return Error::OutOfMemory; + } + dst->data = ptr; + dst->size = reqSize; + return Error::Okay; + } + + Error free(Allocator alloc, Slice block) + { + return munmap(block.data, block.size) ? + Error::IllegalAddressRange : + Error::Okay; + } +#endif +#ifdef _WIN32 + Error alloc( + Allocator alloc, + Slice *dst, + size_t size, + size_t align + ) + { + SYSTEM_INFO sysinfo; + size_t pageSizeSub1; + size_t reqSize; + void *ptr; + GetSystemInfo(&sysinfo); + pageSizeSub1 = (size_t)(sysinfo.dwAllocationGranularity - 1); + // Coderabbit, this is for a 64-bit machine with 48-bit addressing, + // if this overflows, the request was never going to fit into + // memory to begin with. + reqSize = (size + (pageSizeSub1)) & (~pageSizeSub1); + ptr = VirtualAlloc( + nullptr, + reqSize, + MEM_COMMIT | MEM_RESERVE, + PAGE_READWRITE + ); + if (ptr == nullptr) + { + return Error::OutOfMemory; + } + dst->data = ptr; + dst->size = reqSize; + return Error::Okay; + } + + Error allocLargePages( + Allocator alloc, + Slice *dst, + size_t size, + size_t align + ) + { + size_t pageSize = GetLargePageMinimum(); + size_t pageSizeSub1 = (pageSize ? pageSize : (4 * 1024)) - 1; + // Coderabbit, this is for a 64-bit machine with 48-bit addressing, + // if this overflows, the request was never going to fit into + // memory to begin with. + size_t reqSize = (size + (pageSizeSub1)) & (~pageSizeSub1); + void *ptr; + ptr = VirtualAlloc( + nullptr, + reqSize, + MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES, + PAGE_READWRITE + ); + if (ptr == nullptr) + { + return Error::OutOfMemory; + } + dst->data = ptr; + dst->size = reqSize; + return Error::Okay; + } + + Error free(Allocator alloc, Slice block) + { + return VirtualFree(block.data, 0, MEM_RELEASE) ? + Error::Okay : + Error::IllegalAddressRange; + } +#endif +} diff --git a/engine/native/core/memory/pageAllocator.cppm b/engine/native/core/memory/pageAllocator.cppm new file mode 100644 index 00000000..cc36b073 --- /dev/null +++ b/engine/native/core/memory/pageAllocator.cppm @@ -0,0 +1,27 @@ +module; + +#include + +export module core.memory.pageAllocator; +export import core.memory.allocator; +export import core.memory.slice; + +export namespace draco::memory +{ + namespace page + { + Error alloc(Allocator alloc, Slice *dst, size_t size, size_t align); + + Error free(Allocator alloc, Slice block); + + AllocatorVTbl pageAllocatorVtbl = { + .alloc = alloc, + .free = free, + .freeAll = nilFreeAll, + }; + Allocator pageAllocator = { + .vtbl = &pageAllocatorVtbl, + .allocatorData = nullptr, + }; + } +} diff --git a/engine/native/core/memory/root.cppm b/engine/native/core/memory/root.cppm new file mode 100644 index 00000000..6928cd7b --- /dev/null +++ b/engine/native/core/memory/root.cppm @@ -0,0 +1,7 @@ +module; + +export module core.memory; +export import core.memory.allocator; +export import core.memory.fixedAllocator; +export import core.memory.pageAllocator; +export import core.memory.bumpAllocator; diff --git a/engine/native/core/memory/slice.cppm b/engine/native/core/memory/slice.cppm new file mode 100644 index 00000000..0a632d16 --- /dev/null +++ b/engine/native/core/memory/slice.cppm @@ -0,0 +1,14 @@ +module; + +#include + +export module core.memory.slice; + +export namespace draco::memory +{ + struct Slice + { + void *data; + size_t size; + }; +}