Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/generators.h
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,14 @@ struct OrtGlobals {
std::unique_ptr<OrtEnv> env_;

struct Allocator {
std::unique_ptr<Ort::Allocator> allocator_;
// Field order matters here. The OrtAllocator returned by OrtApi::CreateAllocator (called via
// Ort::Allocator::Create) "wraps the internal allocator from the OrtSession and becomes invalid when the session
// does" -- see
// https://github.com/microsoft/onnxruntime/blob/3c8c46029735a89c8d1ea0aa6c1812db5b78ad72/include/onnxruntime/core/session/onnxruntime_c_api.h#L2852-L2862
// Members are destroyed in reverse declaration order, so session_ must be declared BEFORE allocator_ so that
// ~allocator_ runs first.
std::unique_ptr<OrtSession> session_;
std::unique_ptr<Ort::Allocator> allocator_;
};
Allocator device_allocators_[static_cast<int>(DeviceType::MAX)];

Expand Down
3 changes: 3 additions & 0 deletions src/models/onnxruntime_api.h
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,9 @@ struct Abstract {
*/
struct Allocator : OrtAllocator {
static Allocator& GetWithDefaultOptions(); ///< ::OrtAllocator default instance that is owned by Onnxruntime
/// Wraps the internal allocator owned by `session` (forwards to OrtApi::CreateAllocator). The returned Allocator
/// becomes invalid when `session` is destroyed -- callers must ensure the Allocator is destroyed before the
/// OrtSession.
static std::unique_ptr<Allocator> Create(const OrtSession& session, const OrtMemoryInfo& memory_info);

void* Alloc(size_t size);
Expand Down
12 changes: 12 additions & 0 deletions src/ort_genai.h
Original file line number Diff line number Diff line change
Expand Up @@ -847,6 +847,18 @@ struct OgaEngine : OgaAbstract {
static void operator delete(void* p) { OgaDestroyEngine(reinterpret_cast<OgaEngine*>(p)); }
};

/**
* \brief RAII wrapper that calls OgaShutdown() on destruction.
*
* \warning Without explicit shutdown, GenAI's globals are destroyed at static-destruction time in undefined order,
* which may crash.
*
* \note Typical usage is to construct an instance early in the program so its destructor runs before process exit.
*
* \note Only one OgaHandle should be live in the process, and its scope must encompass all GenAI use. GenAI's globals
* are not re-creatable after OgaShutdown(); a second OgaHandle whose lifetime starts after the first one's
* destruction would leave subsequent GenAI calls broken.
*/
struct OgaHandle {
OgaHandle() = default;
~OgaHandle() noexcept {
Expand Down
11 changes: 10 additions & 1 deletion src/ort_genai_c.h
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,16 @@ typedef struct OgaStreamingProcessor OgaStreamingProcessor;
*/

/**
* \brief Call this on process exit to cleanly shutdown the genai library & its onnxruntime usage
* \brief Shuts down the GenAI library and releases all GenAI-owned ONNX Runtime globals.
*
* \warning Callers SHOULD invoke OgaShutdown() before process exit. If it is not called, GenAI's globals are destroyed
* at static-destruction time in undefined order, which may crash.
*
* \note C++ callers should prefer the OgaHandle RAII wrapper in ort_genai.h; C# callers should prefer the
* OgaHandle IDisposable wrapper. Both invoke OgaShutdown() on destruction.
*
* \note Must be the last GenAI call in the process. Any OgaModel / OgaGenerator / OgaTokenizer / etc. handles owned
* by the caller must be released first.
*/
OGA_EXPORT void OGA_API_CALL OgaShutdown();

Expand Down
Loading