ggml, server: add ggml_backend_dev_reset() for sleep mode

ggml/include/ggml-backend.h

@@ -190,6 +190,7 @@ extern "C" {
     GGML_API bool                          ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
     GGML_API bool                          ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft);
     GGML_API bool                          ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
+    GGML_API bool                          ggml_backend_dev_reset(ggml_backend_dev_t device);
 
     //
     // Backend (reg)

ggml/src/ggml-backend-impl.h

@@ -199,6 +199,10 @@ extern "C" {
         ggml_backend_event_t (*event_new)         (ggml_backend_dev_t dev);
         void                 (*event_free)        (ggml_backend_dev_t dev, ggml_backend_event_t event);
         void                 (*event_synchronize) (ggml_backend_dev_t dev, ggml_backend_event_t event);
+
+        // (optional) release all resources held for this device (eg. GPU memory/context), the device must remain usable afterwards
+        // returns false if unsupported
+        bool (*reset)(ggml_backend_dev_t dev);
     };
 
     struct ggml_backend_device {

ggml/src/ggml-backend-meta.cpp

@@ -191,6 +191,7 @@ static const ggml_backend_device_i ggml_backend_meta_device_iface = {
     /* .event_new            = */ nullptr,
     /* .event_free           = */ nullptr,
     /* .event_synchronize    = */ nullptr,
+    /* .reset                = */ nullptr,
 };
 
 static bool ggml_backend_dev_is_meta(ggml_backend_dev_t dev) {

ggml/src/ggml-backend.cpp

@@ -639,6 +639,15 @@ bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_te
     return false;
 }
 
+bool ggml_backend_dev_reset(ggml_backend_dev_t device) {
+    GGML_ASSERT(device);
+    if (device->iface.reset != NULL) {
+        return device->iface.reset(device);
+    }
+
+    return false;
+}
+
 // Backend (reg)
 
 const char * ggml_backend_reg_name(ggml_backend_reg_t reg) {

ggml/src/ggml-blas/ggml-blas.cpp

@@ -461,6 +461,7 @@ static const struct ggml_backend_device_i ggml_backend_blas_device_i = {
     /* .event_new            = */ NULL,
     /* .event_free           = */ NULL,
     /* .event_synchronize    = */ NULL,
+    /* .reset                = */ NULL,
 };
 
 // backend reg interface

ggml/src/ggml-cann/ggml-cann.cpp

@@ -2944,6 +2944,7 @@ static const ggml_backend_device_i ggml_backend_cann_device_interface = {
     /* .event_new               = */ ggml_backend_cann_device_event_new,
     /* .event_free              = */ ggml_backend_cann_device_event_free,
     /* .event_synchronize       = */ ggml_backend_cann_device_event_synchronize,
+    /* .reset                   = */ NULL,
 };
 
 // backend reg

ggml/src/ggml-cpu/ggml-cpu.cpp

@@ -494,6 +494,7 @@ static const struct ggml_backend_device_i ggml_backend_cpu_device_i = {
     /* .event_new            = */ NULL,
     /* .event_free           = */ NULL,
     /* .event_synchronize    = */ NULL,
+    /* .reset                = */ NULL,
 };
 
 // CPU backend - backend (reg)

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -5595,6 +5595,14 @@ static void ggml_backend_cuda_device_event_synchronize(ggml_backend_dev_t dev, g
     CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));
 }
 
+static bool ggml_backend_cuda_device_reset(ggml_backend_dev_t dev) {
+    ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
+    auto & device = dev_ctx->device;
+    CUDA_CHECK(cudaSetDevice(device));
+    CUDA_CHECK(cudaDeviceReset());
+    return true;
+}
+
 static const ggml_backend_device_i ggml_backend_cuda_device_interface = {
     /* .get_name                = */ ggml_backend_cuda_device_get_name,
     /* .get_description         = */ ggml_backend_cuda_device_get_description,
@@ -5611,6 +5619,7 @@ static const ggml_backend_device_i ggml_backend_cuda_device_interface = {
     /* .event_new               = */ ggml_backend_cuda_device_event_new,
     /* .event_free              = */ ggml_backend_cuda_device_event_free,
     /* .event_synchronize       = */ ggml_backend_cuda_device_event_synchronize,
+    /* .reset                   = */ ggml_backend_cuda_device_reset,
 };
 
 // backend reg

ggml/src/ggml-cuda/vendors/hip.h

@@ -60,6 +60,7 @@
 #define cudaDeviceGetPCIBusId hipDeviceGetPCIBusId
 #define cudaDeviceProp hipDeviceProp_t
 #define cudaDeviceSynchronize hipDeviceSynchronize
+#define cudaDeviceReset hipDeviceReset
 #define cudaError_t hipError_t
 #define cudaErrorMemoryAllocation hipErrorOutOfMemory
 #define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled

ggml/src/ggml-cuda/vendors/musa.h

@@ -44,6 +44,7 @@
 #define cudaDeviceGetPCIBusId musaDeviceGetPCIBusId
 #define cudaDeviceProp musaDeviceProp
 #define cudaDeviceSynchronize musaDeviceSynchronize
+#define cudaDeviceReset musaDeviceReset
 #define cudaError_t musaError_t
 #define cudaErrorMemoryAllocation musaErrorMemoryAllocation
 #define cudaErrorPeerAccessAlreadyEnabled musaErrorPeerAccessAlreadyEnabled

ggml/src/ggml-hexagon/ggml-hexagon.cpp

@@ -4155,6 +4155,7 @@ static const struct ggml_backend_device_i ggml_backend_hexagon_device_i = {
     /* .event_new            = */ NULL,
     /* .event_free           = */ NULL,
     /* .event_synchronize    = */ NULL,
+    /* .reset                = */ NULL,
 };
 
 //** backend registry

ggml/src/ggml-metal/ggml-metal.cpp

@@ -810,6 +810,7 @@ static ggml_backend_device_i ggml_backend_metal_device_i = {
     /* .event_new            = */ ggml_backend_metal_device_event_new,
     /* .event_free           = */ ggml_backend_metal_device_event_free,
     /* .event_synchronize    = */ ggml_backend_metal_device_event_synchronize,
+    /* .reset                = */ NULL,
 };
 
 // backend registry

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -9454,6 +9454,7 @@ struct ggml_backend_device_i ggml_backend_opencl_device_i = {
     /* .event_new            = */ NULL,
     /* .event_free           = */ NULL,
     /* .event_synchronize    = */ NULL,
+    /* .reset                = */ NULL,
 };
 }
 

ggml/src/ggml-openvino/ggml-openvino.cpp

@@ -1307,6 +1307,7 @@ static const struct ggml_backend_device_i ggml_backend_openvino_device_interface
     /* .event_new            = */ NULL,
     /* .event_free           = */ NULL,
     /* .event_synchronize    = */ NULL,
+    /* .reset                = */ NULL,
 };
 
 struct ggml_backend_openvino_reg_context {

ggml/src/ggml-rpc/ggml-rpc.cpp

@@ -1851,6 +1851,7 @@ static const struct ggml_backend_device_i ggml_backend_rpc_device_i = {
     /* .event_new            = */ NULL,
     /* .event_free           = */ NULL,
     /* .event_synchronize    = */ NULL,
+    /* .reset                = */ NULL,
 };
 
 // backend reg interface

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -5835,6 +5835,7 @@ static const ggml_backend_device_i ggml_backend_sycl_device_interface = {
     /* .event_new               = */ ggml_backend_sycl_device_event_new,
     /* .event_free              = */ ggml_backend_sycl_device_event_free,
     /* .event_synchronize       = */ ggml_backend_sycl_device_event_synchronize,
+    /* .reset                   = */ nullptr,
 };
 
 // backend reg

ggml/src/ggml-virtgpu/ggml-backend-device.cpp

@@ -157,4 +157,5 @@ const ggml_backend_device_i ggml_backend_remoting_device_interface = {
     /* .event_new            = */ NULL,
     /* .event_free           = */ NULL,
     /* .event_synchronize    = */ NULL,
+    /* .reset                = */ NULL,
 };

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -17871,6 +17871,7 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {
     /* .event_new            = */ ggml_backend_vk_device_event_new,
     /* .event_free           = */ ggml_backend_vk_device_event_free,
     /* .event_synchronize    = */ ggml_backend_vk_device_event_synchronize,
+    /* .reset                = */ nullptr,
 };
 
 static const char * ggml_backend_vk_reg_get_name(ggml_backend_reg_t reg) {

ggml/src/ggml-webgpu/ggml-webgpu.cpp

@@ -4457,6 +4457,7 @@ static struct ggml_backend_device_i ggml_backend_webgpu_device_i = {
     /* .event_new            = */ ggml_backend_webgpu_device_event_new,
     /* .event_free           = */ ggml_backend_webgpu_device_event_free,
     /* .event_synchronize    = */ ggml_backend_webgpu_device_event_synchronize,
+    /* .reset                = */ NULL,
 };
 
 /* End GGML Backend Device Interface */

ggml/src/ggml-zdnn/ggml-zdnn.cpp

@@ -546,6 +546,7 @@ static ggml_backend_device_i ggml_backend_zdnn_device_i = {
     /* .event_new            = */ NULL,
     /* .event_free           = */ NULL,
     /* .event_synchronize    = */ NULL,
+    /* .reset                = */ NULL,
 };
 
 //

ggml/src/ggml-zendnn/ggml-zendnn.cpp

@@ -646,6 +646,7 @@ static const struct ggml_backend_device_i ggml_backend_zendnn_device_i = {
     /* .event_new              = */ NULL,
     /* .event_free             = */ NULL,
     /* .event_synchronize      = */ NULL,
+    /* .reset                  = */ NULL,
 };
 
 // backend reg interface

tools/server/server-context.cpp

@@ -937,7 +937,7 @@ struct server_context_impl {
 
     int64_t t_last_load_progress_ms = 0;
 
-    void destroy() {
+    void destroy(bool reset_devices = false) {
         spec.reset();
         ctx_dft.reset();
         model_dft.reset();
@@ -949,13 +949,20 @@ struct server_context_impl {
 
         mtmd_free(mctx);
         mctx = nullptr;
+
+        // in sleep mode, we need to reset the devices to free up memory
+        if (reset_devices) {
+            for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
+                ggml_backend_dev_reset(ggml_backend_dev_get(i));
+            }
+        }
     }
 
     void handle_sleeping_state(bool new_state) {
         GGML_ASSERT(sleeping != new_state);
         if (new_state) {
             SRV_INF("%s", "server is entering sleeping state\n");
-            destroy();
+            destroy(true);
         } else {
             SRV_INF("%s", "server is exiting sleeping state\n");
             if (!load_model(params_base)) {