Skip to content

Commit

Permalink
Memory monitor (#28)
Browse files Browse the repository at this point in the history
* memory monitor

* fix memory monitor
  • Loading branch information
koide3 authored Jul 20, 2024
1 parent 5798839 commit 7484f46
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 3 deletions.
10 changes: 9 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,15 @@ if(BUILD_WITH_VIEWER)
glim
)

list(APPEND glim_LIBRARIES standard_viewer interactive_viewer)
# Memory monitor
add_library(memory_monitor SHARED
src/glim/viewer/memory_monitor.cpp
)
target_link_libraries(memory_monitor
glim
)

list(APPEND glim_LIBRARIES standard_viewer interactive_viewer memory_monitor)
endif()

#############
Expand Down
1 change: 1 addition & 0 deletions config/config_ros.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"acc_scale": 1.0,
// Extension modules
"extension_modules": [
"libmemory_monitor.so",
"libstandard_viewer.so",
"librviz_viewer.so"
// "libimu_validator.so"
Expand Down
4 changes: 2 additions & 2 deletions docs/docker.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ cp -R /tmp/glim/config ./config
# Pull image from docker hub
docker pull koide3/glim_ros2:humble_cuda12.2

# Launch glim_ros:noetic image with GPU and DISPLAY support
# Launch glim_ros2:humble_cuda12.2 image with GPU and DISPLAY support
docker run \
-it \
--rm \
Expand Down Expand Up @@ -59,7 +59,7 @@ nano config/config.json
# Pull image from docker hub
docker pull koide3/glim_ros2:humble

# Launch glim_ros:noetic image with DISPLAY support
# Launch glim_ros2:humble image with DISPLAY support
docker run \
-it \
--rm \
Expand Down
98 changes: 98 additions & 0 deletions src/glim/viewer/memory_monitor.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#include <thread>
#include <fstream>
#include <sstream>
#include <iostream>
#include <glim/util/logging.hpp>
#include <glim/util/extension_module.hpp>
#include <gtsam_points/cuda/cuda_memory.hpp>

namespace glim {

class MemoryMonitor : public ExtensionModule {
public:
MemoryMonitor() : logger(create_module_logger("mem")) {
kill_switch = false;
thread = std::thread([this] { task(); });
}

~MemoryMonitor() {
kill_switch = true;
thread.join();
}

void task() {
auto last_update_time = std::chrono::high_resolution_clock::now();
while (!kill_switch) {
std::this_thread::sleep_for(std::chrono::seconds(1));

auto now = std::chrono::high_resolution_clock::now();
if (std::chrono::duration_cast<std::chrono::seconds>(now - last_update_time).count() < 5) {
continue;
}
last_update_time = now;

#ifdef BUILD_GTSAM_POINTS_GPU
size_t gpu_free = 0;
size_t gpu_total = 0;
gtsam_points::cuda_mem_get_info(&gpu_free, &gpu_total);
const double gpu_used = static_cast<double>(gpu_total - gpu_free) / gpu_total;

if (gpu_used > 0.8) {
const size_t gpu_used_mb = (gpu_total - gpu_free) / 1024 / 1024;
const size_t gpu_total_mb = gpu_total / 1024 / 1024;
logger->warn("GPU memory usage: {}/{} MB {:.2f}%", gpu_used_mb, gpu_total_mb, gpu_used * 100);
}
#endif

size_t mem_free_kb, mem_total_kb;
mem_usage(mem_free_kb, mem_total_kb);

const double cpu_used = static_cast<double>(mem_total_kb - mem_free_kb) / mem_total_kb;

if (cpu_used > 0.8) {
const double used_mb = static_cast<double>(mem_total_kb - mem_free_kb) / 1024.0;
const double total_mb = static_cast<double>(mem_total_kb) / 1024.0;
logger->warn("CPU memory usage: {:.2f} / {:.2f} MB {:.2f}%", used_mb, total_mb, cpu_used * 100);
}
}
}

void mem_usage(size_t& mem_free_kb, size_t& mem_total_kb) {
mem_free_kb = 0;
mem_total_kb = 0;

std::ifstream ifs("/proc/meminfo");
if (!ifs) {
logger->warn("Failed to open /proc/meminfo");
return;
}

std::string line;
while (!ifs.eof() && std::getline(ifs, line) && !line.empty()) {
std::istringstream sst(line);
std::string name;
if (line.find("MemTotal:") != std::string::npos) {
sst >> name >> mem_total_kb;
} else if (line.find("MemAvailable:") != std::string::npos) {
sst >> name >> mem_free_kb;
}

if (mem_free_kb > 0 && mem_total_kb > 0) {
break;
}
}
}

private:
std::atomic_bool kill_switch;
std::thread thread;

// Logging
std::shared_ptr<spdlog::logger> logger;
};

} // namespace glim

extern "C" glim::ExtensionModule* create_extension_module() {
return new glim::MemoryMonitor();
}

0 comments on commit 7484f46

Please sign in to comment.