Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DataLogger] Log data is saved as a zip archive #1327

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.xml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
<build_depend>openhrp3</build_depend>
<build_depend>python-tk</build_depend>
<build_depend>sdl</build_depend>
<build_depend>libzip-dev</build_depend>

<run_depend>cv_bridge</run_depend>
<run_depend>glut</run_depend>
Expand Down
4 changes: 2 additions & 2 deletions rtc/DataLogger/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
set(comp_sources DataLogger.cpp DataLoggerService_impl.cpp)
set(libs hrpsysBaseStub)
add_library(DataLogger SHARED ${comp_sources})
target_link_libraries(DataLogger ${libs})
target_link_libraries(DataLogger ${libs} zip)
set_target_properties(DataLogger PROPERTIES PREFIX "")

add_executable(DataLoggerComp DataLoggerComp.cpp ${comp_sources})
target_link_libraries(DataLoggerComp ${libs})
target_link_libraries(DataLoggerComp ${libs} zip)

add_executable(logSplitter logSplitter.cpp)
target_link_libraries(logSplitter ${libs})
Expand Down
83 changes: 83 additions & 0 deletions rtc/DataLogger/DataLogger.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@
#include "hrpsys/idl/RobotHardwareService.hh"
#include "DataLogger.h"

extern "C" {
#include <zip.h>
}
#include <sstream>
#include <memory>

typedef coil::Guard<coil::Mutex> Guard;

Expand Down Expand Up @@ -502,6 +507,12 @@ bool DataLogger::add(const char *i_type, const char *i_name)

bool DataLogger::save(const char *i_basename)
{
{
std::string tmp_basename = i_basename;
if ( tmp_basename.size() > 4 && tmp_basename.substr(tmp_basename.size() - 4) == ".zip" ) {
return this->save_zip(tmp_basename);
}
}
suspendLogging();
bool ret = true;
for (unsigned int i=0; i<m_ports.size(); i++){
Expand All @@ -521,6 +532,78 @@ bool DataLogger::save(const char *i_basename)
return ret;
}

bool DataLogger::save_zip(const std::string &i_basename)
{
std::string dirname = "";
std::string basename = i_basename.substr(0, i_basename.size() - 4);
int pos = basename.find_last_of('/');
if (pos != std::string::npos) {
dirname = basename.substr(0, pos -1);
basename = basename.substr(pos+1);
}

int err = 0;
zip_t *zipf = zip_open(i_basename.c_str(), ZIP_CREATE | ZIP_EXCL, &err);
if (zipf == NULL) {
zip_error_t zip_err;
zip_error_init_with_code(&zip_err, err);
std::string err_str = zip_error_strerror(&zip_err);
std::cerr << "[" << m_profile.instance_name << "] faild zip_open / " << i_basename
<< " / " << err_str << std::endl;
return false;
}
// start logging
suspendLogging();
bool ret = true;
std::vector<std::shared_ptr< std::string> > strbuf; // require valid buffer until zip_close
for (unsigned int i=0; i<m_ports.size(); i++) {
std::string fname = basename;
fname.append(".");
fname.append(m_ports[i]->name());
std::ostringstream os;

m_ports[i]->dumpLog(os, m_log_precision);

std::shared_ptr< std::string> str = std::make_shared<std::string> (os.str());
strbuf.push_back(str);
zip_source_t *zs = zip_source_buffer(zipf, str->c_str(), str->size(), 0);
if (zs == NULL) {
std::string errstr = zip_strerror(zipf);
std::cerr << "[" << m_profile.instance_name << "] faild zip_source_buffer / " << fname
<< " / " << errstr << std::endl;
ret = false;
break;
}
long index = -1;
if ( (index = zip_file_add(zipf, fname.c_str(), zs, ZIP_FL_OVERWRITE)) < 0 ) {
std::string errstr = zip_strerror(zipf);
std::cerr << "[" << m_profile.instance_name << "] faild zip_file_add / " << fname
<< " / " << errstr << std::endl;
zip_source_free(zs);
ret = false;
break;
}
// set no-compression mode
if ( zip_set_file_compression(zipf, index, ZIP_CM_STORE, 0) < 0 ) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Naoki-Hiraoka
log保存時に現状より余分に時間がかかってしまうので無圧縮にしています。
ZIP_CM_DEFAULT にすると圧縮します。
https://libzip.org/documentation/zip_set_file_compression.html

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ログの容量が大きいと制御PCでsaveしてから遠隔PCに送るのに時間がかかるので圧縮できると便利だと思うのですが、

このPRはログを圧縮するのが目的かと勘違いして、この機能は便利でいいですね という意図で書いたのですが、勘違いでした。すみません。

ログを1つにまとめられる、ということですね。

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

圧縮と無圧縮どちらが良いかは環境に依存しそうで、サービスで設定できるようにするのが筋だろうとは思っていました。

std::string errstr = zip_strerror(zipf);
std::cerr << "[" << m_profile.instance_name << "] faild zip_set_file_compression / " << fname
<< " / " << errstr << std::endl;
}
}
resumeLogging();
// finish logging
if (zip_close(zipf) < 0) {
ret = false;
std::string errstr = zip_strerror(zipf);
std::cerr << "[" << m_profile.instance_name << "] faild zip_close / " << i_basename
<< " / " << errstr << std::endl;
}

if (ret) std::cerr << "[" << m_profile.instance_name << "] Save log to " << i_basename << ".*" << std::endl;

return ret;
}

bool DataLogger::clear()
{
suspendLogging();
Expand Down
1 change: 1 addition & 0 deletions rtc/DataLogger/DataLogger.h
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ class DataLogger
coil::Mutex m_suspendFlagMutex;
unsigned int m_log_precision;
int dummy;
bool save_zip(const std::string &i_basename);
};


Expand Down