Skip to content

Commit

Permalink
Patch 20190208.
Browse files Browse the repository at this point in the history
  • Loading branch information
@yangsiran
yangsiran committed Apr 10, 2019
1 parent 5e270e5 commit 653bb01
Show file tree
Hide file tree
Showing 33 changed files with 746 additions and 449 deletions.
6 changes: 3 additions & 3 deletions cmake/jemalloc.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@ ExternalProject_Add(jemalloc
ExternalProject_Get_Property(jemalloc INSTALL_DIR)

add_library(jemalloc_STATIC STATIC IMPORTED)
set_property(TARGET jemalloc_STATIC PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libjemallloc.a)
set_property(TARGET jemalloc_STATIC PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/src/jemalloc/lib/libjemalloc.a)
add_dependencies(jemalloc_STATIC jemalloc)

add_library(jemalloc_STATIC_PIC STATIC IMPORTED)
set_property(TARGET jemalloc_STATIC_PIC PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libjemallloc_pic.a)
set_property(TARGET jemalloc_STATIC_PIC PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/src/jemalloc/lib/libjemalloc_pic.a)
add_dependencies(jemalloc_STATIC_PIC jemalloc)

add_library(jemalloc_SHARED SHARED IMPORTED)
set_property(TARGET jemalloc_SHARED PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libjemallloc.so)
set_property(TARGET jemalloc_SHARED PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/src/jemalloc/lib/libjemalloc.so)
add_dependencies(jemalloc_SHARED jemalloc)

if (!APPLE)
Expand Down
2 changes: 1 addition & 1 deletion euler/client/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ add_library(client SHARED
rpc_client.cc
rpc_manager.cc
status.cc)
target_link_libraries(client common core proto glog)
target_link_libraries(client common core proto glog jemalloc_STATIC_PIC)

add_executable(graph_test graph_test.cc)
target_link_libraries(graph_test client
Expand Down
11 changes: 7 additions & 4 deletions euler/common/compact_weighted_collection.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,10 @@ class CompactWeightedCollection : public WeightedCollection<T> {
CompactWeightedCollection() {
}

void Init(const std::vector<T>& ids,
bool Init(const std::vector<T>& ids,
const std::vector<float>& weights) override;

void Init(const std::vector<std::pair<T, float>>& id_weight_pairs) override;
bool Init(const std::vector<std::pair<T, float>>& id_weight_pairs) override;

std::pair<T, float> Sample() const override;

Expand All @@ -78,7 +78,7 @@ class CompactWeightedCollection : public WeightedCollection<T> {
};

template<class T>
void CompactWeightedCollection<T>::Init(const std::vector<T>& ids,
bool CompactWeightedCollection<T>::Init(const std::vector<T>& ids,
const std::vector<float>& weights) {
if (ids.size() == weights.size()) {
sum_weight_ = 0.0;
Expand All @@ -89,13 +89,15 @@ void CompactWeightedCollection<T>::Init(const std::vector<T>& ids,
sum_weight_ += weights[i];
sum_weights_[i] = sum_weight_;
}
return true;
} else {
LOG(ERROR) << "ids size != weights size, init error";
return false;
}
}

template<class T>
void CompactWeightedCollection<T>::Init(
bool CompactWeightedCollection<T>::Init(
const std::vector<std::pair<T, float>>& id_weight_pairs) {
sum_weight_ = 0.0;
ids_.resize(id_weight_pairs.size());
Expand All @@ -105,6 +107,7 @@ void CompactWeightedCollection<T>::Init(
sum_weight_ += id_weight_pairs[i].second;
sum_weights_[i] = sum_weight_;
}
return true;
}

template<class T>
Expand Down
14 changes: 7 additions & 7 deletions euler/common/fast_weighted_collection.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ namespace common {
template<class T>
class FastWeightedCollection : public WeightedCollection<T> {
public:
void Init(const std::vector<T>& ids,
bool Init(const std::vector<T>& ids,
const std::vector<float>& weights) override;

void Init(const std::vector<std::pair<T, float>>& id_weight_pairs) override;
bool Init(const std::vector<std::pair<T, float>>& id_weight_pairs) override;

std::pair<T, float> Sample() const override;

Expand All @@ -48,10 +48,10 @@ class FastWeightedCollection : public WeightedCollection<T> {
};

template<class T>
void FastWeightedCollection<T>::Init(const std::vector<T>& ids,
bool FastWeightedCollection<T>::Init(const std::vector<T>& ids,
const std::vector<float>& weights) {
if (ids.size() != weights.size()) {
return;
return false;
}
ids_.resize(ids.size());
weights_.resize(weights.size());
Expand All @@ -66,11 +66,11 @@ void FastWeightedCollection<T>::Init(const std::vector<T>& ids,
norm_weights[i] /= sum_weight_;
}
alias_.Init(norm_weights);
return;
return true;
}

template<class T>
void FastWeightedCollection<T>::Init(
bool FastWeightedCollection<T>::Init(
const std::vector<std::pair<T, float>>& id_weight_pairs) {
ids_.resize(id_weight_pairs.size());
weights_.resize(id_weight_pairs.size());
Expand All @@ -85,7 +85,7 @@ void FastWeightedCollection<T>::Init(
norm_weights[i] /= sum_weight_;
}
alias_.Init(norm_weights);
return;
return true;
}

template<class T>
Expand Down
4 changes: 2 additions & 2 deletions euler/common/weighted_collection.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ template <class T>
class WeightedCollection {
public:
virtual ~WeightedCollection() {}
virtual void Init(const std::vector<T>& ids,
virtual bool Init(const std::vector<T>& ids,
const std::vector<float>& weights) = 0;
virtual void Init(
virtual bool Init(
const std::vector<std::pair<T, float> >& id_weight_pairs) = 0;
virtual std::pair<T, float> Sample() const = 0;
virtual size_t GetSize() const = 0;
Expand Down
20 changes: 10 additions & 10 deletions euler/core/compact_edge.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ bool CompactEdge::DeSerialize(const char* s, size_t size) {
}
if (!bytes_reader.GetInt32(&type_) || // parse type
!bytes_reader.GetFloat(&weight_)) { // parse weight
LOG(ERROR) << "edge info error";
LOG(ERROR) << "edge info error, edge_id: " << src_id << "," << dst_id;
return false;
}

Expand All @@ -55,12 +55,12 @@ bool CompactEdge::DeSerialize(const char* s, size_t size) {
// parse uint64 feature
int32_t uint64_feature_type_num = 0;
if (!bytes_reader.GetInt32(&uint64_feature_type_num)) {
LOG(ERROR) << "uint64 feature type num error";
LOG(ERROR) << "uint64 feature type num error, edge_id: " << src_id << "," << dst_id;
return false;
}
if (!bytes_reader.GetInt32List(uint64_feature_type_num,
&uint64_features_idx_)) {
LOG(ERROR) << "uint64 feature idx list error";
LOG(ERROR) << "uint64 feature idx list error, edge_id: " << src_id << "," << dst_id;
return false;
}
int32_t uint64_fv_num = 0;
Expand All @@ -69,19 +69,19 @@ bool CompactEdge::DeSerialize(const char* s, size_t size) {
uint64_features_idx_[i] = uint64_fv_num;
}
if (!bytes_reader.GetUInt64List(uint64_fv_num, &uint64_features_)) {
LOG(ERROR) << "uint64 feature value list error";
LOG(ERROR) << "uint64 feature value list error, edge_id: " << src_id << "," << dst_id;
return false;
}

// parse float feature
int32_t float_feature_type_num = 0;
if (!bytes_reader.GetInt32(&float_feature_type_num)) {
LOG(ERROR) << "float feature type num error";
LOG(ERROR) << "float feature type num error, edge_id: " << src_id << "," << dst_id;
return false;
}
if (!bytes_reader.GetInt32List(float_feature_type_num,
&float_features_idx_)) {
LOG(ERROR) << "float feature idx list error";
LOG(ERROR) << "float feature idx list error, edge_id: " << src_id << "," << dst_id;
return false;
}
int32_t float_fv_num = 0;
Expand All @@ -90,19 +90,19 @@ bool CompactEdge::DeSerialize(const char* s, size_t size) {
float_features_idx_[i] = float_fv_num;
}
if (!bytes_reader.GetFloatList(float_fv_num, &float_features_)) {
LOG(ERROR) << "float feature value list error";
LOG(ERROR) << "float feature value list error, edge_id: " << src_id << "," << dst_id;
return false;
}

// parse binary feature
int32_t binary_feature_type_num = 0;
if (!bytes_reader.GetInt32(&binary_feature_type_num)) {
LOG(ERROR) << "binary feature type num error";
LOG(ERROR) << "binary feature type num error, edge_id: " << src_id << "," << dst_id;
return false;
}
if (!bytes_reader.GetInt32List(binary_feature_type_num,
&binary_features_idx_)) {
LOG(ERROR) << "binary feature idx list error";
LOG(ERROR) << "binary feature idx list error, edge_id: " << src_id << "," << dst_id;
return false;
}
int32_t binary_fv_num = 0;
Expand All @@ -111,7 +111,7 @@ bool CompactEdge::DeSerialize(const char* s, size_t size) {
binary_features_idx_[i] = binary_fv_num;
}
if (!bytes_reader.GetString(binary_fv_num, &binary_features_)) {
LOG(ERROR) << "binary feature value list error";
LOG(ERROR) << "binary feature value list error, edge_id: " << src_id << "," << dst_id;
return false;
}

Expand Down
36 changes: 18 additions & 18 deletions euler/core/compact_node.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@ bool CompactNode::DeSerialize(const char* s, size_t size) {

int32_t edge_group_num = 0;
if (!bytes_reader.GetInt32(&edge_group_num)) {
LOG(ERROR) << "edge group num error";
LOG(ERROR) << "edge group num error, node_id: " << id_;
return false;
}

Expand All @@ -293,20 +293,22 @@ bool CompactNode::DeSerialize(const char* s, size_t size) {
std::vector<int32_t> edge_group_size_list;
if (!bytes_reader.GetInt32List(edge_group_num,
&edge_group_size_list)) {
LOG(ERROR) << "edge group size list error";
LOG(ERROR) << "edge group size list error, node_id: " << id_;
return false;
}

std::vector<float> edge_group_weight_list;
if (!bytes_reader.GetFloatList(edge_group_num,
&edge_group_weight_list)) {
LOG(ERROR) << "edge group weight list error";
LOG(ERROR) << "edge group weight list error, node_id: " << id_;
return false;
}

// build edge_group_collection_
edge_group_collection_.Init(edge_group_ids, edge_group_weight_list);

if (!edge_group_collection_.Init(edge_group_ids, edge_group_weight_list)) {
LOG(ERROR) << "edge group collection error, node_id: " << id_;
return false;
}
// build neighbors info
int32_t total_neighbors_num = 0;
std::vector<std::vector<uint64_t>> ids_list(edge_group_num);
Expand All @@ -317,7 +319,7 @@ bool CompactNode::DeSerialize(const char* s, size_t size) {
ids_list[i] = std::vector<uint64_t>();
if (!bytes_reader.GetUInt64List(edge_group_size_list[i],
&ids_list[i])) {
LOG(ERROR) << "neighbor id list error";
LOG(ERROR) << "neighbor id list error, node_id: " << id_;
return false;
}
}
Expand All @@ -327,7 +329,7 @@ bool CompactNode::DeSerialize(const char* s, size_t size) {
weights_list[i] = std::vector<float>();
if (!bytes_reader.GetFloatList(edge_group_size_list[i],
&weights_list[i])) {
LOG(ERROR) << "neighbor weight list error";
LOG(ERROR) << "neighbor weight list error, node_id: " << id_;
return false;
}
}
Expand Down Expand Up @@ -356,16 +358,15 @@ bool CompactNode::DeSerialize(const char* s, size_t size) {
neighbors_weight_.push_back(sum_weight);
}
}

// parse uint64 feature
int32_t uint64_feature_type_num = 0;
if (!bytes_reader.GetInt32(&uint64_feature_type_num)) {
LOG(ERROR) << "uint64 feature type num error";
LOG(ERROR) << "uint64 feature type num error, node_id: " << id_;
return false;
}
if (!bytes_reader.GetInt32List(uint64_feature_type_num,
&uint64_features_idx_)) {
LOG(ERROR) << "uint64 feature idx list error";
LOG(ERROR) << "uint64 feature idx list error, node_id: " << id_;
return false;
}
int32_t uint64_fv_num = 0;
Expand All @@ -374,19 +375,19 @@ bool CompactNode::DeSerialize(const char* s, size_t size) {
uint64_features_idx_[i] = uint64_fv_num;
}
if (!bytes_reader.GetUInt64List(uint64_fv_num, &uint64_features_)) {
LOG(ERROR) << "uint64 feature value list error";
LOG(ERROR) << "uint64 feature value list error, node_id: " << id_;
return false;
}

// parse float feature
int32_t float_feature_type_num = 0;
if (!bytes_reader.GetInt32(&float_feature_type_num)) {
LOG(ERROR) << "float feature type num error";
LOG(ERROR) << "float feature type num error, node_id: " << id_;
return false;
}
if (!bytes_reader.GetInt32List(float_feature_type_num,
&float_features_idx_)) {
LOG(ERROR) << "float feature idx list error";
LOG(ERROR) << "float feature idx list error, node_id: " << id_;
return false;
}
int32_t float_fv_num = 0;
Expand All @@ -395,19 +396,19 @@ bool CompactNode::DeSerialize(const char* s, size_t size) {
float_features_idx_[i] = float_fv_num;
}
if (!bytes_reader.GetFloatList(float_fv_num, &float_features_)) {
LOG(ERROR) << "float feature value list error";
LOG(ERROR) << "float feature value list error, node_id: " << id_;
return false;
}

// parse binary feature
int32_t binary_feature_type_num = 0;
if (!bytes_reader.GetInt32(&binary_feature_type_num)) {
LOG(ERROR) << "binary feature type num error";
LOG(ERROR) << "binary feature type num error, node_id: " << id_;
return false;
}
if (!bytes_reader.GetInt32List(binary_feature_type_num,
&binary_features_idx_)) {
LOG(ERROR) << "binary feature idx list error";
LOG(ERROR) << "binary feature idx list error, node_id: " << id_;
return false;
}
int32_t binary_fv_num = 0;
Expand All @@ -416,10 +417,9 @@ bool CompactNode::DeSerialize(const char* s, size_t size) {
binary_features_idx_[i] = binary_fv_num;
}
if (!bytes_reader.GetString(binary_fv_num, &binary_features_)) {
LOG(ERROR) << "binary feature value list error";
LOG(ERROR) << "binary feature value list error, node_id: " << id_;
return false;
}

return true;
}

Expand Down
Loading

0 comments on commit 653bb01

Please sign in to comment.