rdma: refactor (#14)

See bytedance/ps-lite#14 for details.

Co-authored-by: Yibo Zhu <[email protected]>

Makefile

@@ -20,10 +20,10 @@ endif
 
 INCPATH = -I./src -I./include -I$(DEPS_PATH)/include
 CFLAGS = -std=c++14 -msse2 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS)
-LIBS = -pthread
+LIBS = -pthread -lrt
 
 ifeq ($(USE_RDMA), 1)
-LIBS += -lrdmacm -libverbs -lrt
+LIBS += -lrdmacm -libverbs
 CFLAGS += -DDMLC_USE_RDMA
 endif
 
@@ -38,25 +38,21 @@ include make/deps.mk
 
 clean:
 	rm -rf build $(TEST) tests/*.d tests/*.dSYM
-	find src -name "*.pb.[ch]*" -delete
 
 lint:
 	python tests/lint.py ps all include/ps src
 
 ps: build/libps.a
 
-OBJS = $(addprefix build/, customer.o postoffice.o van.o meta.pb.o)
+OBJS = $(addprefix build/, customer.o postoffice.o van.o)
 build/libps.a: $(OBJS)
 	ar crv $@ $(filter %.o, $?)
 
-build/%.o: src/%.cc ${ZMQ} src/meta.pb.h
+build/%.o: src/%.cc ${ZMQ}
 	@mkdir -p $(@D)
 	$(CXX) $(INCPATH) -std=c++0x -MM -MT build/$*.o $< >build/$*.d
 	$(CXX) $(CFLAGS) $(LIBS) -c $< -o $@
 
-src/%.pb.cc src/%.pb.h : src/%.proto ${PROTOBUF}
-	$(PROTOC) --cpp_out=./src --proto_path=./src $<
-
 -include build/*.d
 -include build/*/*.d
 

include/ps/internal/van.h

@@ -16,7 +16,6 @@
 #include "ps/internal/message.h"
 namespace ps {
 class Resender;
-class PBMeta;
 /**
  * \brief Van sends messages to remote nodes
  *
@@ -107,14 +106,14 @@ class Van {
   virtual int SendMsg(Message &msg) = 0;
 
   /**
-   * \brief pack meta into a string
+   * \brief get the length of pack meta
    */
-  void PackMeta(const Meta &meta, char **meta_buf, int *buf_size);
+  int GetPackMetaLen(const Meta &meta);
 
   /**
-   * \brief pack meta into protobuf
+   * \brief pack meta into a string
    */
-  void PackMetaPB(const Meta &meta, PBMeta *pb);
+  void PackMeta(const Meta &meta, char **meta_buf, int *buf_size);
 
   /**
    * \brief unpack meta from a string

make/deps.mk

@@ -1,21 +1,10 @@
 # Install dependencies
 
 URL1=https://raw.githubusercontent.com/mli/deps/master/build
-URL2=https://github.com/google/protobuf/releases/download/v3.5.1
 ifndef WGET
 WGET = wget
 endif
 
-# protobuf
-PROTOBUF = ${DEPS_PATH}/include/google/protobuf/message.h
-${PROTOBUF}:
-	$(eval FILE=protobuf-cpp-3.5.1.tar.gz)
-	$(eval DIR=protobuf-3.5.1)
-	rm -rf $(FILE) $(DIR)
-	$(WGET) $(URL2)/$(FILE) && tar --no-same-owner -zxf $(FILE)
-	cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install
-	rm -rf $(FILE) $(DIR)
-
 # zmq
 ZMQ = ${DEPS_PATH}/include/zmq.h
 

make/ps.mk

@@ -9,5 +9,5 @@ ifeq ($(USE_KEY32), 1)
 ADD_CFLAGS += -DUSE_KEY32=1
 endif
 
-PS_LDFLAGS_SO = -L$(DEPS_PATH)/lib -lprotobuf-lite -lzmq
-PS_LDFLAGS_A = $(addprefix $(DEPS_PATH)/lib/, libprotobuf-lite.a libzmq.a)
+PS_LDFLAGS_SO = -L$(DEPS_PATH)/lib -lzmq
+PS_LDFLAGS_A = $(addprefix $(DEPS_PATH)/lib/, libzmq.a)

src/meta.h

@@ -0,0 +1,76 @@
+/**
+ *  Copyright (c) 2018-2019 Bytedance Inc.
+ *  Author: [email protected] (Yibo Zhu)
+*/
+#ifndef PS_LITE_META_H_
+#define PS_LITE_META_H_
+
+#include<stdint.h>
+
+namespace ps {
+
+struct RawNode {
+  // the node role
+  int role;
+  // node id
+  int id;
+  // hostname or ip
+  char hostname[64];
+  // the port this node is binding
+  int port;
+  // whether this node is created by failover
+  bool is_recovery;
+  // the locally unique id of an customer
+  int customer_id;
+};
+
+// system control info
+struct RawControl {
+  int cmd;
+  int node_size;
+  int barrier_group;
+  uint64_t msg_sig;
+};
+
+// mete information about a message
+struct RawMeta {
+  // message.head
+  int head;
+  // message.body
+  int body_size;
+  // if set, then it is system control task. otherwise, it is for app
+  RawControl control;
+  // true: a request task
+  // false: the response task to the request task with the same *time*
+  bool request;
+  // the unique id of an application
+  int app_id;
+  // the timestamp of this message
+  int timestamp;
+  // data type of message.data[i]
+  int data_type_size;
+  // the locally unique id of an customer
+  int customer_id;
+  // whether or not a push message
+  bool push;
+  // whether or not it's for SimpleApp
+  bool simple_app;
+  // message.data_size 
+  int data_size;
+  // message.key
+  uint64_t key;
+  // message.addr
+  uint64_t addr;
+  // the length of the message's value
+  int val_len;
+  // the option field
+  int option;
+
+  // body
+  // data_type
+  // node
+};
+
+} // namespace
+
+#endif