From f02472f166e60676f7000d191e228e7fa794b1d4 Mon Sep 17 00:00:00 2001 From: zhongyuchen Date: Wed, 5 Feb 2020 21:51:35 +0800 Subject: [PATCH 1/3] Relax Check --- include/ps/kv_app.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ps/kv_app.h b/include/ps/kv_app.h index 50bdde6b..6d78e39f 100644 --- a/include/ps/kv_app.h +++ b/include/ps/kv_app.h @@ -686,7 +686,7 @@ int KVWorker::AddPullCB( if (vals->empty()) { vals->resize(total_val); } else { - CHECK_EQ(vals->size(), total_val); + CHECK_GE(vals->size(), total_val); } Val* p_vals = vals->data(); int *p_lens = nullptr; From 4b1f8e8377bdecb7330ed2ca4fb9327024f2a1e8 Mon Sep 17 00:00:00 2001 From: zhongyuchen Date: Mon, 10 Feb 2020 17:00:17 +0800 Subject: [PATCH 2/3] update docs --- docs/api.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/api.md b/docs/api.md index 3744989e..e655ff18 100644 --- a/docs/api.md +++ b/docs/api.md @@ -6,12 +6,10 @@ The data communicated are presented as key-value index and the value might be the according `float` gradient. 1. Basic synchronization functions: \ref ps::KVWorker::Push, \ref ps::KVWorker::Pull, and \ref ps::KVWorker::Wait - 2. Dynamic length value push and pull: \ref ps::KVWorker::VPush and \ref - ps::KVWorker::VPull - 3. Zero-copy versions: \ref ps::KVWorker::ZPush, \ref - ps::KVWorker::ZPull, \ref ps::KVWorker::ZVPush and \ref - ps::KVWorker::ZVPull + 2. Zero-copy versions: \ref ps::KVWorker::ZPush, \ref + ps::KVWorker::ZPull +To support dynamic length, pull operations(`Pull` and `ZPull`), do not require the buffer(`vals`) to be the same size as the total data size of pulling down. Larger buffer is allowed while `lens` records the actual size of each key. So the reliable way to read valid message is to read the length of the actual size. If you ensure that the data size of a key does not change during push or pull, you can verify it by checking whether `lens` of the key is equal to the fixed size. often server *i* handles the keys (feature indices) within the i-th segment of [0, uint64_max]. The server node allows user-defined handles to From d1b849d403f220107931cf01ebaa983b6ef5734a Mon Sep 17 00:00:00 2001 From: zhongyuchen Date: Thu, 13 Feb 2020 01:55:08 +0800 Subject: [PATCH 3/3] update docs --- docs/api.md | 2 +- include/ps/kv_app.h | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index e655ff18..510e68b4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -9,7 +9,7 @@ The data communicated are presented as key-value 2. Zero-copy versions: \ref ps::KVWorker::ZPush, \ref ps::KVWorker::ZPull -To support dynamic length, pull operations(`Pull` and `ZPull`), do not require the buffer(`vals`) to be the same size as the total data size of pulling down. Larger buffer is allowed while `lens` records the actual size of each key. So the reliable way to read valid message is to read the length of the actual size. If you ensure that the data size of a key does not change during push or pull, you can verify it by checking whether `lens` of the key is equal to the fixed size. +To support dynamic length, pull operations(`Pull` and `ZPull`), do not require the buffer(`vals`) to be the same size as the total data size of pulling down. Larger buffer is allowed while `lens` records the actual size of each key. So the reliable way to read a valid message is to read `lens` bytes. If you ensure that the data size of a key does not change during push or pull, you can verify it by checking whether `lens` of the key is equal to the fixed size. often server *i* handles the keys (feature indices) within the i-th segment of [0, uint64_max]. The server node allows user-defined handles to diff --git a/include/ps/kv_app.h b/include/ps/kv_app.h index 6d78e39f..c33de0bb 100644 --- a/include/ps/kv_app.h +++ b/include/ps/kv_app.h @@ -48,6 +48,12 @@ struct KVPairs { * \brief A worker node that can \ref Push (\ref Pull) key-value pairs to (from) server * nodes * + * 1. Basic synchronization functions: \ref ps::KVWorker::Push, + * \ref ps::KVWorker::Pull, and \ref ps::KVWorker::Wait + * + * 2. Zero-copy versions: \ref ps::KVWorker::ZPush, + * \ref ps::KVWorker::ZPull + * * \tparam Val the type of value, which should be primitive types such as * int32_t and float */ @@ -140,6 +146,12 @@ class KVWorker : public SimpleApp { * It's a non-blocking call. The actual pulling is finished, * namely \a vals (and \a lens) is filled with pulled values, only * if \ref Wait returns or the callback is called. + * + * Note that \a vals can be larger than the length of the pulled values. + * \a lens is the actual length of the pulled values. The reliable way to + * read a valid message is to read \a lens bytes. If you ensure that the + * data size of a key does not change during push or pull, you can verify + * it by checking whether \a lens of the key is equal to the fixed size. * * @param keys a list of keys, must be unique and sorted in increasing order * @param vals the buffer for the pulled values. It can be 0 size.